diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a8ca2bffe630a107f1c5bdefe2a0c27d94752b35..d611679b713eeac77e034ee7e8c639c70a97fcc7 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -5,7 +5,7 @@ variables:
AWS_SERVICE: indexer
AWS_SERVICE_GATEWAY: osdu-gateway
AWS_ENVIRONMENT: dev
- AWS_DEPLOY_TARGET: HELM
+ AWS_DEPLOY_TARGET: TF
AWS_EKS_DEPLOYMENT_NAME: os-indexer
GCP_BUILD_SUBDIR: provider/indexer-gcp
@@ -60,6 +60,7 @@ include:
file: "cloud-providers/gc-global.yml"
- local: "devops/gc/pipeline/override-stages.yml"
+ - local: "devops/aws/pipeline/override-stages.yml"
aws-test-java:
tags: ["aws-internal-test"]
diff --git a/NOTICE b/NOTICE
index 5f06149bfca6bae1674f9b4822af767828a713a6..ba7891fad64d315704b373871343c6a9acbd3352 100644
--- a/NOTICE
+++ b/NOTICE
@@ -549,6 +549,7 @@ The following software have components provided under the terms of this license:
- Redisson (from http://redisson.org)
- Retrofit (from https://github.com/square/retrofit, https://repo1.maven.org/maven2/com/squareup/retrofit2/retrofit)
- RxJava (from https://github.com/ReactiveX/RxJava)
+- Servlet API (from https://repo1.maven.org/maven2/org/mortbay/jetty/servlet-api)
- Servlet Specification 2.5 API (from http://jetty.mortbay.org, https://repo1.maven.org/maven2/org/mortbay/jetty/servlet-api-2.5)
- SnakeYAML (from http://code.google.com/p/snakeyaml/, http://www.snakeyaml.org, https://bitbucket.org/snakeyaml/snakeyaml)
- Spatial4J (from https://projects.eclipse.org/projects/locationtech.spatial4j)
@@ -892,6 +893,7 @@ The following software have components provided under the terms of this license:
- Jakarta Annotations API (from https://projects.eclipse.org/projects/ee4j.ca)
- Jakarta Bean Validation API (from https://beanvalidation.org)
- Jakarta XML Binding API (from https://repo1.maven.org/maven2/jakarta/xml/bind/jakarta.xml.bind-api, https://repo1.maven.org/maven2/org/jboss/spec/javax/xml/bind/jboss-jaxb-api_2.3_spec)
+- Jetty :: Utilities (from http://jetty.mortbay.org, http://www.eclipse.org/jetty, https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-util, https://repo1.maven.org/maven2/org/mortbay/jetty/jetty-util)
- Logback Contrib :: JSON :: Classic (from https://repo1.maven.org/maven2/ch/qos/logback/contrib/logback-json-classic)
- Logback Contrib :: JSON :: Core (from https://repo1.maven.org/maven2/ch/qos/logback/contrib/logback-json-core)
- Logback Contrib :: Jackson (from https://repo1.maven.org/maven2/ch/qos/logback/contrib/logback-jackson)
diff --git a/devops/aws/chart/README.md b/devops/aws/chart/README.md
index 24f2071dc160e5d12f0e9c1c31b4ab594f4d4744..7035aa7f5a1c3f65a3bfe135a9841678e35784e2 100644
--- a/devops/aws/chart/README.md
+++ b/devops/aws/chart/README.md
@@ -1,14 +1,14 @@
-# Helm Chart
+# OSDU on AWS Service Helm Chart
## Introduction
The following document outlines how to deploy and update the service application onto an existing Kubernetes deployment using the [Helm](https://helm.sh) package manager.
## Prerequisites
The below software must be installed before continuing:
-* [AWS CLI ^2.7.0](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html)
-* [kubectl 1.21-1.22](https://kubernetes.io/docs/tasks/tools/)
-* [Helm ^3.7.1](https://helm.sh/docs/intro/install/)
-* [Helm S3 Plugin ^0.12.0](https://github.com/hypnoglow/helm-s3)
+* [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html)
+* [kubectl](https://kubernetes.io/docs/tasks/tools/)
+* [Helm](https://helm.sh/docs/intro/install/)
+* [Helm S3 Plugin](https://github.com/hypnoglow/helm-s3)
Additionally, an OSDU on AWS environment must be deployed.
@@ -34,20 +34,55 @@ To revert to a previous release, use the following command:
helm rollback [RELEASE] [REVISION] -n [NAMESPACE]
```
-### Customizing the Deployment
+Refer to the [Helm CLI guide](https://helm.sh/docs/helm/helm/) for additional commands.
+
+## Customizing the Deployment
It is possible to modify the default values specified in the **values.yaml** file using the --set option. The below parameters can be modified by advanced users to customize the deployment configuration:
+### Globals
+Global Helm values apply to all services within the parent chart deployment. Global values will not override service defaults or locally set values.
| Name | Example Value | Description | Type | Required |
| --- | ------------- | ----------- | ---- | -------- |
-| `global.accountID` | `000123456789` | The AWS account ID. | int | yes |
-| `global.region` | `us-east-1` | The AWS region containing the OSDU deployment. | str | yes |
-| `global.resourcePrefix` | `osdu` | The resource prefix of the OSDU deployment. | str | yes |
-| `podAnnotations` | `podAnnotations.version=v1.0.0` | Additional annotations on the service pod | dict | no |
+| `global.allowOrigins` | `{http://localhost,https://www.osdu.aws}` | A list of domains that are permitted by CORS policy. An empty list permits all origins. | array[str] | no |
+| `global.metricsServerAddress` | `http://prometheus-service.monitoring:8080` | The URL of the accessible metrics server for evaluating autoscaling decisions. | str | no |
+| `global.podAnnotations` | `podAnnotations.version=v1.0.0` | Additional annotations on the service pod | dict | no |
+| `global.podSecurityContext` | `fsGroup: 1337` | The [pod security context](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) apply to all containers in the pod | str | no |
+| `global.securityContext` | `fsGroup: 1337` | The security context is the container specific security context. Will inherit [pod security context](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) | str | no |
+
+### Local
+Local Helm values apply to specific services. Local Helm values will override global values and default presets.
+| Name | Example Value | Description | Type | Required |
+| --- | ------------- | ----------- | ---- | -------- |
+| `image` | `registry.repo.osdu.aws/service:0.21.0` | The custom image of the service deployment. | str | no |
| `imagePullPolicy` | `IfNotPresent` | The service image pull policy | str | no |
-| `replicaCount` | `1` | The number of pod replicas to be deployed | int | no |
+| `resources.limits.cpu` | `500M` | [CPU resource management limit for pods](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) | str | no |
+| `resources.limits.memory` | `900M` | [Memory resource management limit for pods](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) | str | no |
+| `resources.requests.cpu` | `500M` | [MemoCPUry resource management for pods](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) | str | no |
+| `resources.requests.memory` | `900M` | [Memory resource management for pods](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) | str | no |
+| `replicaCount` | `1` | The number of pod replicas to be initially deployed | int | no |
| `autoscaling.minReplicas` | `1` | Minimum number of pod replicas | int | no |
| `autoscaling.maxReplicas` | `100` | Maximum number of pod replicas | int | no |
| `autoscaling.targetCPUUtilizationPercentage` | `80` | CPU utilization target | int | no |
+| `autoscaling.targetMemoryUtilizationPercentage` | `80` | Memory utilization target | int | no |
+| `autoscaling.ServiceRequestCountThreshold` | `25` | The number of requests per second threshold averaged over a minute to trigger a scaling event. | int | no |
+| `autoscaling.ServiceRequestDurationAverage` | `300` | The response time measured in miliseconds averaged over 3 minutes to trigger a scaling event. | int | no |
+| `autoscaling.coolDownPeriod` | `120` | The period to wait after the last trigger reported active before scaling the resource back to 0. Managed by Keda. | int | no |
+| `autoscaling.pollingInterval` | `1` | This is the interval to check each trigger on. | int | no |
+| `livenessProbe.failureThreshold` | `3` | [Kubernetes probe configuration](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes). | int | no |
+| `livenessProbe.periodSeconds` | `10` | [Kubernetes probe configuration](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes). | int | no |
+| `livenessProbe.successThreshold` | `1` | [Kubernetes probe configuration](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes). | int | no |
+| `livenessProbe.timeoutSeconds` | `1` | [Kubernetes probe configuration](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes). | int | no |
+| `readinessProbe.initialDelaySeconds` | `30` | [Kubernetes probe configuration](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes). | int | no |
+| `readinessProbe.failureThreshold` | `3` | [Kubernetes probe configuration](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes). | int | no |
+| `readinessProbe.periodSeconds` | `10` | [Kubernetes probe configuration](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes). | int | no |
+| `readinessProbe.successThreshold` | `1` | [Kubernetes probe configuration](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes). | int | no |
+| `readinessProbe.timeoutSeconds` | `1` | [Kubernetes probe configuration](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes). | int | no |
+| `maxPendingRequests` | `10000` | Maximum number of requests that will be queued while waiting for a ready connection pool connection. Used for circuit breaking. Used for [circuit breaking.](https://istio.io/latest/docs/tasks/traffic-management/circuit-breaking/). | int | no |
+| `maxRequestsPerConnection` | `100` | Maximum number of active requests to a destination. Used for [circuit breaking.](https://istio.io/latest/docs/tasks/traffic-management/circuit-breaking/). | int | no |
+| `maxConnections` | `0` | Maximum number of HTTP1 /TCP connections to a destination host. Used for [circuit breaking.](https://istio.io/latest/docs/tasks/traffic-management/circuit-breaking/). | int | no |
+| `podAnnotations` | `podAnnotations.version=v1.0.0` | Additional annotations on the service pod | dict | no |
+| `podSecurityContext` | `fsGroup: 1337` | The [pod security context](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) apply to all containers in the pod | str | no |
+| `securityContext` | `fsGroup: 1337` | The security context is the container specific security context. Will inherit [pod security context](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) | str | no |
## Uninstalling the Chart
To uninstall the helm release:
diff --git a/devops/aws/chart/templates/hpa.yaml b/devops/aws/chart/templates/hpa.yaml
deleted file mode 100644
index a1dbd529707dc8f782e9880fc6a67189a12de6fd..0000000000000000000000000000000000000000
--- a/devops/aws/chart/templates/hpa.yaml
+++ /dev/null
@@ -1,4 +0,0 @@
-{{- template "common.hpa" (list . "os-indexer.hpa") -}}
-{{- define "os-indexer.hpa" -}}
-## Define overrides for the service HPA resource here
-{{- end -}}
diff --git a/devops/aws/chart/templates/scaledobject.yaml b/devops/aws/chart/templates/scaledobject.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..dac44558cab0a44447cd9ef499ec8aa99d7fd368
--- /dev/null
+++ b/devops/aws/chart/templates/scaledobject.yaml
@@ -0,0 +1,4 @@
+{{- template "common.scaledobject" (list . "os-indexer.scaledobject") -}}
+{{- define "os-indexer.scaledobject" -}}
+## Define overrides for the service's scaled object resource here
+{{- end -}}
\ No newline at end of file
diff --git a/devops/aws/chart/values.schema.json b/devops/aws/chart/values.schema.json
index d01920940ba1ae3ac7b6078c7321bf8042d24eca..105edd0d286f491b2bec56f07a65afa6f14bc047 100644
--- a/devops/aws/chart/values.schema.json
+++ b/devops/aws/chart/values.schema.json
@@ -1,12 +1,10 @@
{
- "$schema": "osdu-aws-services-helm",
+ "$schema": "osdu-aws-services-public",
"type": "object",
"default": {},
"required": [
"image",
- "imagePullPolicy",
"service",
- "replicaCount",
"serviceAccountRole"
],
"properties": {
@@ -22,8 +20,6 @@
"service": {
"type": "object",
"required": [
- "type",
- "port",
"apiPath"
],
"properties": {
@@ -260,8 +256,7 @@
"type": "string",
"title": "Allowed principal",
"examples": [
- "cluster.local/ns/istio-system/sa/istio-ingressgateway",
- "cluster.local/ns/osdu-services/sa/compliance-queue"
+ "cluster.local/ns/istio-system/sa/istio-ingressgateway"
]
}
}
diff --git a/devops/aws/chart/values.yaml b/devops/aws/chart/values.yaml
index 3712471b6e607f5f01078378726f9d3181f137d4..f4717013a28647882e57a8ccad83bfc2cf0d7429 100644
--- a/devops/aws/chart/values.yaml
+++ b/devops/aws/chart/values.yaml
@@ -1,110 +1,2 @@
# Service Config
image: __CONTAINER__
-imagePullPolicy: Always
-service:
- type: ClusterIP
- port: 8080
- apiPath: /api/indexer/v2/
- readinessType: exec
- readinessProbeHttpPath: /api/indexer/v2/liveness_check
- startupProbeHttpPath: /api/indexer/v2/liveness_check
-
-serviceParameters:
-- objectName: "/osdu/{{ .Values.global.resourcePrefix }}/storage/storage-sqs-url"
- objectType: "ssmparameter"
- objectAlias: storage-sqs-url
-- objectName: "/osdu/{{ .Values.global.resourcePrefix }}/indexer/indexer-sns-topic-arn"
- objectType: "ssmparameter"
- objectAlias: indexer-sns-topic-arn
-- objectName: "/osdu/{{ .Values.global.resourcePrefix }}/indexer-queue/indexer-deadletter-queue-sqs-url"
- objectType: "ssmparameter"
- objectAlias: indexer-deadletter-queue-sqs-url
-- objectName: "/osdu/{{ .Values.global.resourcePrefix }}/elasticsearch/end-point"
- objectType: "ssmparameter"
- objectAlias: "elasticsearch_host"
-- objectName: "/osdu/{{ .Values.global.resourcePrefix }}/elasticsearch/end-point-port"
- objectType: "ssmparameter"
- objectAlias: "elasticsearch_port"
-- objectName: "/osdu/{{ .Values.global.resourcePrefix }}/elasticsearch/credentials"
- objectType: "secretsmanager"
- objectAlias: "elasticsearch_credentials"
-- objectName: "/osdu/{{ .Values.global.resourcePrefix }}/redis/redisauthtoken"
- objectType: "secretsmanager"
- objectAlias: "CACHE_CLUSTER_KEY"
-- objectName: "/osdu/{{ .Values.global.resourcePrefix }}/redis-core/end-point"
- objectType: "ssmparameter"
- objectAlias: "CACHE_CLUSTER_ENDPOINT"
-- objectName: "/osdu/{{ .Values.global.resourcePrefix }}/redis-core/end-point-port"
- objectType: "ssmparameter"
- objectAlias: "CACHE_CLUSTER_PORT"
-environmentVariables:
- - name: APPLICATION_PORT
- value: "{{ .Values.service.port }}"
- - name: AWS_REGION
- value: "{{ .Values.global.region }}"
- - name: ENVIRONMENT
- value: "{{ .Values.global.resourcePrefix }}"
- - name: JAVA_OPTS
- value: "-Xms538M -Xmx900M"
- - name: LOG_LEVEL
- value: "{{ default `INFO` .Values.global.logLevel }}"
- - name: SSM_ENABLED
- value: "True"
- - name: SSL_ENABLED
- value: "false"
- - name: ENTITLEMENTS_BASE_URL
- value: "http://os-entitlements:8080"
- - name: PARTITION_BASE_URL
- value: "http://os-partition:8080"
- - name: STORAGE_BASE_URL
- value: "http://os-storage:8080"
- - name: SCHEMA_BASE_URL
- value: "http://os-schema:8080"
- - name: ELASTIC_DISABLE_CERTIFICATE_TRUST
- value: "true"
- - name: PARAMETER_MOUNT_PATH
- value: "/mnt/params"
- - name: TMP_VOLUME_PATH
- value: "/tmp"
- - name: SEARCH_BASE_URL
- value: "http://os-search:8080"
-
-# Resource Config
-replicaCount: 1
-resources:
- limits:
- memory: 1200M
- requests:
- cpu: 500m
- memory: 1200M
-autoscaling:
- enabled: true
- minReplicas: 1
- maxReplicas: 100
- targetCPUUtilizationPercentage: 80
- # targetMemoryUtilizationPercentage: 80
-
-# Security Config
-serviceAccountRole: arn:aws:iam::{{ .Values.global.accountID }}:role/osdu-{{ .Values.global.resourcePrefix }}-{{ .Values.global.region }}-{{ include "common.name" . }}
-securityContext:
- runAsUser: 10001
- runAsNonRoot: true
- readOnlyRootFilesystem: true
- allowPrivilegeEscalation: false
- capabilities:
- drop:
- - ALL
-podSecurityContext:
- fsGroup: 1337
- seccompProfile:
- type: RuntimeDefault
-allowedPrincipals:
- - cluster.local/ns/istio-system/sa/istio-ingressgateway
- - cluster.local/ns/{{ .Release.Namespace }}/sa/indexer-queue
- - cluster.local/ns/aws-binary-dms/sa/binary-dms
- - cluster.local/ns/osdu-airflow/sa/airflow-dag-upload
- - cluster.local/ns/osdu-ingest/sa/os-data-workflow
- - cluster.local/ns/osdu-ingest/sa/os-ingestion-workflow
- - cluster.local/ns/osdu-seismic-ddms/sa/os-seismic-store
- - cluster.local/ns/osdu-well-delivery/sa/os-welldelivery
- - cluster.local/ns/osdu-wellbore-ddms/sa/os-wellbore-ddms
diff --git a/devops/aws/pipeline/override-stages.yml b/devops/aws/pipeline/override-stages.yml
new file mode 100644
index 0000000000000000000000000000000000000000..0c48375d5baf61c09aceda46477925e53750d0d7
--- /dev/null
+++ b/devops/aws/pipeline/override-stages.yml
@@ -0,0 +1,40 @@
+aws-test-java:
+ extends:
+ - .maven
+ - .aws
+ - .aws_common_variables
+ - .aws_variables
+ stage: integration
+ needs: [{ job: 'aws-update-tf', optional: true }, { job: 'aws-update-helm', optional: true }, { job: 'aws-update-eks', optional: true }]
+ retry: 1
+ before_script:
+ - !reference [.maven, before_script]
+ - !reference [.aws, before_script]
+ - !reference [.aws_variables, before_script]
+ script:
+ - echo os-indexer override
+ - export ELASTIC_HOST=localhost
+ - export KUBECONFIG=/tmp/kubeconfig-${RANDOM}.yaml
+ - aws eks update-kubeconfig --region $AWS_REGION --name $EKS_CLUSTER_NAME --role-arn $EKS_CLUSTER_MGMT_ROLE
+
+ - localPort=$ELASTIC_PORT
+ - echo $localPort
+ - kubectl port-forward -n $TENANT_GROUP_NAME-tenant-$EKS_TENANT_NAME-elasticsearch svc/elasticsearch-es-http $localPort:$ELASTIC_PORT > /dev/null 2>&1 &
+ - export ELASTIC_PORT=$localPort
+ - pid=$!
+ - |
+ trap '{
+ echo killing "Port forward process: "$pid
+ kill $pid
+ rm $KUBECONFIG
+ }' EXIT
+ - $MAVEN_BUILD $INTEGRATION_TEST_DIR maven-aws-integration-test-output.txt ${AWS_MAVEN_TEST_COMMAND_OVERRIDE:-test} --update-snapshots -DdisableXmlReport=true
+ only:
+ variables:
+ #Default if not defined
+ - $AWS_SKIP_DEPLOY != 'true' && $AWS_SKIP_TESTS != 'true' && $AWS == '1' && ($AWS_INT_TEST_TYPE == 'java' || $AWS_INT_TEST_TYPE == null)
+ artifacts:
+ when: always
+ paths:
+ - $INTEGRATION_TEST_DIR
+ expire_in: 2 days
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SearchServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SearchServiceImpl.java
index 8fba37d728670f33d8a5a96612bd26900a2c7583..fb76fb49ae6a12fec77ba5d25e5b65d7a91fe3d1 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SearchServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SearchServiceImpl.java
@@ -76,7 +76,7 @@ public class SearchServiceImpl implements SearchService {
return gson.fromJson(response.getBody(), SearchResponse.class);
}
else {
- jaxRsDpsLog.error("SearchService", response.getException());
+ jaxRsDpsLog.error(String.format("Search service: failed to call the search service: %d", response.getResponseCode()));
return new SearchResponse();
}
}
diff --git a/provider/indexer-aws/build-aws/buildspec.yaml b/provider/indexer-aws/build-aws/buildspec.yaml
index 79089e2939c45cd82cba06e20526a19e4d93181f..9fb114e537457c8e2fd1fafa5479361759f9412a 100644
--- a/provider/indexer-aws/build-aws/buildspec.yaml
+++ b/provider/indexer-aws/build-aws/buildspec.yaml
@@ -82,7 +82,6 @@ phases:
- echo "Building integration testing assemblies and gathering artifacts..."
- ./testing/indexer-test-aws/build-aws/prepare-dist.sh
- - sed -i "s|__CHART_VERSION__|$VERSION|" devops/aws/chart/Chart.yaml
- sed -i "s|__CONTAINER__|$ECR_IMAGE|" devops/aws/chart/values.yaml
- echo "--Copying Helm Charts to ${OUTPUT_DIR:-dist}--"
- rsync -r devops/aws/* "${OUTPUT_DIR:-dist}"
diff --git a/provider/indexer-aws/pom.xml b/provider/indexer-aws/pom.xml
index a1487720ee43997c12f46d62575342ce74d75006..9836c17ca9691b8846dddce43635db3d626ad4ac 100644
--- a/provider/indexer-aws/pom.xml
+++ b/provider/indexer-aws/pom.xml
@@ -71,7 +71,7 @@
<dependency>
<groupId>org.opengroup.osdu.core.aws</groupId>
<artifactId>os-core-lib-aws</artifactId>
- <version>0.21.0-rc1</version>
+ <version>0.21.0-rc5</version>
</dependency>
<!-- AWS managed packages -->
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/AwsServiceAccountAuthToken.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/AwsServiceAccountAuthToken.java
index b875de3cad198022045d6071293980c30464fbb0..f562dd260aa0eab16bc24dfc3902d2363af82a65 100644
--- a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/AwsServiceAccountAuthToken.java
+++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/AwsServiceAccountAuthToken.java
@@ -14,77 +14,44 @@
package org.opengroup.osdu.indexer.aws.util;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.services.simplesystemsmanagement.AWSSimpleSystemsManagement;
-import com.amazonaws.services.simplesystemsmanagement.AWSSimpleSystemsManagementClientBuilder;
-import com.amazonaws.services.simplesystemsmanagement.model.GetParameterRequest;
-import com.amazonaws.services.simplesystemsmanagement.model.GetParameterResult;
-import org.opengroup.osdu.core.aws.iam.IAMConfig;
-import org.opengroup.osdu.core.aws.secrets.SecretsManager;
-
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Component;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.google.gson.Gson;
+import com.google.gson.JsonSyntaxException;
+import org.apache.http.HttpStatus;
+import org.opengroup.osdu.core.aws.ssm.K8sLocalParameterProvider;
+import org.opengroup.osdu.core.aws.ssm.K8sParameterNotFoundException;
import org.opengroup.osdu.core.common.http.HttpClient;
import org.opengroup.osdu.core.common.http.HttpRequest;
import org.opengroup.osdu.core.common.http.HttpResponse;
import org.opengroup.osdu.core.common.http.IHttpClient;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.PostConstruct;
import java.util.Base64;
import java.util.HashMap;
import java.util.Map;
-import com.google.gson.Gson;
-import com.google.gson.JsonSyntaxException;
-import javax.annotation.PostConstruct;
-
@Component
public class AwsServiceAccountAuthToken {
-
- @Value("${ENVIRONMENT}")
- private String awsEnvironment;
-
- @Value("${AWS_REGION}")
- private String awsRegion;
-
- private final static String ENVIRONMENT = "ENVIRONMENT";
- private final static String REGION = "AWS_REGION";
-
private String client_credentials_secret;
private String client_credentials_clientid;
private String tokenUrl;
private String oauthCustomScope;
private String token= null;
private long expirationTimeMillis;
- private AWSSimpleSystemsManagement ssmManager;
@PostConstruct
private void init() {
- SecretsManager sm = new SecretsManager();
- String environment = awsEnvironment;
- if (environment == null) {
- environment = System.getProperty(ENVIRONMENT, System.getenv(ENVIRONMENT));
- }
- String amazonRegion = awsRegion;
- if (amazonRegion == null) {
- amazonRegion = System.getProperty(REGION, System.getenv(REGION));
- }
-
- String oauth_token_url = "/osdu/" + environment + "/oauth-token-uri";
- String oauth_custom_scope = "/osdu/" + environment + "/oauth-custom-scope";
-
- String client_credentials_client_id = "/osdu/" + environment + "/client-credentials-client-id";
- String client_secret_key = "client_credentials_client_secret";
- String client_secret_secretName = "/osdu/" + environment + "/client_credentials_secret";
- AWSCredentialsProvider amazonAWSCredentials = IAMConfig.amazonAWSCredentials();
- this.ssmManager= AWSSimpleSystemsManagementClientBuilder.standard()
- .withCredentials(amazonAWSCredentials)
- .withRegion(amazonRegion)
- .build();
-
- this.client_credentials_clientid = getSsmParameter(client_credentials_client_id);
- this.client_credentials_secret = sm.getSecret(client_secret_secretName, amazonRegion, client_secret_key);
- this.tokenUrl = getSsmParameter(oauth_token_url);
- this.oauthCustomScope = getSsmParameter(oauth_custom_scope);
+ K8sLocalParameterProvider provider = new K8sLocalParameterProvider();
+ try {
+ this.client_credentials_clientid = provider.getParameterAsString("CLIENT_CREDENTIALS_ID");
+ this.client_credentials_secret = provider.getCredentialsAsMap("CLIENT_CREDENTIALS_SECRET").get("client_credentials_client_secret");
+ this.tokenUrl = provider.getParameterAsString("OAUTH_TOKEN_URI");
+ this.oauthCustomScope = provider.getParameterAsString("OAUTH_CUSTOM_SCOPE");
+ } catch (K8sParameterNotFoundException | JsonProcessingException e) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "ParameterNotFoundException", e.getMessage());
+ }
}
public String getAuthToken() throws AppException {
@@ -136,10 +103,4 @@ public class AwsServiceAccountAuthToken {
throw new Exception("Invalid Response");
}
}
-
- private String getSsmParameter(String parameterKey) {
- GetParameterRequest paramRequest = (new GetParameterRequest()).withName(parameterKey).withWithDecryption(true);
- GetParameterResult paramResult = ssmManager.getParameter(paramRequest);
- return paramResult.getParameter().getValue();
- }
}
diff --git a/provider/indexer-aws/src/main/resources/application.properties b/provider/indexer-aws/src/main/resources/application.properties
index 2d9d7d6efe70e50168eef5cbbd8b5b965282ddc2..7e4f362ba85465e3f1d61006b3e5af6ba3b4f9f4 100644
--- a/provider/indexer-aws/src/main/resources/application.properties
+++ b/provider/indexer-aws/src/main/resources/application.properties
@@ -5,8 +5,6 @@ AUTHORIZE_API=${ENTITLEMENTS_BASE_URL}/api/entitlements/v2
logging.level.org.springframework.web=${LOG_LEVEL}
server.port=${APPLICATION_PORT}
-aws.environment=${ENVIRONMENT}
-
aws.threads=50
DEFAULT_DATA_COUNTRY=US
CRON_INDEX_CLEANUP_THRESHOLD_DAYS=3
@@ -51,14 +49,7 @@ MAX_CACHE_VALUE_SIZE=1000
## AWS Lambda configuration
#aws.lambda.get-groups-function-name=${ENVIRONMENT}-os-entitlements-GroupsFunction
-
-## Default DynamoDB Settings
-aws.dynamodb.table.prefix=${ENVIRONMENT}-
-aws.dynamodb.endpoint=dynamodb.${AWS_REGION}.amazonaws.com
-
aws.ssm=${SSM_ENABLED}
-aws.ssm.prefix=/osdu/${ENVIRONMENT}
-aws.parameter.prefix=/osdu/${ENVIRONMENT}
server.ssl.enabled=${SSL_ENABLED:true}
server.ssl.key-store-type=PKCS12
diff --git a/provider/indexer-azure/pom.xml b/provider/indexer-azure/pom.xml
index 3ae26a7809760cdb5a0478f946872f920fabbe23..9533622b4fc072ad626e2559dd4844c0121b4a8c 100644
--- a/provider/indexer-azure/pom.xml
+++ b/provider/indexer-azure/pom.xml
@@ -219,6 +219,11 @@
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty</artifactId>
+ <version>6.1.23</version>
+ </dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
diff --git a/provider/indexer-gc/pom.xml b/provider/indexer-gc/pom.xml
index 11f9db99c0c7eef965e60a2738bb9b7a4d04baa9..634032280d30488dc829882408020011869d670d 100644
--- a/provider/indexer-gc/pom.xml
+++ b/provider/indexer-gc/pom.xml
@@ -69,6 +69,11 @@
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty</artifactId>
+ <version>6.1.23</version>
+ </dependency>
<!-- Test Dependencies -->
<dependency>
diff --git a/testing/indexer-test-aws/build-aws/run-tests.sh b/testing/indexer-test-aws/build-aws/run-tests.sh
index b641359ac150de12cc6b8657ab16e3cd88ed17f3..e2fd0042e142c2b469f99cc3de98103cae5d532c 100755
--- a/testing/indexer-test-aws/build-aws/run-tests.sh
+++ b/testing/indexer-test-aws/build-aws/run-tests.sh
@@ -28,11 +28,32 @@ export ENTITLEMENTS_DOMAIN=example.com
export OTHER_RELEVANT_DATA_COUNTRIES=US
export STORAGE_HOST=$STORAGE_URL
export HOST=$SCHEMA_URL
-export ELASTIC_HOST=$ELASTIC_HOST
-export ELASTIC_PORT=$ELASTIC_PORT
+export ELASTIC_HOST=localhost
export ELASTIC_PASSWORD=$ELASTIC_PASSWORD
export ELASTIC_USER_NAME=$ELASTIC_USERNAME
+################ Elastic search port forwarding ########
+##Check if port is available
+localPort=$ELASTIC_PORT
+while netstat -an | grep $localPort | grep -i listen ; do
+ echo "$localPort Port in use"
+ ((localPort++))
+done
+echo "Using local port: "$localPort
+
+export KUBECONFIG=/tmp/kubeconfig-int-test$(date +%s).yaml
+aws eks update-kubeconfig --name $EKS_CLUSTER_NAME --region $AWS_REGION --role-arn $CLUSTER_MANAGEMENT_ROLE_ARN
+kubectl port-forward -n $TENANT_GROUP_NAME-tenant-$EKS_TENANT_NAME-elasticsearch svc/elasticsearch-es-http $localPort:$ELASTIC_PORT > /dev/null 2>&1 &
+
+export ELASTIC_PORT=$localPort
+pid=$!
+
+trap '{
+ echo killing "Port forward process: "$pid
+ kill $pid
+ rm $KUBECONFIG
+}' EXIT
+
#### RUN INTEGRATION TEST #########################################################################
mvn -ntp test -f "$SCRIPT_SOURCE_DIR"/../pom.xml -Dcucumber.options="--plugin junit:target/junit-report.xml"
diff --git a/testing/indexer-test-aws/pom.xml b/testing/indexer-test-aws/pom.xml
index 4873e4dd208a357a1edd026e1c49b299ca5dded0..de8b92b86ab78d93d3763a30abd352ad5ec49fbe 100644
--- a/testing/indexer-test-aws/pom.xml
+++ b/testing/indexer-test-aws/pom.xml
@@ -51,7 +51,7 @@
<dependency>
<groupId>org.opengroup.osdu.core.aws</groupId>
<artifactId>os-core-lib-aws</artifactId>
- <version>0.19.0-rc3</version>
+ <version>0.21.0-rc5</version>
</dependency>
<!-- Testing -->