diff --git a/provider/indexer-aws/README.md b/provider/indexer-aws/README.md
index c7ce9c86485c059e53af5bea69c9b432d0e9f853..8838d881da45f7b3080af10d6cda15c67618307d 100644
--- a/provider/indexer-aws/README.md
+++ b/provider/indexer-aws/README.md
@@ -13,28 +13,44 @@ Pre-requisites
* Lombok 1.28 or later
* OSDU Instance deployed on AWS
-### Service Configuration
-In order to run the service locally or remotely, you will need to have the following environment variables defined.
-
-| name | example value | required | description | sensitive? |
-| --- | --- | --- | --- | --- |
-| `LOCAL_MODE` | `true` | yes | Set to 'true' to use env vars in place of the k8s variable resolver | no |
-| `APPLICATION_PORT` | `8080` | yes | The port the service will be hosted on. | no |
-| `AWS_REGION` | `us-east-1` | yes | The region where resources needed by the service are deployed | no |
-| `AWS_ACCESS_KEY_ID` | `ASIAXXXXXXXXXXXXXX` | yes | The AWS Access Key for a user with access to Backend Resources required by the service | yes |
-| `AWS_SECRET_ACCESS_KEY` | `super-secret-key==` | yes | The AWS Secret Key for a user with access to Backend Resources required by the service | yes |
-| `AWS_SESSION_TOKEN` | `session-token-xxxxxxxxxx` | no | AWS Session token needed if using an SSO user session to authenticate | yes |
-| `ENVIRONMENT` | `osdu-prefix` | yes | The Resource Prefix defined during deployment | no |
-| `LOG_LEVEL` | `DEBUG` | yes | The Log Level severity to use (https://www.tutorialspoint.com/log4j/log4j_logging_levels.htm) | no |
-| `SSM_ENABLED` | `true` | yes | Set to 'true' to use SSM to resolve config properties, otherwise use env vars | no |
-| `SSL_ENABLED` | `false` | no | Set to 'false' to disable SSL for local development | no |
-| `ENTITLEMENTS_BASE_URL` | `http://localhost:8081` or `https://some-hosted-url` | yes | Specify the base url for an entitlements service instance. Can be run locally or remote | no |
-| `PARTITION_BASE_URL` | `http://localhost:8082` or `https://some-hosted-url` | yes | Specify the base url for a partition service instance. Can be run locally or remote | no |
-| `STORAGE_BASE_URL` | `http://localhost:8082` or `https://some-hosted-url` | yes | Specify the base url for a partition service instance. Can be run locally or remote | no |
-| `SCHEMA_BASE_URL` | `http://localhost:8082` or `https://some-hosted-url` | yes | Specify the base url for a partition service instance. Can be run locally or remote | no |
-| `STORAGE_RECORDS_BATCH_SIZE` | 20 | Batch size for storage API `POST {endpoint}/query/records:batch` | no | - |
-| `STORAGE_RECORDS_BY_KIND_BATCH_SIZE` | - | Batch size for storage API `GET {endpoint}/query/records`. If this is not present, defaults to value of `STORAGE_RECORDS_BATCH_SIZE` | no | - |
-
+### Run Locally
+In order to run the service locally, you will need to have the following environment variables defined.
+To run the service remotely, please refer to the Helm Charts defined in the `indexer.tf` file of the `aws-terraform-deployment` repository.
+
+| name | example value | required | description | sensitive? |
+|--------------------------------------|--------------------------------------------------------------------------------|----------|---------------------------------------------------------------------------------------------------------------------------------------|------------|
+| `LOCAL_MODE` | `true` | yes | Set to 'true' to use env vars in place of the k8s variable resolver | no |
+| `APPLICATION_PORT` | `8080` | yes | The port the service will be hosted on. | no |
+| `AWS_REGION` | `us-east-1` | yes | The region where resources needed by the service are deployed | no |
+| `LOG_LEVEL` | `DEBUG` | yes | The Log Level severity to use (https://www.tutorialspoint.com/log4j/log4j_logging_levels.htm) | no |
+| `SSM_ENABLED` | `true` | yes | Set to 'true' to use SSM to resolve config properties, otherwise use env vars | no |
+| `SSL_ENABLED` | `false` | yes | Set to 'false' to disable SSL for local development | no |
+| `DISABLE_USER_AGENT` | `false` | yes | | |
+| `OSDU_VERSION` | `0.0.0` | yes | | |
+| `ENTITLEMENTS_BASE_URL` | `http://localhost:8081` or `https://your.osdu.instance.cluster.com` | yes | Specify the base url for an entitlements service instance. Can be run locally or remote. Don't include the API path, only the domain. | no |
+| `PARTITION_BASE_URL` | `http://localhost:8082` or `https://your.osdu.instance.cluster.com` | yes | Specify the base url for a partition service instance. Can be run locally or remote. Don't include the API path, only the domain. | no |
+| `STORAGE_BASE_URL` | `http://localhost:8082` or `https://your.osdu.instance.cluster.com` | yes | Specify the base url for a partition service instance. Can be run locally or remote. Don't include the API path, only the domain. | no |
+| `SCHEMA_BASE_URL` | `http://localhost:8082` or `https://your.osdu.instance.cluster.com` | yes | Specify the base url for a partition service instance. Can be run locally or remote. Don't include the API path, only the domain. | no |
+| `CLIENT_CREDENTIALS_ID` | `<CLIENT_ID>` | yes | Usually found inside SSM under `client-credentials/id` suffix. | |
+| `CLIENT_CREDENTIALS_SECRET` | `'{"client_credentials_client_secret": "<SECRET>"}'` | yes | Usually found inside Secret Manager under the `client-credentials-secret` suffix. Include the full JSON dict, not just the value | |
+| `OAUTH_TOKEN_URI` | `https://osdu-1234567890.auth.us-east-1.amazoncognito.com/oauth2/token` | yes | Usually found inside SSM under `oauth/token-uri` suffix | |
+| `OAUTH_CUSTOM_SCOPE` | `osduOnAws/osduOnAWSService` | yes | Usually found inside SSM under `oauth/custom-scope` suffix | |
+| `STORAGE_SQS_URL` | `https://sqs.us-east-1.amazonaws.com/1234567890/main-storage-queue` | yes | Can be found inside SSM, under `sqs/storage-queue/url` suffix | yes |
+| `INDEXER_DEADLETTER_QUEUE_SQS_URL` | `https://sqs.us-east-1.amazonaws.com/1234567890/main-indexer-deadletter-queue` | yes | Can be found inside SSM, under `indexer-queue/indexer-deadletter-queue/url` suffix | yes |
+| `INDEXER_SNS_TOPIC_ARN` | `arn:aws:sns:us-east-1:1234567890:osdu-tenant-group-indexer-messages` | yes | Can be found in SSM under `core/indexer/sns/arn` suffix | yes |
+| `ELASTICSEARCH_HOST` | `localhost` | yes | See note below this table. | |
+| `ELASTICSEARCH_PORT` | `9200` | yes | See note below this table. | |
+| `ELASTICSEARCH_CREDENTIALS` | `{"username":"<USERNAME>", "password": "<PASSWORD>"}` | yes | If using ES instance deployed in cluster, can be usually found in Secret Manager under `elasticsearch/credentials` suffix. | |
+| `STORAGE_RECORDS_BATCH_SIZE` | 20 | no | Batch size for storage API `POST {endpoint}/query/records:batch` | no |
+| `STORAGE_RECORDS_BY_KIND_BATCH_SIZE` | - | no | Batch size for storage API `GET {endpoint}/query/records`. If this is not present, defaults to value of `STORAGE_RECORDS_BATCH_SIZE` | no |
+
+
+For ElasticSearch, if you already have an OSDU environment deployed, you can use your existing ES instance by using port forwarding:
+```bash
+kubectl port-forward -n osdu-tenant-TENANT_NAME-elasticsearch svc/elasticsearch-es-http 9200:9200
+```
+And then just use `localhost` and `9200` for host and port.
+If you want to run ES locally, there are explanations below on this Readme on how to install it.
### Run Locally
Check that maven is installed:
@@ -123,28 +139,27 @@ You should see in the logs that pop up what url and port it runs on. By default
export ELASTIC_PASSWORD=$ELASTIC_PASSWORD
export ELASTIC_USER_NAME=$ELASTIC_USERNAME
- | name | example value | description | sensitive?
- | --- |----------------------------------------------------------------------------------------| --- | --- |
- | `AWS_ACCESS_KEY_ID` | `ASIAXXXXXXXXXXXXXX` | The AWS Access Key for a user with access to Backend Resources required by the service | yes |
- | `AWS_SECRET_ACCESS_KEY` | `super-secret-key==` | The AWS Secret Key for a user with access to Backend Resources required by the service | yes |
- | `AWS_SESSION_TOKEN` | `session-token-xxxxxxxxx` | AWS Session token needed if using an SSO user session to authenticate | yes |
- | `AWS_COGNITO_USER_POOL_ID` | `us-east-1_xxxxxxxx` | User Pool Id for the reference cognito | no |
- | `AWS_COGNITO_CLIENT_ID` | `xxxxxxxxxxxx` | Client ID for the Auth Flow integrated with the Cognito User Pool | no |
- | `AWS_COGNITO_AUTH_FLOW` | `USER_PASSWORD_AUTH` | Auth flow used by reference cognito deployment | no |
- | `DEFAULT_DATA_PARTITION_ID_TENANT1` | `opendes` | Partition used to create and index record | no |
- | `DEFAULT_DATA_PARTITION_ID_TENANT2` | `common` | Another needed partition | no |
- | `AWS_COGNITO_AUTH_PARAMS_USER` | `int-test-user@testing.com` | Int Test Username | no |
- | `AWS_COGNITO_AUTH_PARAMS_USER_NO_ACCESS` | `noaccess@testing.com` | No Access Username | no |
- | `AWS_COGNITO_AUTH_PARAMS_PASSWORD` | `some-secure-password` | Int Test User/NoAccessUser Password | yes |
- | `ENTITLEMENTS_DOMAIN` | `example.com` | Domain for user's groups | no |
- | `OTHER_RELEVANT_DATA_COUNTRIES` | `US` | Used to create demo legal tag | no |
- | `STORAGE_HOST` | `http://localhost:8080/api/storage/v2/` | The url where the storage API is hosted | no |
- | `HOST` | `http://localhost:8080` | Base url for deployment | no |
- | `ELASTIC_HOST` | `localhost` | Url for elasticsearch | no |
- | `ELASTIC_PORT` | `9300` | Port for elasticsearch | no |
- | `ELASTIC_PASSWORD` | `xxxxxxxxxxxxxxx` | Password for user to access elasticsearch | yes |
- | `ELASTIC_USER_NAME` | `xxxxxxxxxxxxxxxx` | Username for user to access elasticsearch | yes |
- | `CUCUMBER_OPTIONS` | `--tags '~@indexer-extended'` OR `--tags '~@* and @indexer-extended'` | By default `--tags '~@* and @indexer-extended'` to enable experimental feature testing | no |
+ | name | example value | description | sensitive?|
+ |------------------------------------------|----------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------| --- |
+ | `AWS_ACCESS_KEY_ID` | `ASIAXXXXXXXXXXXXXX` | The AWS Access Key for a user with access to Backend Resources required by the service | yes |
+ | `AWS_SECRET_ACCESS_KEY` | `super-secret-key==` | The AWS Secret Key for a user with access to Backend Resources required by the service | yes |
+ | `AWS_SESSION_TOKEN` | `session-token-xxxxxxxxx` | AWS Session token needed if using an SSO user session to authenticate | yes |
+ | `AWS_COGNITO_USER_POOL_ID` | `us-east-1_xxxxxxxx` | User Pool Id for the reference cognito | no |
+ | `AWS_COGNITO_CLIENT_ID` | `xxxxxxxxxxxx` | Client ID for the Auth Flow integrated with the Cognito User Pool | no |
+ | `AWS_COGNITO_AUTH_FLOW` | `USER_PASSWORD_AUTH` | Auth flow used by reference cognito deployment | no |
+ | `DEFAULT_DATA_PARTITION_ID_TENANT1` | `opendes` | Partition used to create and index record | no |
+ | `DEFAULT_DATA_PARTITION_ID_TENANT2` | `common` | Another needed partition | no |
+ | `AWS_COGNITO_AUTH_PARAMS_USER` | `int-test-user@testing.com` | Int Test Username | no |
+ | `AWS_COGNITO_AUTH_PARAMS_USER_NO_ACCESS` | `noaccess@testing.com` | No Access Username | no |
+ | `AWS_COGNITO_AUTH_PARAMS_PASSWORD` | `some-secure-password` | Int Test User/NoAccessUser Password | yes |
+ | `ENTITLEMENTS_DOMAIN` | `example.com` | Domain for user's groups | no |
+ | `OTHER_RELEVANT_DATA_COUNTRIES` | `US` | Used to create demo legal tag | no |
+ | `STORAGE_HOST` | `http://localhost:8080/api/storage/v2/` | The url where the storage API is hosted | no |
+ | `HOST` | `http://localhost:8080` | Base url for deployment | no |
+ | `ELASTIC_HOST` | `localhost` | Url for elasticsearch | no |
+ | `ELASTIC_PORT` | `9300` | Port for elasticsearch | no |
+ | `ELASTICSEARCH_CREDENTIALS` | `{"username":"<USERNAME>", "password": "<PASSWORD>"}` | Login/password for user to access elasticsearch | yes |
+ | `CUCUMBER_OPTIONS` | `--tags '~@indexer-extended'` OR `--tags '~@* and @indexer-extended'` | By default `--tags '~@* and @indexer-extended'` to enable experimental feature testing | no |
**Creating a new user to use for integration tests**
diff --git a/provider/indexer-aws/build-aws/buildspec.yaml b/provider/indexer-aws/build-aws/buildspec.yaml
index 966c6201a9a50caef7d817c725536fbac697728e..6d723f8049cf3b035bf0c8372f78f57d415ea89b 100644
--- a/provider/indexer-aws/build-aws/buildspec.yaml
+++ b/provider/indexer-aws/build-aws/buildspec.yaml
@@ -73,7 +73,8 @@ phases:
- if [ "$GIT_SECRETS_SCAN_RESULT" = "FAILED" ]; then echo "Secrets detected!" && exit 1; fi
- echo "Building primary service assemblies..."
- - mvn -ntp -B test install sonar:sonar -pl .,indexer-core,provider/indexer-aws -Ddeployment.environment=prod -Dsonar.login=${SONAR_USERNAME} -Dsonar.password=${SONAR_PASSWORD} -Dsonar.branch.name=${BRANCH_NAME}
+ - mvn -ntp -B test install -pl .,indexer-core,provider/indexer-aws -Ddeployment.environment=prod
+ - mvn sonar:sonar -pl .,provider/indexer-aws -Dsonar.login=${SONAR_USERNAME} -Dsonar.password=${SONAR_PASSWORD} -Dsonar.branch.name=${BRANCH_NAME}
- echo "Building integration testing assemblies and gathering artifacts..."
- ./testing/indexer-test-aws/build-aws/prepare-dist.sh
diff --git a/provider/indexer-aws/pom.xml b/provider/indexer-aws/pom.xml
index 99e904cb79cf210d1ef53c6552101804d142e109..6bff842ca7b02ff0592a44e088d74c56df556b22 100644
--- a/provider/indexer-aws/pom.xml
+++ b/provider/indexer-aws/pom.xml
@@ -230,6 +230,25 @@
</execution>
</executions>
</plugin>
+ <plugin>
+ <groupId>org.jacoco</groupId>
+ <artifactId>jacoco-maven-plugin</artifactId>
+ <version>0.8.10</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>prepare-agent</goal>
+ </goals>
+ </execution>
+ <execution>
+ <id>report</id>
+ <phase>test</phase>
+ <goals>
+ <goal>report</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
</project>
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/persistence/ElasticRepositoryImpl.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/persistence/ElasticRepositoryImpl.java
index 1529b5ee6b0bfd0aca5269e71dfc218d92bbe8c0..f84def3ff37f50683178097c469f48dd69459339 100644
--- a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/persistence/ElasticRepositoryImpl.java
+++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/persistence/ElasticRepositoryImpl.java
@@ -53,9 +53,9 @@ public class ElasticRepositoryImpl implements IElasticRepository {
@PostConstruct
private void postConstruct() throws K8sParameterNotFoundException, JsonProcessingException {
K8sLocalParameterProvider provider = new K8sLocalParameterProvider();
- host = provider.getParameterAsStringOrDefault("elasticsearch_host", host);
- port = Integer.parseInt(provider.getParameterAsStringOrDefault("elasticsearch_port", String.valueOf(port)));
- Map<String, String> val = provider.getCredentialsAsMap("elasticsearch_credentials");
+ host = provider.getParameterAsStringOrDefault("ELASTICSEARCH_HOST", host);
+ port = Integer.parseInt(provider.getParameterAsStringOrDefault("ELASTICSEARCH_PORT", String.valueOf(port)));
+ Map<String, String> val = provider.getCredentialsAsMap("ELASTICSEARCH_CREDENTIALS");
if (val != null){
username = val.get("username");
password = val.get("password");
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/publish/PublisherImpl.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/publish/PublisherImpl.java
index ecb999bd29f5725ca041ec664830dbf3f2bedb81..fb81abbbbb8100094869e1d442b91d3bb6533ae5 100644
--- a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/publish/PublisherImpl.java
+++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/publish/PublisherImpl.java
@@ -45,7 +45,7 @@ public class PublisherImpl implements IPublisher {
AmazonSNSConfig snsConfig = new AmazonSNSConfig(amazonSNSRegion);
snsClient = snsConfig.AmazonSNS();
K8sLocalParameterProvider provider = new K8sLocalParameterProvider();
- amazonSNSTopic = provider.getParameterAsString("indexer-sns-topic-arn");
+ amazonSNSTopic = provider.getParameterAsString("INDEXER_SNS_TOPIC_ARN");
}
public void publishStatusChangedTagsToTopic(DpsHeaders headers, JobStatus indexerBatchStatus) throws Exception
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java
index dbc4e36fb84a4f8b2d4a2a449e61174610f94aea..3f96435d7bcff6ad54ecdd27d0eb11c826b080cb 100644
--- a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java
+++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java
@@ -57,8 +57,8 @@ public class IndexerQueueTaskBuilderAws extends IndexerQueueTaskBuilder {
sqsClient = config.AmazonSQS();
gson =new Gson();
K8sLocalParameterProvider provider = new K8sLocalParameterProvider();
- storageQueue = provider.getParameterAsString("storage-sqs-url");
- dlq = provider.getParameterAsString("indexer-deadletter-queue-sqs-url");
+ storageQueue = provider.getParameterAsString("STORAGE_SQS_URL");
+ dlq = provider.getParameterAsString("INDEXER_DEADLETTER_QUEUE_SQS_URL");
}
@Override
diff --git a/testing/indexer-test-aws/build-aws/run-tests.sh b/testing/indexer-test-aws/build-aws/run-tests.sh
index fcedeab643617d82533eec78fc0ce55702c206e9..9535a28b49bcfaabbd90515d59b1f1f7204d8c19 100755
--- a/testing/indexer-test-aws/build-aws/run-tests.sh
+++ b/testing/indexer-test-aws/build-aws/run-tests.sh
@@ -56,9 +56,19 @@ trap '{
#### RUN INTEGRATION TEST #########################################################################
+CUCUMBER_PROPERTY="$SCRIPT_SOURCE_DIR/../src/test/resources/cucumber.properties"
+
+while IFS='=' read -r key value
+do
+ key=$(echo $key | tr '.' '_')
+ eval ${key}=\${value}
+done < "$CUCUMBER_PROPERTY"
+
+echo "Cucumber option cucumber.options = " ${cucumber_options}
+
+mvn -ntp test -f "$SCRIPT_SOURCE_DIR"/../pom.xml -Dcucumber.options="--plugin junit:target/junit-report.xml $cucumber_options"
JAVA_HOME=$JAVA17_HOME
-mvn -ntp test -f "$SCRIPT_SOURCE_DIR"/../pom.xml -Dcucumber.options="--plugin junit:target/junit-report.xml"
# mvn -Dmaven.surefire.debug test -f "$SCRIPT_SOURCE_DIR"/../pom.xml -Dcucumber.options="--plugin junit:target/junit-report.xml"
TEST_EXIT_CODE=$?