From 69e6732c1800ace362857a0fe29aaf33cd237d40 Mon Sep 17 00:00:00 2001
From: Guillaume Caillet <cailletg@amazon.com>
Date: Tue, 1 Aug 2023 10:38:03 -0400
Subject: [PATCH] Update readme and service parameters aliases to have
ENV-compatible names
---
provider/indexer-aws/README.md | 82 ++++++++++---------
.../persistence/ElasticRepositoryImpl.java | 6 +-
.../indexer/aws/publish/PublisherImpl.java | 2 +-
.../aws/util/IndexerQueueTaskBuilderAws.java | 4 +-
4 files changed, 48 insertions(+), 46 deletions(-)
diff --git a/provider/indexer-aws/README.md b/provider/indexer-aws/README.md
index 01b9d492c..78ea8fdb1 100644
--- a/provider/indexer-aws/README.md
+++ b/provider/indexer-aws/README.md
@@ -16,24 +16,27 @@ Pre-requisites
### Service Configuration
In order to run the service locally or remotely, you will need to have the following environment variables defined.
-| name | example value | required | description | sensitive? |
-| --- | --- | --- | --- | --- |
-| `LOCAL_MODE` | `true` | yes | Set to 'true' to use env vars in place of the k8s variable resolver | no |
-| `APPLICATION_PORT` | `8080` | yes | The port the service will be hosted on. | no |
-| `AWS_REGION` | `us-east-1` | yes | The region where resources needed by the service are deployed | no |
-| `AWS_ACCESS_KEY_ID` | `ASIAXXXXXXXXXXXXXX` | yes | The AWS Access Key for a user with access to Backend Resources required by the service | yes |
-| `AWS_SECRET_ACCESS_KEY` | `super-secret-key==` | yes | The AWS Secret Key for a user with access to Backend Resources required by the service | yes |
-| `AWS_SESSION_TOKEN` | `session-token-xxxxxxxxxx` | no | AWS Session token needed if using an SSO user session to authenticate | yes |
-| `ENVIRONMENT` | `osdu-prefix` | yes | The Resource Prefix defined during deployment | no |
-| `LOG_LEVEL` | `DEBUG` | yes | The Log Level severity to use (https://www.tutorialspoint.com/log4j/log4j_logging_levels.htm) | no |
-| `SSM_ENABLED` | `true` | yes | Set to 'true' to use SSM to resolve config properties, otherwise use env vars | no |
-| `SSL_ENABLED` | `false` | no | Set to 'false' to disable SSL for local development | no |
-| `ENTITLEMENTS_BASE_URL` | `http://localhost:8081` or `https://some-hosted-url` | yes | Specify the base url for an entitlements service instance. Can be run locally or remote | no |
-| `PARTITION_BASE_URL` | `http://localhost:8082` or `https://some-hosted-url` | yes | Specify the base url for a partition service instance. Can be run locally or remote | no |
-| `STORAGE_BASE_URL` | `http://localhost:8082` or `https://some-hosted-url` | yes | Specify the base url for a partition service instance. Can be run locally or remote | no |
-| `SCHEMA_BASE_URL` | `http://localhost:8082` or `https://some-hosted-url` | yes | Specify the base url for a partition service instance. Can be run locally or remote | no |
-| `STORAGE_RECORDS_BATCH_SIZE` | 20 | Batch size for storage API `POST {endpoint}/query/records:batch` | no | - |
-| `STORAGE_RECORDS_BY_KIND_BATCH_SIZE` | - | Batch size for storage API `GET {endpoint}/query/records`. If this is not present, defaults to value of `STORAGE_RECORDS_BATCH_SIZE` | no | - |
+| name | example value | required | description | sensitive? |
+|--------------------------------------|--------------------------------------------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------------------|------------|
+| `LOCAL_MODE` | `true` | yes | Set to 'true' to use env vars in place of the k8s variable resolver | no |
+| `APPLICATION_PORT` | `8080` | yes | The port the service will be hosted on. | no |
+| `AWS_REGION` | `us-east-1` | yes | The region where resources needed by the service are deployed | no |
+| `AWS_ACCESS_KEY_ID` | `ASIAXXXXXXXXXXXXXX` | yes | The AWS Access Key for a user with access to Backend Resources required by the service | yes |
+| `AWS_SECRET_ACCESS_KEY` | `super-secret-key==` | yes | The AWS Secret Key for a user with access to Backend Resources required by the service | yes |
+| `AWS_SESSION_TOKEN` | `session-token-xxxxxxxxxx` | no | AWS Session token needed if using an SSO user session to authenticate | yes |
+| `ENVIRONMENT` | `osdu-prefix` | yes | The Resource Prefix defined during deployment | no |
+| `LOG_LEVEL` | `DEBUG` | yes | The Log Level severity to use (https://www.tutorialspoint.com/log4j/log4j_logging_levels.htm) | no |
+| `SSM_ENABLED` | `true` | yes | Set to 'true' to use SSM to resolve config properties, otherwise use env vars | no |
+| `SSL_ENABLED` | `false` | no | Set to 'false' to disable SSL for local development | no |
+| `ENTITLEMENTS_BASE_URL` | `http://localhost:8081` or `https://some-hosted-url` | yes | Specify the base url for an entitlements service instance. Can be run locally or remote | no |
+| `PARTITION_BASE_URL` | `http://localhost:8082` or `https://some-hosted-url` | yes | Specify the base url for a partition service instance. Can be run locally or remote | no |
+| `STORAGE_BASE_URL` | `http://localhost:8082` or `https://some-hosted-url` | yes | Specify the base url for a partition service instance. Can be run locally or remote | no |
+| `SCHEMA_BASE_URL` | `http://localhost:8082` or `https://some-hosted-url` | yes | Specify the base url for a partition service instance. Can be run locally or remote | no |
+| `STORAGE_RECORDS_BATCH_SIZE` | 20 | no | Batch size for storage API `POST {endpoint}/query/records:batch` | no |
+| `STORAGE_RECORDS_BY_KIND_BATCH_SIZE` | - | no | Batch size for storage API `GET {endpoint}/query/records`. If this is not present, defaults to value of `STORAGE_RECORDS_BATCH_SIZE` | no |
+| `STORAGE_SQS_URL` | `https://sqs.us-east-1.amazonaws.com/1234567890/main-storage-queue` | yes | Can be found inside SSM, under `sqs/storage-queue/url` suffix | yes |
+| `INDEXER_DEADLETTER_QUEUE_SQS_URL` | `https://sqs.us-east-1.amazonaws.com/1234567890/main-indexer-deadletter-queue` | yes | Can be found inside SSM, under `indexer-queue/indexer-deadletter-queue/url` suffix | yes |
+| `INDEXER_SNS_TOPIC_ARN` | `arn:aws:sns:us-east-1:1234567890:osdu-tenant-group-indexer-messages` | yes | Can be found in SSM under `core/indexer/sns/arn` suffix | yes |
### Run Locally
@@ -123,28 +126,27 @@ You should see in the logs that pop up what url and port it runs on. By default
export ELASTIC_PASSWORD=$ELASTIC_PASSWORD
export ELASTIC_USER_NAME=$ELASTIC_USERNAME
- | name | example value | description | sensitive?
- | --- |----------------------------------------------------------------------------------------| --- | --- |
- | `AWS_ACCESS_KEY_ID` | `ASIAXXXXXXXXXXXXXX` | The AWS Access Key for a user with access to Backend Resources required by the service | yes |
- | `AWS_SECRET_ACCESS_KEY` | `super-secret-key==` | The AWS Secret Key for a user with access to Backend Resources required by the service | yes |
- | `AWS_SESSION_TOKEN` | `session-token-xxxxxxxxx` | AWS Session token needed if using an SSO user session to authenticate | yes |
- | `AWS_COGNITO_USER_POOL_ID` | `us-east-1_xxxxxxxx` | User Pool Id for the reference cognito | no |
- | `AWS_COGNITO_CLIENT_ID` | `xxxxxxxxxxxx` | Client ID for the Auth Flow integrated with the Cognito User Pool | no |
- | `AWS_COGNITO_AUTH_FLOW` | `USER_PASSWORD_AUTH` | Auth flow used by reference cognito deployment | no |
- | `DEFAULT_DATA_PARTITION_ID_TENANT1` | `opendes` | Partition used to create and index record | no |
- | `DEFAULT_DATA_PARTITION_ID_TENANT2` | `common` | Another needed partition | no |
- | `AWS_COGNITO_AUTH_PARAMS_USER` | `int-test-user@testing.com` | Int Test Username | no |
- | `AWS_COGNITO_AUTH_PARAMS_USER_NO_ACCESS` | `noaccess@testing.com` | No Access Username | no |
- | `AWS_COGNITO_AUTH_PARAMS_PASSWORD` | `some-secure-password` | Int Test User/NoAccessUser Password | yes |
- | `ENTITLEMENTS_DOMAIN` | `example.com` | Domain for user's groups | no |
- | `OTHER_RELEVANT_DATA_COUNTRIES` | `US` | Used to create demo legal tag | no |
- | `STORAGE_HOST` | `http://localhost:8080/api/storage/v2/` | The url where the storage API is hosted | no |
- | `HOST` | `http://localhost:8080` | Base url for deployment | no |
- | `ELASTIC_HOST` | `localhost` | Url for elasticsearch | no |
- | `ELASTIC_PORT` | `9300` | Port for elasticsearch | no |
- | `ELASTIC_PASSWORD` | `xxxxxxxxxxxxxxx` | Password for user to access elasticsearch | yes |
- | `ELASTIC_USER_NAME` | `xxxxxxxxxxxxxxxx` | Username for user to access elasticsearch | yes |
- | `CUCUMBER_OPTIONS` | `--tags '~@indexer-extended'` OR `--tags '~@* and @indexer-extended'` | By default `--tags '~@* and @indexer-extended'` to enable experimental feature testing | no |
+ | name | example value | description | sensitive?|
+ |------------------------------------------|-----------------------------------------------------------------------|----------------------------------------------------------------------------------------| --- |
+ | `AWS_ACCESS_KEY_ID` | `ASIAXXXXXXXXXXXXXX` | The AWS Access Key for a user with access to Backend Resources required by the service | yes |
+ | `AWS_SECRET_ACCESS_KEY` | `super-secret-key==` | The AWS Secret Key for a user with access to Backend Resources required by the service | yes |
+ | `AWS_SESSION_TOKEN` | `session-token-xxxxxxxxx` | AWS Session token needed if using an SSO user session to authenticate | yes |
+ | `AWS_COGNITO_USER_POOL_ID` | `us-east-1_xxxxxxxx` | User Pool Id for the reference cognito | no |
+ | `AWS_COGNITO_CLIENT_ID` | `xxxxxxxxxxxx` | Client ID for the Auth Flow integrated with the Cognito User Pool | no |
+ | `AWS_COGNITO_AUTH_FLOW` | `USER_PASSWORD_AUTH` | Auth flow used by reference cognito deployment | no |
+ | `DEFAULT_DATA_PARTITION_ID_TENANT1` | `opendes` | Partition used to create and index record | no |
+ | `DEFAULT_DATA_PARTITION_ID_TENANT2` | `common` | Another needed partition | no |
+ | `AWS_COGNITO_AUTH_PARAMS_USER` | `int-test-user@testing.com` | Int Test Username | no |
+ | `AWS_COGNITO_AUTH_PARAMS_USER_NO_ACCESS` | `noaccess@testing.com` | No Access Username | no |
+ | `AWS_COGNITO_AUTH_PARAMS_PASSWORD` | `some-secure-password` | Int Test User/NoAccessUser Password | yes |
+ | `ENTITLEMENTS_DOMAIN` | `example.com` | Domain for user's groups | no |
+ | `OTHER_RELEVANT_DATA_COUNTRIES` | `US` | Used to create demo legal tag | no |
+ | `STORAGE_HOST` | `http://localhost:8080/api/storage/v2/` | The url where the storage API is hosted | no |
+ | `HOST` | `http://localhost:8080` | Base url for deployment | no |
+ | `ELASTIC_HOST` | `localhost` | Url for elasticsearch | no |
+ | `ELASTIC_PORT` | `9300` | Port for elasticsearch | no |
+ | `ELASTICSEARCH_CREDENTIALS` | `{"username":"<USERNAME>", "password": "<PASSWORD>"}` | Login/passzord for user to access elasticsearch | yes |
+ | `CUCUMBER_OPTIONS` | `--tags '~@indexer-extended'` OR `--tags '~@* and @indexer-extended'` | By default `--tags '~@* and @indexer-extended'` to enable experimental feature testing | no |
**Creating a new user to use for integration tests**
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/persistence/ElasticRepositoryImpl.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/persistence/ElasticRepositoryImpl.java
index 1529b5ee6..f84def3ff 100644
--- a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/persistence/ElasticRepositoryImpl.java
+++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/persistence/ElasticRepositoryImpl.java
@@ -53,9 +53,9 @@ public class ElasticRepositoryImpl implements IElasticRepository {
@PostConstruct
private void postConstruct() throws K8sParameterNotFoundException, JsonProcessingException {
K8sLocalParameterProvider provider = new K8sLocalParameterProvider();
- host = provider.getParameterAsStringOrDefault("elasticsearch_host", host);
- port = Integer.parseInt(provider.getParameterAsStringOrDefault("elasticsearch_port", String.valueOf(port)));
- Map<String, String> val = provider.getCredentialsAsMap("elasticsearch_credentials");
+ host = provider.getParameterAsStringOrDefault("ELASTICSEARCH_HOST", host);
+ port = Integer.parseInt(provider.getParameterAsStringOrDefault("ELASTICSEARCH_PORT", String.valueOf(port)));
+ Map<String, String> val = provider.getCredentialsAsMap("ELASTICSEARCH_CREDENTIALS");
if (val != null){
username = val.get("username");
password = val.get("password");
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/publish/PublisherImpl.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/publish/PublisherImpl.java
index ecb999bd2..fb81abbbb 100644
--- a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/publish/PublisherImpl.java
+++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/publish/PublisherImpl.java
@@ -45,7 +45,7 @@ public class PublisherImpl implements IPublisher {
AmazonSNSConfig snsConfig = new AmazonSNSConfig(amazonSNSRegion);
snsClient = snsConfig.AmazonSNS();
K8sLocalParameterProvider provider = new K8sLocalParameterProvider();
- amazonSNSTopic = provider.getParameterAsString("indexer-sns-topic-arn");
+ amazonSNSTopic = provider.getParameterAsString("INDEXER_SNS_TOPIC_ARN");
}
public void publishStatusChangedTagsToTopic(DpsHeaders headers, JobStatus indexerBatchStatus) throws Exception
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java
index dbc4e36fb..3f96435d7 100644
--- a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java
+++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java
@@ -57,8 +57,8 @@ public class IndexerQueueTaskBuilderAws extends IndexerQueueTaskBuilder {
sqsClient = config.AmazonSQS();
gson =new Gson();
K8sLocalParameterProvider provider = new K8sLocalParameterProvider();
- storageQueue = provider.getParameterAsString("storage-sqs-url");
- dlq = provider.getParameterAsString("indexer-deadletter-queue-sqs-url");
+ storageQueue = provider.getParameterAsString("STORAGE_SQS_URL");
+ dlq = provider.getParameterAsString("INDEXER_DEADLETTER_QUEUE_SQS_URL");
}
@Override
--
GitLab