diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 26563d73502d75f11a6223def0f8946136800dfa..65239865d0e935ceb993f742f82136bf75a976d9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,6 +3,8 @@ variables: AWS_TEST_SUBDIR: testing/indexer-test-aws AWS_SERVICE: indexer AWS_ENVIRONMENT: dev + AWS_DEPLOY_TARGET: EKS + AWS_EKS_DEPLOYMENT_NAME: os-indexer GCP_BUILD_SUBDIR: provider/indexer-gcp GCP_INT_TEST_SUBDIR: testing/indexer-test-gcp diff --git a/provider/indexer-aws/build-aws/buildspec.yaml b/provider/indexer-aws/build-aws/buildspec.yaml index 71e7e2c50d1f2ad6960513070a2200eb36d9105a..c3a35e1aae79d5a919762f110393c78bf91a76a0 100644 --- a/provider/indexer-aws/build-aws/buildspec.yaml +++ b/provider/indexer-aws/build-aws/buildspec.yaml @@ -21,6 +21,11 @@ env: secrets-manager: DOCKER_USERNAME: /osdu/devops/docker_credentials:username DOCKER_PASSWORD: /osdu/devops/docker_credentials:password + SONAR_USERNAME: /osdu/devops/sonar_credentials:username + SONAR_PASSWORD: /osdu/devops/sonar_credentials:password + + parameter-store: + SONAR_URL: /osdu/devops/sonar_url phases: install: @@ -59,8 +64,7 @@ phases: - printenv - echo "Building primary service assemblies..." - - mvn -ntp -B test install -pl indexer-core,provider/indexer-aws -Ddeployment.environment=prod - + - mvn -ntp -B test install sonar:sonar -pl .,indexer-core,provider/indexer-aws -Ddeployment.environment=prod -Dsonar.login=${SONAR_USERNAME} -Dsonar.password=${SONAR_PASSWORD} -Dsonar.branch.name=${BRANCH_NAME} # - echo "Copying assemblies to dist..." # - cp ./provider/indexer-aws/target/*spring-boot.jar ${OUTPUT_DIR}/bin # copy aws jars @@ -84,6 +88,10 @@ phases: python provider/indexer-aws/build-aws/build-info.py --branch ${CODEBUILD_SOURCE_VERSION} --commit ${CODEBUILD_RESOLVED_SOURCE_VERSION} \ --buildid ${CODEBUILD_BUILD_ID} --buildnumber ${CODEBUILD_BUILD_NUMBER} --reponame ${REPO_NAME} --outdir ${OUTPUT_DIR} \ --artifact ${ECR_IMAGE} + + post_build: + commands: + - cp provider/indexer-aws/target/dependency-check-report.html ${OUTPUT_DIR} reports: SurefireReports: # CodeBuild will create a report group called "SurefireReports". files: #Store all of the files diff --git a/provider/indexer-aws/maven/settings.xml b/provider/indexer-aws/maven/settings.xml index 1bfc43ac96aafb6790498c6f0acef3ca0818a98f..3bd1cd4aa7ed183241e62e75d2b7f1a3f5ebc7aa 100644 --- a/provider/indexer-aws/maven/settings.xml +++ b/provider/indexer-aws/maven/settings.xml @@ -54,6 +54,17 @@ limitations under the License. no-default + + sonar + + true + + + + ${env.SONAR_URL} + + + diff --git a/provider/indexer-aws/pom.xml b/provider/indexer-aws/pom.xml index b1345abf80a71e29a8b001d260e1db7135d275cc..11edf9f28af5494bd373437c6b823455ac722ddc 100644 --- a/provider/indexer-aws/pom.xml +++ b/provider/indexer-aws/pom.xml @@ -24,12 +24,12 @@ 4.0.0 indexer-aws - Storage service on AWS + Indexer service on AWS jar 0.11.0-SNAPSHOT - 1.11.637 + 1.11.1018 dev @@ -47,19 +47,19 @@ org.opengroup.osdu.core.aws os-core-lib-aws - 0.3.16 + 0.11.0-SNAPSHOT com.amazonaws aws-java-sdk-core - 1.11.651 + ${aws.version} com.amazonaws aws-java-sdk - 1.11.327 + ${aws.version} @@ -161,6 +161,18 @@ + + org.owasp + dependency-check-maven + 6.2.2 + + + + check + + + + diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/IndexCacheImpl.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/IndexCacheImpl.java index 2c08033ebafb9a2b4e5f2224da2a59153bd49d8c..fc33e78a6275a8a4ef707b586021e8be9b86a40e 100644 --- a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/IndexCacheImpl.java +++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/IndexCacheImpl.java @@ -14,27 +14,64 @@ package org.opengroup.osdu.indexer.aws.cache; +import com.fasterxml.jackson.core.JsonProcessingException; +import org.opengroup.osdu.core.common.cache.ICache; import org.opengroup.osdu.core.common.cache.RedisCache; +import org.opengroup.osdu.core.common.cache.VmCache; +import org.opengroup.osdu.core.aws.cache.DummyCache; +import org.opengroup.osdu.core.aws.ssm.K8sLocalParameterProvider; +import org.opengroup.osdu.core.aws.ssm.K8sParameterNotFoundException; import org.opengroup.osdu.core.common.provider.interfaces.IIndexCache; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; +import java.util.Map; + @Component public class IndexCacheImpl implements IIndexCache, AutoCloseable { - private RedisCache cache; - - public IndexCacheImpl(@Value("${aws.elasticache.cluster.endpoint}") final String REDIS_SEARCH_HOST, - @Value("${aws.elasticache.cluster.port}") final String REDIS_SEARCH_PORT, - @Value("${aws.elasticache.cluster.key}") final String REDIS_SEARCH_KEY, - @Value("${aws.elasticache.cluster.index.expiration}") final String INDEX_CACHE_EXPIRATION) { - cache = new RedisCache<>(REDIS_SEARCH_HOST, Integer.parseInt(REDIS_SEARCH_PORT), REDIS_SEARCH_KEY, - Integer.parseInt(INDEX_CACHE_EXPIRATION) * 60, String.class, Boolean.class); + private ICache cache; + private Boolean local; + @Value("${aws.elasticache.cluster.endpoint}") + String REDIS_SEARCH_HOST; + @Value("${aws.elasticache.cluster.port}") + String REDIS_SEARCH_PORT; + @Value("${aws.elasticache.cluster.key}") + String REDIS_SEARCH_KEY; + @Value("${aws.elasticache.cluster.index.expiration}") + String INDEX_CACHE_EXPIRATION; + public IndexCacheImpl() throws K8sParameterNotFoundException, JsonProcessingException { + int expTimeSeconds = 60 * 60; + K8sLocalParameterProvider provider = new K8sLocalParameterProvider(); + if (provider.getLocalMode()){ + if (Boolean.parseBoolean(System.getenv("DISABLE_CACHE"))){ + cache = new DummyCache<>(); + }else{ + cache = new VmCache(expTimeSeconds, 10); + } + }else { + String host = provider.getParameterAsStringOrDefault("CACHE_CLUSTER_ENDPOINT", REDIS_SEARCH_HOST); + int port = Integer.parseInt(provider.getParameterAsStringOrDefault("CACHE_CLUSTER_PORT", REDIS_SEARCH_PORT)); + Map credential =provider.getCredentialsAsMap("CACHE_CLUSTER_KEY"); + String password; + if (credential !=null){ + password = credential.get("token"); + }else{ + password = REDIS_SEARCH_KEY; + } + cache = new RedisCache(host, port, password, expTimeSeconds, String.class,Boolean.class); + } + local = cache instanceof AutoCloseable; } @Override public void close() throws Exception { - this.cache.close(); + if (local){ + + }else{ + ((AutoCloseable)this.cache).close(); + } + } @Override diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/SchemaCacheImpl.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/SchemaCacheImpl.java index 5b8b8f02e12ed3d5f43170721d53a282d1283b94..27a1f3792a25a47a331117a85e52f55a62510efd 100644 --- a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/SchemaCacheImpl.java +++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/SchemaCacheImpl.java @@ -14,27 +14,63 @@ package org.opengroup.osdu.indexer.aws.cache; +import com.fasterxml.jackson.core.JsonProcessingException; +import org.opengroup.osdu.core.aws.cache.DummyCache; +import org.opengroup.osdu.core.aws.ssm.K8sLocalParameterProvider; +import org.opengroup.osdu.core.aws.ssm.K8sParameterNotFoundException; +import org.opengroup.osdu.core.common.cache.ICache; import org.opengroup.osdu.core.common.cache.RedisCache; +import org.opengroup.osdu.core.common.cache.VmCache; import org.opengroup.osdu.indexer.provider.interfaces.ISchemaCache; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; +import java.util.Map; + @Component public class SchemaCacheImpl implements ISchemaCache, AutoCloseable { - - private RedisCache cache; - - public SchemaCacheImpl(@Value("${aws.elasticache.cluster.endpoint}") final String REDIS_SEARCH_HOST, - @Value("${aws.elasticache.cluster.port}") final String REDIS_SEARCH_PORT, - @Value("${aws.elasticache.cluster.key}") final String REDIS_SEARCH_KEY, - @Value("${aws.elasticache.cluster.schema.expiration}") final String SCHEMA_CACHE_EXPIRATION) { - cache = new RedisCache<>(REDIS_SEARCH_HOST, Integer.parseInt(REDIS_SEARCH_PORT), REDIS_SEARCH_KEY, - Integer.parseInt(SCHEMA_CACHE_EXPIRATION) * 60, String.class, String.class); + @Value("${aws.elasticache.cluster.endpoint}") + String REDIS_SEARCH_HOST; + @Value("${aws.elasticache.cluster.port}") + String REDIS_SEARCH_PORT; + @Value("${aws.elasticache.cluster.key}") + String REDIS_SEARCH_KEY; + @Value("${aws.elasticache.cluster.schema.expiration}") + String SCHEMA_CACHE_EXPIRATION; + private ICache cache; + private Boolean local = false; + public SchemaCacheImpl() throws K8sParameterNotFoundException, JsonProcessingException { + int expTimeSeconds = 60 * 60; + K8sLocalParameterProvider provider = new K8sLocalParameterProvider(); + if (provider.getLocalMode()){ + if (Boolean.parseBoolean(System.getenv("DISABLE_CACHE"))){ + cache = new DummyCache<>(); + }else{ + cache = new VmCache(expTimeSeconds, 10); + } + }else { + String host = provider.getParameterAsStringOrDefault("CACHE_CLUSTER_ENDPOINT", REDIS_SEARCH_HOST); + int port = Integer.parseInt(provider.getParameterAsStringOrDefault("CACHE_CLUSTER_PORT", REDIS_SEARCH_PORT)); + Map credential =provider.getCredentialsAsMap("CACHE_CLUSTER_KEY"); + String password; + if (credential !=null){ + password = credential.get("token"); + }else{ + password = REDIS_SEARCH_KEY; + } + cache = new RedisCache<>(host, port, password, expTimeSeconds, String.class, String.class); + } + local = cache instanceof AutoCloseable; } @Override public void close() throws Exception { - this.cache.close(); + if (this.local){ + // do nothing, this is using local dummy cache + }else { + // cast to redis cache so it can be closed + ((AutoCloseable)this.cache).close(); + } } @Override diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/persistence/ElasticRepositoryImpl.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/persistence/ElasticRepositoryImpl.java index 1a66790b2d7edc612443404271d3f33b8a22374f..1529b5ee6b0bfd0aca5269e71dfc218d92bbe8c0 100644 --- a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/persistence/ElasticRepositoryImpl.java +++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/persistence/ElasticRepositoryImpl.java @@ -14,16 +14,18 @@ package org.opengroup.osdu.indexer.aws.persistence; -import org.opengroup.osdu.core.aws.secrets.SecretsManager; -import org.opengroup.osdu.core.aws.ssm.ParameterStorePropertySource; -import org.opengroup.osdu.core.aws.ssm.SSMConfig; + +import com.fasterxml.jackson.core.JsonProcessingException; +import org.opengroup.osdu.core.aws.ssm.K8sParameterNotFoundException; import org.opengroup.osdu.core.common.model.search.ClusterSettings; import org.opengroup.osdu.core.common.model.tenant.TenantInfo; import org.opengroup.osdu.core.common.provider.interfaces.IElasticRepository; +import org.opengroup.osdu.core.aws.ssm.K8sLocalParameterProvider; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; +import java.util.Map; @Component public class ElasticRepositoryImpl implements IElasticRepository { @@ -45,36 +47,20 @@ public class ElasticRepositoryImpl implements IElasticRepository { String usernameAndPassword; - - @Value("${aws.elasticsearch.port}") - String portParameter; - - @Value("${aws.elasticsearch.host}") - String hostParameter; - - @Value("${aws.elasticsearch.credentials.secret}") - String elasticCredentialsSecret; - @Value("${aws.region}") private String amazonRegion; - @Value("${aws.ssm}") - String ssmEnabledString; - - private ParameterStorePropertySource ssm; - @PostConstruct - private void postConstruct() { - if( Boolean.parseBoolean(ssmEnabledString)) { - SSMConfig ssmConfig = new SSMConfig(); - ssm = ssmConfig.amazonSSM(); - host = ssm.getProperty(hostParameter).toString(); - port = Integer.parseInt(ssm.getProperty(portParameter).toString()); - + private void postConstruct() throws K8sParameterNotFoundException, JsonProcessingException { + K8sLocalParameterProvider provider = new K8sLocalParameterProvider(); + host = provider.getParameterAsStringOrDefault("elasticsearch_host", host); + port = Integer.parseInt(provider.getParameterAsStringOrDefault("elasticsearch_port", String.valueOf(port))); + Map val = provider.getCredentialsAsMap("elasticsearch_credentials"); + if (val != null){ + username = val.get("username"); + password = val.get("password"); } - SecretsManager sm = new SecretsManager(); - username = sm.getSecret(elasticCredentialsSecret,amazonRegion,"username"); - password = sm.getSecret(elasticCredentialsSecret,amazonRegion,"password"); + //elastic expects username:password format usernameAndPassword = String.format("%s:%s", username, password); } diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/publish/PublisherImpl.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/publish/PublisherImpl.java index d5a5ae5195eb483fa7bc11e7eb06839c293f1367..ecb999bd29f5725ca041ec664830dbf3f2bedb81 100644 --- a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/publish/PublisherImpl.java +++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/publish/PublisherImpl.java @@ -17,8 +17,8 @@ package org.opengroup.osdu.indexer.aws.publish; import com.amazonaws.services.sns.model.MessageAttributeValue; import com.amazonaws.services.sns.model.PublishRequest; import com.amazonaws.services.sns.AmazonSNS; -import org.opengroup.osdu.core.aws.ssm.ParameterStorePropertySource; -import org.opengroup.osdu.core.aws.ssm.SSMConfig; + +import org.opengroup.osdu.core.aws.ssm.K8sParameterNotFoundException; import org.opengroup.osdu.core.common.model.http.DpsHeaders; import org.opengroup.osdu.core.aws.sns.AmazonSNSConfig; import org.opengroup.osdu.core.aws.sns.PublishRequestBuilder; @@ -26,7 +26,7 @@ import org.opengroup.osdu.indexer.provider.interfaces.IPublisher; import org.opengroup.osdu.core.common.model.indexer.JobStatus; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; - +import org.opengroup.osdu.core.aws.ssm.K8sLocalParameterProvider; import javax.inject.Inject; import java.util.HashMap; import java.util.Map; @@ -35,25 +35,17 @@ import java.util.Map; public class PublisherImpl implements IPublisher { AmazonSNS snsClient; - - private ParameterStorePropertySource ssm; - private String amazonSNSTopic; @Value("${aws.region}") private String amazonSNSRegion; - @Value("${aws.indexer.sns.topic.arn}") - private String parameter; - - @Inject - public void init(){ + public void init() throws K8sParameterNotFoundException { AmazonSNSConfig snsConfig = new AmazonSNSConfig(amazonSNSRegion); snsClient = snsConfig.AmazonSNS(); - SSMConfig ssmConfig = new SSMConfig(); - ssm = ssmConfig.amazonSSM(); - amazonSNSTopic = ssm.getProperty(parameter).toString(); + K8sLocalParameterProvider provider = new K8sLocalParameterProvider(); + amazonSNSTopic = provider.getParameterAsString("indexer-sns-topic-arn"); } public void publishStatusChangedTagsToTopic(DpsHeaders headers, JobStatus indexerBatchStatus) throws Exception diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java index cb47d53cd17fce4056d18d82a4371f8b1fe2d204..0944a9091c2ac6533ac5b26c88d06fec12808c70 100644 --- a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java +++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java @@ -19,8 +19,8 @@ import org.opengroup.osdu.core.aws.sqs.AmazonSQSConfig; import com.amazonaws.services.sqs.model.MessageAttributeValue; import com.amazonaws.services.sqs.model.SendMessageRequest; import com.google.gson.Gson; -import org.opengroup.osdu.core.aws.ssm.ParameterStorePropertySource; -import org.opengroup.osdu.core.aws.ssm.SSMConfig; +import org.opengroup.osdu.core.aws.ssm.K8sLocalParameterProvider; +import org.opengroup.osdu.core.aws.ssm.K8sParameterNotFoundException; import org.opengroup.osdu.core.common.model.http.DpsHeaders; import org.opengroup.osdu.core.common.model.search.RecordChangedMessages; import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder; @@ -41,8 +41,6 @@ public class IndexerQueueTaskBuilderAws extends IndexerQueueTaskBuilder { private AmazonSQS sqsClient; - private ParameterStorePropertySource ssm; - private String storageQueue; private String dlq; private final String retryString = "retry"; @@ -52,20 +50,14 @@ public class IndexerQueueTaskBuilderAws extends IndexerQueueTaskBuilder { @Value("${aws.region}") private String region; - @Value("${aws.storage.sqs.queue.url}") - String sqsStorageQueueParameter; - @Value("${aws.indexer.sqs.dlq.url}") - String deadLetterQueueParameter; - @Inject - public void init() { + public void init() throws K8sParameterNotFoundException { AmazonSQSConfig config = new AmazonSQSConfig(region); sqsClient = config.AmazonSQS(); gson =new Gson(); - SSMConfig ssmConfig = new SSMConfig(); - ssm = ssmConfig.amazonSSM(); - storageQueue = ssm.getProperty(sqsStorageQueueParameter).toString(); - dlq = ssm.getProperty(deadLetterQueueParameter).toString(); + K8sLocalParameterProvider provider = new K8sLocalParameterProvider(); + storageQueue = provider.getParameterAsString("storage-sqs-url"); + dlq = provider.getParameterAsString("indexer-deadletter-queue-sqs-url"); } @Override diff --git a/provider/indexer-aws/src/main/resources/application.properties b/provider/indexer-aws/src/main/resources/application.properties index 35404a77fa73afa85035cc0bbe89b9b42fddbce0..213017d86608eba1c781ecae4c74857f3224f594 100644 --- a/provider/indexer-aws/src/main/resources/application.properties +++ b/provider/indexer-aws/src/main/resources/application.properties @@ -22,20 +22,19 @@ aws.es.certificate.disableTrust=${ELASTIC_DISABLE_CERTIFICATE_TRUST:false} GAE_SERVICE=indexer -#reusing STORAGE_HOST variable here as the base url to point to schema service -SCHEMA_HOST=${STORAGE_HOST}/api/schema-service/v1/schema +#reusing STORAGE_BASE_URL variable here as the base url to point to schema service +SCHEMA_HOST=${SCHEMA_BASE_URL}/api/schema-service/v1/schema -STORAGE_SCHEMA_HOST=${STORAGE_HOST}/api/storage/v2/schemas -STORAGE_QUERY_RECORD_HOST=${STORAGE_HOST}/api/storage/v2/query/records -STORAGE_QUERY_KINDS_HOST=${STORAGE_HOST}/api/storage/v2/query/kinds -STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=${STORAGE_HOST}/api/storage/v2/query/records:batch +STORAGE_SCHEMA_HOST=${STORAGE_BASE_URL}/api/storage/v2/schemas +STORAGE_QUERY_RECORD_HOST=${STORAGE_BASE_URL}/api/storage/v2/query/records +STORAGE_QUERY_KINDS_HOST=${STORAGE_BASE_URL}/api/storage/v2/query/kinds +STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=${STORAGE_BASE_URL}/api/storage/v2/query/records:batch STORAGE_RECORDS_BATCH_SIZE=20 INDEXER_QUEUE_HOST="" ## AWS ElastiCache configuration -aws.elasticache.cluster.endpoint=${CACHE_CLUSTER_ENDPOINT} -aws.elasticache.cluster.port=${CACHE_CLUSTER_PORT} -aws.elasticache.cluster.key=${CACHE_CLUSTER_KEY} - +aws.elasticache.cluster.endpoint=${CACHE_CLUSTER_ENDPOINT:null} +aws.elasticache.cluster.port=${CACHE_CLUSTER_PORT:null} +aws.elasticache.cluster.key=${CACHE_CLUSTER_KEY:null} ## Cache Settings aws.elasticache.cluster.index.expiration=60 aws.elasticache.cluster.schema.expiration=60 @@ -53,15 +52,7 @@ aws.dynamodb.endpoint=dynamodb.${AWS_REGION}.amazonaws.com aws.ssm=${SSM_ENABLED} aws.ssm.prefix=/osdu/${ENVIRONMENT} - -aws.elasticsearch.host=${aws.ssm.prefix}/elasticsearch/end-point -aws.elasticsearch.port=${aws.ssm.prefix}/elasticsearch/end-point-port -aws.elasticsearch.credentials.secret=${aws.ssm.prefix}/elasticsearch/credentials -aws.indexer.sns.topic.arn=${aws.ssm.prefix}/indexer/indexer-sns-topic-arn -aws.storage.sns.topic.arn=${aws.ssm.prefix}/storage/storage-sns-topic-arn - -aws.storage.sqs.queue.url=${aws.ssm.prefix}/storage/storage-sqs-url -aws.indexer.sqs.dlq.url=${aws.ssm.prefix}/indexer-queue/indexer-deadletter-queue-sqs-url +aws.parameter.prefix=/osdu/${ENVIRONMENT} server.ssl.enabled=${SSL_ENABLED:true} server.ssl.key-store-type=PKCS12