Merge branch 'aws-integration' into 'master'

Move to EKS

See merge request !84

.gitlab-ci.yml

@@ -3,6 +3,8 @@ variables:
   AWS_TEST_SUBDIR: testing/partition-test-aws
   AWS_SERVICE: partition
   AWS_ENVIRONMENT: dev
+  AWS_DEPLOY_TARGET: EKS
+  AWS_EKS_DEPLOYMENT_NAME: os-partition
 
   AZURE_SERVICE: partition
   AZURE_BUILD_SUBDIR: provider/partition-azure

provider/partition-aws/build-aws/buildspec.yaml

@@ -21,6 +21,11 @@ env:
   secrets-manager:
     DOCKER_USERNAME: /osdu/devops/docker_credentials:username
     DOCKER_PASSWORD: /osdu/devops/docker_credentials:password
+    SONAR_USERNAME: /osdu/devops/sonar_credentials:username
+    SONAR_PASSWORD: /osdu/devops/sonar_credentials:password
+  
+  parameter-store:
+    SONAR_URL: /osdu/devops/sonar_url
 
 phases:
   install:
@@ -58,7 +63,7 @@ phases:
       - printenv
 
       - echo "Building primary service assemblies..."
-      - mvn -B test install -pl partition-core,provider/partition-aws -Ddeployment.environment=prod
+      - mvn -B test install sonar:sonar -pl .,partition-core,provider/partition-aws -Ddeployment.environment=prod -Dsonar.login=${SONAR_USERNAME} -Dsonar.password=${SONAR_PASSWORD} -Dsonar.branch.name=${BRANCH_NAME} 
 
       - echo "Building integration testing assemblies and gathering artifacts..."
       - ./testing/partition-test-aws/build-aws/prepare-dist.sh
@@ -78,6 +83,9 @@ phases:
         python provider/partition-aws/build-aws/build-info.py --branch ${CODEBUILD_SOURCE_VERSION} --commit ${CODEBUILD_RESOLVED_SOURCE_VERSION} \
           --buildid ${CODEBUILD_BUILD_ID} --buildnumber ${CODEBUILD_BUILD_NUMBER} --reponame ${REPO_NAME} --outdir ${OUTPUT_DIR} \
           --artifact ${ECR_IMAGE}
+  post_build:
+    commands:
+      - cp provider/partition-aws/target/dependency-check-report.html ${OUTPUT_DIR}
 reports:
   SurefireReports: # CodeBuild will create a report group called "SurefireReports".
     files: #Store all of the files

provider/partition-aws/maven/settings.xml

@@ -54,6 +54,17 @@ limitations under the License.
                 <azure.devops.token>no-default</azure.devops.token>
             </properties>
         </profile>
+        <profile>
+            <id>sonar</id>
+            <activation>
+                <activeByDefault>true</activeByDefault>
+            </activation>
+            <properties>
+                <sonar.host.url>
+                    ${env.SONAR_URL}
+                </sonar.host.url>
+            </properties>
+        </profile>
     </profiles>
 
     <servers>

provider/partition-aws/pom.xml

@@ -57,7 +57,7 @@
         <dependency>
             <groupId>org.opengroup.osdu.core.aws</groupId>
             <artifactId>os-core-lib-aws</artifactId>
-            <version>0.10.0</version>
+            <version>0.11.0-SNAPSHOT</version>
         </dependency>
         <dependency>
             <groupId>org.opengroup.osdu</groupId>
@@ -180,6 +180,18 @@
                     </execution>
                 </executions>
             </plugin>
+            <plugin>
+                <groupId>org.owasp</groupId>
+                <artifactId>dependency-check-maven</artifactId>
+                <version>6.2.2</version>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>check</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
         </plugins>
     </build>
 

provider/partition-aws/src/main/java/org/opengroup/osdu/partition/provider/aws/cache/GroupCache.java

-// Copyright © 2020 Amazon Web Services
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -14,17 +14,50 @@
 
 package org.opengroup.osdu.partition.provider.aws.cache;
 
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import org.opengroup.osdu.core.aws.ssm.K8sLocalParameterProvider;
+import org.opengroup.osdu.core.aws.ssm.K8sParameterNotFoundException;
+import org.opengroup.osdu.core.common.cache.ICache;
+import org.opengroup.osdu.core.common.cache.RedisCache;
+import org.opengroup.osdu.core.common.cache.VmCache;
 import org.opengroup.osdu.core.common.model.http.DpsHeaders;
 import org.opengroup.osdu.core.common.model.entitlements.Groups;
-import org.opengroup.osdu.core.common.cache.RedisCache;
 import org.opengroup.osdu.core.common.util.Crc32c;
+import org.opengroup.osdu.core.aws.cache.DummyCache;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 
+import java.util.Map;
+
+
 @Component
-public class GroupCache extends RedisCache<String, Groups> {
-    public GroupCache(@Value("${aws.elasticache.cluster.endpoint}") final String REDIS_GROUP_HOST, @Value("${aws.elasticache.cluster.port}") final String REDIS_GROUP_PORT, @Value("${aws.elasticache.cluster.key}") final String REDIS_GROUP_KEY) {
-        super(REDIS_GROUP_HOST, Integer.parseInt(REDIS_GROUP_PORT), REDIS_GROUP_KEY, 30, String.class, Groups.class);
+public class GroupCache {
+    @Value("${aws.elasticache.cluster.endpoint}")
+    String REDIS_SEARCH_HOST;
+    @Value("${aws.elasticache.cluster.port}")
+    String REDIS_SEARCH_PORT;
+    @Value("${aws.elasticache.cluster.key}")
+    String REDIS_SEARCH_KEY;
+    public ICache<String, Groups> GetGroupCache() throws K8sParameterNotFoundException, JsonProcessingException {
+        K8sLocalParameterProvider provider = new K8sLocalParameterProvider();
+        if (provider.getLocalMode()){
+            if (Boolean.parseBoolean(System.getenv("DISABLE_CACHE"))){
+                return new DummyCache();
+            }
+            return new VmCache<>(60, 10);
+        }else {
+            String host = provider.getParameterAsStringOrDefault("CACHE_CLUSTER_ENDPOINT", REDIS_SEARCH_HOST);
+            int port = Integer.parseInt(provider.getParameterAsStringOrDefault("CACHE_CLUSTER_PORT", REDIS_SEARCH_PORT));
+            Map<String, String > credential =provider.getCredentialsAsMap("CACHE_CLUSTER_KEY");
+            String password;
+            if (credential !=null){
+                password = credential.get("token");
+            }else{
+                password = REDIS_SEARCH_KEY;
+            }
+            return new RedisCache(host, port, password, 60, String.class, Groups.class);
+        }
     }
 
     public static String getGroupCacheKey(DpsHeaders headers) {

provider/partition-aws/src/main/resources/application.properties

@@ -34,14 +34,8 @@ aws.dynamodb.table.prefix=${ENVIRONMENT}-
 aws.dynamodb.region=${AWS_REGION}
 aws.dynamodb.endpoint=dynamodb.${AWS_REGION}.amazonaws.com
 
-
 aws.environment=${ENVIRONMENT}
 
-## AWS ElastiCache configuration
-aws.elasticache.cluster.endpoint=${CACHE_CLUSTER_ENDPOINT}
-aws.elasticache.cluster.port=${CACHE_CLUSTER_PORT}
-aws.elasticache.cluster.key=${CACHE_CLUSTER_KEY}
-
 server.ssl.enabled=${SSL_ENABLED:true}
 server.ssl.key-store-type=PKCS12
 server.ssl.key-store=${SSL_KEY_STORE_PATH:/certs/osduonaws.p12}
@@ -49,4 +43,8 @@ server.ssl.key-alias=${SSL_KEY_ALIAS:osduonaws}
 server.ssl.key-password=${SSL_KEY_PASSWORD:}
 server.ssl.key-store-password=${SSL_KEY_STORE_PASSWORD:}
 
-spring.autoconfigure.exclude=org.springframework.boot.autoconfigure.security.SecurityAutoConfiguration
\ No newline at end of file
+spring.autoconfigure.exclude=org.springframework.boot.autoconfigure.security.SecurityAutoConfiguration
+## AWS ElastiCache configuration
+aws.elasticache.cluster.endpoint=${CACHE_CLUSTER_ENDPOINT}
+aws.elasticache.cluster.port=${CACHE_CLUSTER_PORT}
+aws.elasticache.cluster.key=${CACHE_CLUSTER_KEY}
\ No newline at end of file

testing/partition-test-aws/pom.xml

@@ -71,7 +71,7 @@
         <dependency>
             <groupId>org.opengroup.osdu.core.aws</groupId>
             <artifactId>os-core-lib-aws</artifactId>
-            <version>0.10.0</version>
+            <version>0.11.0-SNAPSHOT</version>
         </dependency>
 
         <dependency>