Merge remote-tracking branch 'origin/master' into 25-upgrade-core-common-dependency

Conflicts:
	NOTICE

.gitlab-ci.yml

@@ -12,7 +12,7 @@ variables:
   OSDU_GCP_APPLICATION_NAME: os-dataset
   OSDU_GCP_FILE_DMS_BUCKET: "file-dms-bucket"
   OSDU_GCP_EXPIRATION_DAYS: 1
-  OSDU_GCP_ENV_VARS: AUTHORIZE_API=$OSDU_GCP_ENTITLEMENTS_URL,GCP_STORAGE_API=$OSDU_GCP_STORAGE_URL,GCP_SCHEMA_API=$OSDU_GCP_SCHEMA_URL/api/schema-service/v1,FILE_DMS_BUCKET=$OSDU_GCP_FILE_DMS_BUCKET,EXPIRATION_DAYS=$OSDU_GCP_EXPIRATION_DAYS,REDIS_GROUP_HOST=$REDIS_GROUP_HOST,REDIS_GROUP_PORT=$OSDU_GCP_REDIS_SEARCH_PORT,GOOGLE_AUDIENCES=$GOOGLE_AUDIENCE,PARTITION_API=$OSDU_GCP_PARTITION_API --vpc-connector=$OSDU_GCP_VPC_CONNECTOR
+  OSDU_GCP_ENV_VARS: AUTHORIZE_API=$OSDU_GCP_ENTITLEMENTS_V2_URL,GCP_STORAGE_API=$OSDU_GCP_STORAGE_URL,GCP_SCHEMA_API=$OSDU_GCP_SCHEMA_URL/api/schema-service/v1,FILE_DMS_BUCKET=$OSDU_GCP_FILE_DMS_BUCKET,EXPIRATION_DAYS=$OSDU_GCP_EXPIRATION_DAYS,REDIS_GROUP_HOST=$REDIS_GROUP_HOST,REDIS_GROUP_PORT=$OSDU_GCP_REDIS_SEARCH_PORT,GOOGLE_AUDIENCES=$GOOGLE_AUDIENCE,PARTITION_API=$OSDU_GCP_PARTITION_API --vpc-connector=$OSDU_GCP_VPC_CONNECTOR
   OSDU_GCP_TEST_SUBDIR: testing/$OSDU_GCP_SERVICE-test-$OSDU_GCP_VENDOR
 
 include:

dataset-core/src/main/java/org/opengroup/osdu/dataset/logging/AuditEvents.java

@@ -18,6 +18,8 @@
 package org.opengroup.osdu.dataset.logging;
 
 import com.google.common.base.Strings;
+
+import java.util.ArrayList;
 import java.util.List;
 import org.opengroup.osdu.core.common.logging.audit.AuditAction;
 import org.opengroup.osdu.core.common.logging.audit.AuditPayload;
@@ -47,25 +49,27 @@ public class AuditEvents {
   }
 
   public AuditPayload getReadStorageInstructionsEvent(AuditStatus status, List<String> resources) {
+    //not including resources to avoid logging sensitive access information
     return AuditPayload.builder()
         .action(AuditAction.READ)
         .status(status)
         .user(this.user)
         .actionId(READ_STORAGE_INSTRUCTIONS_ACTION_ID)
         .message(getStatusMessage(status, READ_STORAGE_INSTRUCTIONS_MESSAGE))
-        .resources(resources)
+        .resources(new ArrayList<>())
         .build();
   }
 
   public AuditPayload getReadRetrievalInstructionsEvent(AuditStatus status,
       List<String> resources) {
+    //not including resources to avoid logging sensitive access information
     return AuditPayload.builder()
         .action(AuditAction.READ)
         .status(status)
         .user(this.user)
         .actionId(READ_RETRIEVAL_INSTRUCTIONS_ACTION_ID)
         .message(getStatusMessage(status, READ_RETRIEVAL_INSTRUCTIONS_MESSAGE))
-        .resources(resources)
+        .resources(new ArrayList<>())
         .build();
   }
 

provider/dataset-aws/maven/settings.xml

 <?xml version="1.0" encoding="UTF-8"?>
 <!--
-  Copyright © 2021 Amazon Web Services
 
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
+Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.​
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
 
-       http://www.apache.org/licenses/LICENSE-2.0
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
 
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
 -->
 
 <settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
@@ -63,15 +64,14 @@
         </server>
     </servers>
 
-    <!-- CodeArtifact doesn't support external repos yet that aren't Maven Central.  ETA Q4 2020. -->
-    <!-- <mirrors> -->
-    <!-- <mirror> -->
-    <!-- <id>aws-osdu-dev-maven</id> -->
-    <!-- <name>aws-osdu-dev-maven</name> -->
-    <!-- <url>https://osdu-dev-888733619319.d.codeartifact.us-east-1.amazonaws.com/maven/osdu-maven/</url> -->
-    <!-- <mirrorOf>*,!gitlab-os-core-common-maven</mirrorOf> -->
-    <!-- </mirror> -->
-    <!-- </mirrors> -->
+    <mirrors>
+        <mirror>
+            <id>aws-osdu-dev-maven</id>
+            <name>aws-osdu-dev-maven</name>
+            <url>https://osdu-dev-${AWS_ACCOUNT_ID}.d.codeartifact.us-east-1.amazonaws.com/maven/osdu-maven/</url>
+            <mirrorOf>central,!gitlab-os-core-common-maven,!gitlab-os-core-lib-aws-maven</mirrorOf>
+        </mirror>
+    </mirrors>
 
     <activeProfiles>
         <activeProfile>credentialsConfiguration</activeProfile>

provider/dataset-aws/pom.xml

@@ -58,7 +58,7 @@
         <dependency>
             <groupId>org.opengroup.osdu.core.aws</groupId>
             <artifactId>os-core-lib-aws</artifactId>
-            <version>0.9.0</version>
+            <version>0.10.0-SNAPSHOT</version>
         </dependency>
         <dependency>
             <groupId>org.opengroup.osdu</groupId>

provider/dataset-aws/src/main/java/org/opengroup/osdu/dataset/provider/aws/DatasetDmsServiceMapImpl.java

@@ -87,8 +87,17 @@ public class DatasetDmsServiceMapImpl implements IDatasetDmsServiceMap {
                 HashMap<String, DmsServiceProperties> resourceTypeToDmsServiceMap = new HashMap<>();                
         
                 for (DynamoDmsRegistration reg : test) {
+
+                    String apiBase = "";
+                    if (StringUtils.isNotEmpty(DMS_API_BASE)) {
+                        apiBase = DMS_API_BASE;
+                    }
+                    else {
+                        apiBase = reg.getApiBase();
+                    }
+
                     resourceTypeToDmsServiceMap.put(reg.getDatasetKind(), new DmsServiceProperties(
-                        StringUtils.join(DMS_API_BASE, reg.getRoute()),
+                        StringUtils.join(apiBase, reg.getRoute()),
                         reg.getIsStorageAllowed()
                     ));
                 }

provider/dataset-aws/src/main/java/org/opengroup/osdu/dataset/provider/aws/cache/DmsRegistrationCache.java

@@ -24,8 +24,8 @@ import org.springframework.stereotype.Component;
 
 @Component
 public class DmsRegistrationCache extends RedisCache<String, DmsRegistrations> {
-    public DmsRegistrationCache(@Value("${aws.elasticache.cluster.endpoint}") final String REDIS_HOST, @Value("${aws.elasticache.cluster.port}") final String REDIS_PORT) {
-        super(REDIS_HOST, Integer.parseInt(REDIS_PORT), 300, String.class, DmsRegistrations.class);
+    public DmsRegistrationCache(@Value("${aws.elasticache.cluster.endpoint}") final String REDIS_HOST, @Value("${aws.elasticache.cluster.port}") final String REDIS_PORT, @Value("${aws.elasticache.cluster.key}") final String REDIS_KEY) {
+        super(REDIS_HOST, Integer.parseInt(REDIS_PORT), REDIS_KEY, 300, String.class, DmsRegistrations.class);
     }
 
     public static String getCacheKey(DpsHeaders headers) {

provider/dataset-aws/src/main/java/org/opengroup/osdu/dataset/provider/aws/cache/GroupCache.java

@@ -23,8 +23,8 @@ import org.springframework.stereotype.Component;
 
 @Component
 public class GroupCache extends RedisCache<String, Groups> {
-    public GroupCache(@Value("${aws.elasticache.cluster.endpoint}") final String REDIS_GROUP_HOST, @Value("${aws.elasticache.cluster.port}") final String REDIS_GROUP_PORT) {
-        super(REDIS_GROUP_HOST, Integer.parseInt(REDIS_GROUP_PORT), 60, String.class, Groups.class);
+    public GroupCache(@Value("${aws.elasticache.cluster.endpoint}") final String REDIS_GROUP_HOST, @Value("${aws.elasticache.cluster.port}") final String REDIS_GROUP_PORT, @Value("${aws.elasticache.cluster.key}") final String REDIS_GROUP_KEY ) {
+        super(REDIS_GROUP_HOST, Integer.parseInt(REDIS_GROUP_PORT), REDIS_GROUP_KEY, 60, String.class, Groups.class);
     }
 
     public static String getGroupCacheKey(DpsHeaders headers) {

provider/dataset-aws/src/main/java/org/opengroup/osdu/dataset/provider/aws/model/DynamoDmsRegistration.java

@@ -17,6 +17,9 @@ public class DynamoDmsRegistration {
     @DynamoDBHashKey(attributeName = "datasetKind")
     private String datasetKind;
 
+    @DynamoDBAttribute(attributeName = "apiBase")
+    private String apiBase;
+
     @DynamoDBAttribute(attributeName = "route")
     private String route;
 

provider/dataset-aws/src/main/resources/application.properties

@@ -17,7 +17,8 @@ LOG_PREFIX=dataset
 AUTHORIZE_API=${ENTITLEMENTS_BASE_URL}/api/entitlements/v2
 STORAGE_API=${STORAGE_BASE_URL}/api/storage/v2
 SCHEMA_API=${SCHEMA_BASE_URL}/api/schema-service/v1
-DMS_API_BASE=${DMS_BASE_URL}
+#Use this for local dev, on K8S, the service mesh handles this
+DMS_API_BASE=${DMS_BASE_URL:}
 
 server.servlet.contextPath=/api/dataset/v1/
 logging.level.org.springframework.web=${LOG_LEVEL:INFO}
@@ -38,7 +39,7 @@ aws.region=${AWS_REGION}
 ## AWS ElastiCache configuration
 aws.elasticache.cluster.endpoint=${CACHE_CLUSTER_ENDPOINT}
 aws.elasticache.cluster.port=${CACHE_CLUSTER_PORT}
-
+aws.elasticache.cluster.key=${CACHE_CLUSTER_KEY}
 
 # aws.ssm=${SSM_ENABLED}
 aws.parameter.prefix=/osdu/${ENVIRONMENT}
@@ -55,4 +56,6 @@ server.ssl.key-store-type=PKCS12
 server.ssl.key-store=${SSL_KEY_STORE_PATH:/certs/osduonaws.p12}
 server.ssl.key-alias=${SSL_KEY_ALIAS:osduonaws}
 server.ssl.key-password=${SSL_KEY_PASSWORD:}
-server.ssl.key-store-password=${SSL_KEY_STORE_PASSWORD:}
\ No newline at end of file
+server.ssl.key-store-password=${SSL_KEY_STORE_PASSWORD:}
+
+spring.autoconfigure.exclude=org.springframework.boot.autoconfigure.security.SecurityAutoConfiguration
\ No newline at end of file

testing/dataset-test-aws/pom.xml

@@ -40,7 +40,7 @@
         <dependency>
             <groupId>org.opengroup.osdu.core.aws</groupId>
             <artifactId>os-core-lib-aws</artifactId>
-            <version>0.9.0</version>
+            <version>0.10.0-SNAPSHOT</version>
         </dependency>
 
         <dependency>