Squashed commit of the following

commit 1d1b4ef8
Author: zhijie wang <wanzhiji@amazon.com>
Date: Mon Dec 21 2020 15:34:26 GMT-0800 (Pacific Standard Time)

    bump os-core-lib-aws version, supress maven transport progress log

partition-core/src/main/java/org/opengroup/osdu/partition/middleware/PartitionFilter.java

@@ -15,6 +15,7 @@
 package org.opengroup.osdu.partition.middleware;
 
 import org.opengroup.osdu.core.common.http.ResponseHeaders;
+import org.opengroup.osdu.core.common.http.ResponseHeadersFactory;
 import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
 import org.opengroup.osdu.core.common.model.http.DpsHeaders;
 import org.opengroup.osdu.core.common.model.http.Request;
@@ -42,6 +43,12 @@ public class PartitionFilter implements Filter {
     @Value("${ACCEPT_HTTP:false}")
     private boolean acceptHttp;
 
+    // defaults to * for any front-end, string must be comma-delimited if more than one domain
+    @Value("${ACCESS_CONTROL_ALLOW_ORIGIN_DOMAINS:*}")
+    String ACCESS_CONTROL_ALLOW_ORIGIN_DOMAINS;
+
+    private ResponseHeadersFactory responseHeadersFactory = new ResponseHeadersFactory();
+
     @Override
     public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException {
         HttpServletRequest httpServletRequest = (HttpServletRequest) request;
@@ -105,8 +112,8 @@ public class PartitionFilter implements Filter {
     }
 
     private void setResponseHeaders(HttpServletResponse httpServletResponse) {
-        Map<String, List<Object>> standardHeaders = ResponseHeaders.STANDARD_RESPONSE_HEADERS;
-        for (Map.Entry<String, List<Object>> header : standardHeaders.entrySet()) {
+        Map<String, String> responseHeaders = responseHeadersFactory.getResponseHeaders(ACCESS_CONTROL_ALLOW_ORIGIN_DOMAINS);
+        for(Map.Entry<String, String> header : responseHeaders.entrySet()){
             if("Cache-Control".equalsIgnoreCase(header.getKey())){
                 httpServletResponse.addHeader(header.getKey(), "private, max-age=300");
             }else {

partition-core/src/test/java/org/opengroup/osdu/partition/middleware/PartitionFilterTest.java

@@ -48,26 +48,26 @@ public class PartitionFilterTest {
         HttpServletRequest httpServletRequest = mock(HttpServletRequest.class);
         HttpServletResponse httpServletResponse = mock(HttpServletResponse.class);
         when(httpServletRequest.getRequestURL()).thenReturn(new StringBuffer("https://test.com"));
-        FilterChain filterChain = mock(FilterChain.class);
-        when(headers.getCorrelationId()).thenReturn("correlation-id-value");
-        when(httpServletRequest.getMethod()).thenReturn("POST");
+        FilterChain filterChain = Mockito.mock(FilterChain.class);
+        Mockito.when(headers.getCorrelationId()).thenReturn("correlation-id-value");
+        Mockito.when(httpServletRequest.getMethod()).thenReturn("POST");
+        org.springframework.test.util.ReflectionTestUtils.setField(partitionFilter, "ACCESS_CONTROL_ALLOW_ORIGIN_DOMAINS", "custom-domain");
 
         partitionFilter.doFilter(httpServletRequest, httpServletResponse, filterChain);
 
-        verify(httpServletResponse).addHeader("Access-Control-Allow-Origin", singletonList("*").toString());
-        verify(httpServletResponse).addHeader("Access-Control-Allow-Headers", singletonList("origin, content-type, accept, authorization, data-partition-id, correlation-id, appkey").toString());
-        verify(httpServletResponse).addHeader("Access-Control-Allow-Methods", singletonList("GET, POST, PUT, DELETE, OPTIONS, HEAD, PATCH").toString());
-        verify(httpServletResponse).addHeader("Access-Control-Allow-Credentials", singletonList("true").toString());
-        verify(httpServletResponse).addHeader("X-Frame-Options", singletonList("DENY").toString());
-        verify(httpServletResponse).addHeader("X-XSS-Protection", singletonList("1; mode=block").toString());
-        verify(httpServletResponse).addHeader("X-Content-Type-Options", singletonList("nosniff").toString());
-        verify(httpServletResponse).addHeader("Cache-Control", "private, max-age=300");
-        verify(httpServletResponse).addHeader("Content-Security-Policy", singletonList("default-src 'self'").toString());
-        verify(httpServletResponse).addHeader("Strict-Transport-Security", singletonList("max-age=31536000; includeSubDomains").toString());
-        verify(httpServletResponse).addHeader("Expires", singletonList("0").toString());
-        verify(httpServletResponse).addHeader("correlation-id", "correlation-id-value");
-        verify(filterChain).doFilter(httpServletRequest, httpServletResponse);
-        verify(logger).request(Mockito.any(Request.class));
+        Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Origin", "custom-domain");
+        Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Headers", "origin, content-type, accept, authorization, data-partition-id, correlation-id, appkey");
+        Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS, HEAD, PATCH");
+        Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Credentials", "true");
+        Mockito.verify(httpServletResponse).addHeader("X-Frame-Options", "DENY");
+        Mockito.verify(httpServletResponse).addHeader("X-XSS-Protection", "1; mode=block");
+        Mockito.verify(httpServletResponse).addHeader("X-Content-Type-Options", "nosniff");
+        Mockito.verify(httpServletResponse).addHeader("Cache-Control", "private, max-age=300");
+        Mockito.verify(httpServletResponse).addHeader("Content-Security-Policy", "default-src 'self'");
+        Mockito.verify(httpServletResponse).addHeader("Strict-Transport-Security", "max-age=31536000; includeSubDomains");
+        Mockito.verify(httpServletResponse).addHeader("Expires", "0");
+        Mockito.verify(httpServletResponse).addHeader("correlation-id", "correlation-id-value");
+        Mockito.verify(filterChain).doFilter(httpServletRequest, httpServletResponse);
     }
 
     @Test
@@ -77,6 +77,7 @@ public class PartitionFilterTest {
         when(httpServletRequest.getRequestURL()).thenReturn(new StringBuffer("http://test.com"));
         FilterChain filterChain = mock(FilterChain.class);
         when(httpServletRequest.getMethod()).thenReturn("POST");
+        org.springframework.test.util.ReflectionTestUtils.setField(partitionFilter, "ACCESS_CONTROL_ALLOW_ORIGIN_DOMAINS", "custom-domain");
 
         partitionFilter.doFilter(httpServletRequest, httpServletResponse, filterChain);
 
@@ -90,6 +91,7 @@ public class PartitionFilterTest {
         when(httpServletRequest.getRequestURL()).thenReturn(new StringBuffer("https://test.com"));
         FilterChain filterChain = mock(FilterChain.class);
         when(httpServletRequest.getMethod()).thenReturn("OPTIONS");
+        org.springframework.test.util.ReflectionTestUtils.setField(partitionFilter, "ACCESS_CONTROL_ALLOW_ORIGIN_DOMAINS", "custom-domain");
 
         partitionFilter.doFilter(httpServletRequest, httpServletResponse, filterChain);
 

pom.xml

@@ -26,7 +26,7 @@
     <maven.compiler.target>1.8</maven.compiler.target>
     <maven.compiler.source>1.8</maven.compiler.source>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-    <os-core-common.version>0.3.16</os-core-common.version>
+    <os-core-common.version>0.3.28</os-core-common.version>
     <tomcat_embed_core_version>9.0.37</tomcat_embed_core_version>
   </properties>
 

provider/partition-aws/build-aws/Dockerfile

@@ -16,7 +16,14 @@
 FROM amazoncorretto:8
 
 ARG JAR_FILE=provider/partition-aws/target/*spring-boot.jar
+
+#Default to using self signed generated TLS cert
+ENV USE_SELF_SIGNED_SSL_CERT true
+
 WORKDIR /
 COPY ${JAR_FILE} app.jar
+COPY /provider/partition-aws/build-aws/ssl.sh /ssl.sh
+COPY /provider/partition-aws/build-aws/entrypoint.sh /entrypoint.sh
 EXPOSE 8080
-ENTRYPOINT java $JAVA_OPTS -jar /app.jar
+
+ENTRYPOINT ["/bin/sh", "-c", ". /entrypoint.sh"]

provider/partition-aws/build-aws/buildspec.yaml

@@ -27,6 +27,8 @@ phases:
     runtime-versions:
       java: corretto8
     commands:
+      # fix error noted here: https://github.com/yarnpkg/yarn/issues/7866
+      - curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -
       - if [ $(echo $CODEBUILD_SOURCE_VERSION | grep -c  ^refs/heads.*) -eq 1 ]; then echo "Branch name found"; else echo "This build only supports branch builds" && exit 1; fi
       - apt-get update -y
       - apt-get install -y maven

provider/partition-aws/build-aws/entrypoint.sh

+
+
+if [ -n $USE_SELF_SIGNED_SSL_CERT ];
+then    
+    export SSL_KEY_PASSWORD=$RANDOM$RANDOM$RANDOM;
+    export SSL_KEY_STORE_PASSWORD=$SSL_KEY_PASSWORD;
+    export SSL_KEY_STORE_DIR=/tmp/certs;
+    export SSL_KEY_STORE_NAME=osduonaws.p12;
+    export SSL_KEY_STORE_PATH=$SSL_KEY_STORE_DIR/$SSL_KEY_STORE_NAME;
+    export SSL_KEY_ALIAS=osduonaws;
+    
+    ./ssl.sh;
+fi
+
+java $JAVA_OPTS -jar /app.jar
\ No newline at end of file

provider/partition-aws/build-aws/ssl.sh

+# Copyright © 2021 Amazon Web Services
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/usr/bin/env bash
+
+#Future: Support for using Amazon Cert Manager
+# if [ "$1" == "webserver" ] && [ -n $ACM_CERTIFICATE_ARN ];
+# then
+
+#   aws acm export-certificate --certificate-arn $ACM_CERTIFICATE_ARN --passphrase $(echo -n 'aws123' | openssl base64 -e) | jq -r '"\(.PrivateKey)"' > ${SSL_KEY_PATH}.enc
+#   openssl rsa -in ${SSL_KEY_PATH}.enc -out $SSL_KEY_PATH -passin pass:aws123
+#   aws acm get-certificate --certificate-arn $ACM_CERTIFICATE_ARN | jq -r '"\(.CertificateChain)"' > $SSL_CERT_PATH
+#   aws acm get-certificate --certificate-arn $ACM_CERTIFICATE_ARN | jq -r '"\(.Certificate)"' >> $SSL_CERT_PATH
+
+# fi
+
+if [ -n $USE_SELF_SIGNED_SSL_CERT ];
+then
+    mkdir -p $SSL_KEY_STORE_DIR
+    pushd $SSL_KEY_STORE_DIR
+    keytool -genkeypair -alias $SSL_KEY_ALIAS -keyalg RSA -keysize 2048 -storetype PKCS12 -keystore $SSL_KEY_STORE_NAME -validity 3650 -keypass $SSL_KEY_PASSWORD -storepass $SSL_KEY_PASSWORD -dname "CN=localhost, OU=AWS, O=Energy, L=Houston, ST=TX, C=US"
+    popd
+fi

provider/partition-aws/pom.xml

@@ -58,7 +58,7 @@
         <dependency>
             <groupId>org.opengroup.osdu.core.aws</groupId>
             <artifactId>os-core-lib-aws</artifactId>
-            <version>0.3.7</version>
+            <version>0.3.17</version>
         </dependency>
         <dependency>
             <groupId>org.opengroup.osdu</groupId>

provider/partition-aws/src/main/java/org/opengroup/osdu/partition/provider/aws/security/EntitlementsClientFactory.java

@@ -18,10 +18,13 @@ package org.opengroup.osdu.partition.provider.aws.security;
 import org.opengroup.osdu.core.common.entitlements.EntitlementsAPIConfig;
 import org.opengroup.osdu.core.common.entitlements.EntitlementsFactory;
 import org.opengroup.osdu.core.common.entitlements.IEntitlementsFactory;
+import org.opengroup.osdu.core.common.http.json.HttpResponseBodyMapper;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.beans.factory.config.AbstractFactoryBean;
 import org.springframework.stereotype.Component;
 
+import javax.inject.Inject;
+
 @Component
 public class EntitlementsClientFactory extends AbstractFactoryBean<IEntitlementsFactory> {
 
@@ -31,14 +34,16 @@ public class EntitlementsClientFactory extends AbstractFactoryBean<IEntitlements
 	@Value("${AUTHORIZE_API_KEY:}")
 	private String AUTHORIZE_API_KEY;
 
-	@Override
-	protected IEntitlementsFactory createInstance() throws Exception {
+	@Inject
+	private HttpResponseBodyMapper httpResponseBodyMapper;
 
+	@Override
+	protected IEntitlementsFactory createInstance() {
 		return new EntitlementsFactory(EntitlementsAPIConfig
 				.builder()
 				.rootUrl(AUTHORIZE_API)
 				.apiKey(AUTHORIZE_API_KEY)
-				.build());
+				.build(), httpResponseBodyMapper);
 	}
 
 	@Override

provider/partition-aws/src/main/resources/application.properties

@@ -36,4 +36,11 @@ aws.dynamodb.endpoint=dynamodb.${AWS_REGION}.amazonaws.com
 
 ## AWS ElastiCache configuration
 aws.elasticache.cluster.endpoint=${CACHE_CLUSTER_ENDPOINT}
-aws.elasticache.cluster.port=${CACHE_CLUSTER_PORT}
\ No newline at end of file
+aws.elasticache.cluster.port=${CACHE_CLUSTER_PORT}
+
+server.ssl.enabled=${SSL_ENABLED:true}
+server.ssl.key-store-type=PKCS12
+server.ssl.key-store=${SSL_KEY_STORE_PATH:/certs/osduonaws.p12}
+server.ssl.key-alias=${SSL_KEY_ALIAS:osduonaws}
+server.ssl.key-password=${SSL_KEY_PASSWORD:}
+server.ssl.key-store-password=${SSL_KEY_STORE_PASSWORD:}
\ No newline at end of file

provider/partition-gcp/pom.xml

@@ -21,6 +21,11 @@
       <artifactId>partition-core</artifactId>
       <version>0.6.0-SNAPSHOT</version>
     </dependency>
+    <dependency>
+      <groupId>org.opengroup.osdu</groupId>
+      <artifactId>os-core-common</artifactId>
+      <version>0.3.16</version>
+    </dependency>
 
     <dependency>
       <groupId>org.opengroup.osdu</groupId>

provider/partition-ibm/pom.xml

@@ -25,11 +25,11 @@
 
     <dependencies>
         <!-- Internal packages -->
-        <!-- <dependency>
+        <dependency>
             <groupId>org.opengroup.osdu</groupId>
             <artifactId>os-core-common</artifactId>
-            <version>${os-core-common.version}</version>
-        </dependency> -->
+            <version>0.3.16</version>
+        </dependency>
         <dependency>
             <groupId>org.opengroup.osdu</groupId>
             <artifactId>os-core-lib-ibm</artifactId>

testing/partition-test-aws/build-aws/prepare-dist.sh

@@ -38,13 +38,13 @@ echo $INTEGRATION_TEST_OUTPUT_BIN_DIR
 rm -rf "$INTEGRATION_TEST_OUTPUT_DIR"
 mkdir -p "$INTEGRATION_TEST_OUTPUT_DIR" && mkdir -p "$INTEGRATION_TEST_OUTPUT_BIN_DIR"
 echo "Building integration testing assemblies and gathering artifacts..."
-mvn install -f "$INTEGRATION_TEST_SOURCE_DIR_CORE"/pom.xml
-mvn install dependency:copy-dependencies -DskipTests -f "$INTEGRATION_TEST_SOURCE_DIR_AWS"/pom.xml -DincludeGroupIds=org.opengroup.osdu -Dmdep.copyPom
+mvn -ntp install -f "$INTEGRATION_TEST_SOURCE_DIR_CORE"/pom.xml
+mvn -ntp install dependency:copy-dependencies -DskipTests -f "$INTEGRATION_TEST_SOURCE_DIR_AWS"/pom.xml -DincludeGroupIds=org.opengroup.osdu -Dmdep.copyPom
 cp "$INTEGRATION_TEST_SOURCE_DIR_AWS"/target/dependency/* "${INTEGRATION_TEST_OUTPUT_BIN_DIR}"
-(cd "${INTEGRATION_TEST_OUTPUT_BIN_DIR}" && ls *.jar | sed -e 's/\.jar$//' | xargs -I {} echo mvn install:install-file -Dfile={}.jar -DpomFile={}.pom >> install-deps.sh)
+(cd "${INTEGRATION_TEST_OUTPUT_BIN_DIR}" && ls *.jar | sed -e 's/\.jar$//' | xargs -I {} echo mvn -ntp install:install-file -Dfile={}.jar -DpomFile={}.pom >> install-deps.sh)
 chmod +x "${INTEGRATION_TEST_OUTPUT_BIN_DIR}"/install-deps.sh
-mvn clean -f "$INTEGRATION_TEST_SOURCE_DIR_AWS"/pom.xml
+mvn -ntp clean -f "$INTEGRATION_TEST_SOURCE_DIR_AWS"/pom.xml
 cp -R "$INTEGRATION_TEST_SOURCE_DIR_AWS"/* "${INTEGRATION_TEST_OUTPUT_DIR}"/
 
 #copy testing parent pom to output
-cp "$INTEGRATION_TEST_SOURCE_DIR/pom.xml" "${OUTPUT_DIR}/testing"
\ No newline at end of file
+cp "$INTEGRATION_TEST_SOURCE_DIR/pom.xml" "${OUTPUT_DIR}/testing"

testing/partition-test-aws/build-aws/run-tests.sh

@@ -46,7 +46,7 @@ export ENVIRONMENT=$RESOURCE_PREFIX
 
 #### RUN INTEGRATION TEST #########################################################################
 
-mvn test -f "$SCRIPT_SOURCE_DIR"/../pom.xml
+mvn -ntp test -f "$SCRIPT_SOURCE_DIR"/../pom.xml
 TEST_EXIT_CODE=$?
 
 #### COPY TEST REPORTS #########################################################################
@@ -59,4 +59,4 @@ fi
 
 echo "### Partition Service Integration Tests Finished ###"
 
-exit $TEST_EXIT_CODE
\ No newline at end of file
+exit $TEST_EXIT_CODE

testing/partition-test-aws/pom.xml

@@ -71,7 +71,7 @@
         <dependency>
             <groupId>org.opengroup.osdu.core.aws</groupId>
             <artifactId>os-core-lib-aws</artifactId>
-            <version>0.3.7</version>
+            <version>0.3.16</version>
         </dependency>
 
         <dependency>

testing/partition-test-core/src/main/java/org/opengroup/osdu/partition/util/BaseTestTemplate.java

@@ -68,17 +68,17 @@ public abstract class BaseTestTemplate extends TestBase {
         ClientResponse response = descriptor.run(getId(), token);
         deleteResource();
         assertEquals(error(response.getStatus() == 204 ? "" : response.getEntity(String.class)), expectedOkResponseCode(), response.getStatus());
-        assertEquals("[GET, POST, PUT, DELETE, OPTIONS, HEAD, PATCH]", response.getHeaders().getFirst("Access-Control-Allow-Methods"));
-        assertEquals("[origin, content-type, accept, authorization, data-partition-id, correlation-id, appkey]", response.getHeaders().getFirst("Access-Control-Allow-Headers"));
-        assertEquals("[*]", response.getHeaders().getFirst("Access-Control-Allow-Origin"));
-        assertEquals("[true]", response.getHeaders().getFirst("Access-Control-Allow-Credentials"));
-        assertEquals("[default-src 'self']", response.getHeaders().getFirst("Content-Security-Policy"));
-        assertEquals("[max-age=31536000; includeSubDomains]", response.getHeaders().getFirst("Strict-Transport-Security"));
-        assertEquals("[0]", response.getHeaders().getFirst("Expires"));
+        assertEquals("GET, POST, PUT, DELETE, OPTIONS, HEAD, PATCH", response.getHeaders().getFirst("Access-Control-Allow-Methods"));
+        assertEquals("origin, content-type, accept, authorization, data-partition-id, correlation-id, appkey", response.getHeaders().getFirst("Access-Control-Allow-Headers"));
+        assertEquals("*", response.getHeaders().getFirst("Access-Control-Allow-Origin"));
+        assertEquals("true", response.getHeaders().getFirst("Access-Control-Allow-Credentials"));
+        assertEquals("default-src 'self'", response.getHeaders().getFirst("Content-Security-Policy"));
+        assertEquals("max-age=31536000; includeSubDomains", response.getHeaders().getFirst("Strict-Transport-Security"));
+        assertEquals("0", response.getHeaders().getFirst("Expires"));
         assertEquals("DENY", response.getHeaders().getFirst("X-Frame-Options"));
         assertEquals("private, max-age=300", response.getHeaders().getFirst("Cache-Control"));
-        assertEquals("[1; mode=block]", response.getHeaders().getFirst("X-XSS-Protection"));
-        assertEquals("[nosniff]", response.getHeaders().getFirst("X-Content-Type-Options"));
+        assertEquals("1; mode=block", response.getHeaders().getFirst("X-XSS-Protection"));
+        assertEquals("nosniff", response.getHeaders().getFirst("X-Content-Type-Options"));
     }
 
     @Test