Merge branch 'aws-integration-only' into 'master'

Aws integration only

See merge request !87

.gitignore

@@ -46,3 +46,6 @@ dist/
 # Environment configuration
 *.env
 .envrc
+
+# Python gen files
+*.pyc
\ No newline at end of file

NOTICE

@@ -23,6 +23,20 @@ The following software have components provided under the terms of this license:
 
 - ASM Core (from )
 - ASM based accessors helper used by json-smart (from )
+- AWS Event Stream (from https://github.com/awslabs/aws-eventstream-java)
+- AWS Java SDK :: AWS Core (from https://aws.amazon.com/sdkforjava)
+- AWS Java SDK :: Annotations (from )
+- AWS Java SDK :: Auth (from https://aws.amazon.com/sdkforjava)
+- AWS Java SDK :: Core :: Protocols :: AWS Json Protocol (from https://aws.amazon.com/sdkforjava)
+- AWS Java SDK :: Core :: Protocols :: Protocol Core (from https://aws.amazon.com/sdkforjava)
+- AWS Java SDK :: HTTP Client Interface (from )
+- AWS Java SDK :: HTTP Clients :: Apache (from )
+- AWS Java SDK :: HTTP Clients :: Netty Non-Blocking I/O (from )
+- AWS Java SDK :: Profiles (from https://aws.amazon.com/sdkforjava)
+- AWS Java SDK :: Regions (from )
+- AWS Java SDK :: SDK Core (from https://aws.amazon.com/sdkforjava)
+- AWS Java SDK :: Services :: AWS Simple Systems Management (SSM) (from https://aws.amazon.com/sdkforjava)
+- AWS Java SDK :: Utilities (from )
 - AWS Java SDK for AWS Amplify (from https://aws.amazon.com/sdkforjava)
 - AWS Java SDK for AWS App Mesh (from https://aws.amazon.com/sdkforjava)
 - AWS Java SDK for AWS AppSync (from https://aws.amazon.com/sdkforjava)
@@ -331,8 +345,9 @@ The following software have components provided under the terms of this license:
 - Microsoft Application Insights Log4j 2 Appender (from https://github.com/Microsoft/ApplicationInsights-Java)
 - Microsoft Azure Netty HTTP Client Library (from https://github.com/Azure/azure-sdk-for-java)
 - Microsoft Azure SDK for SQL API of Azure Cosmos DB Service (from https://github.com/Azure/azure-sdk-for-java)
-- Mockito (from http://mockito.org)
 - Mockito (from http://www.mockito.org)
+- Mockito (from http://mockito.org)
+- Netty Reactive Streams HTTP support (from )
 - Netty Reactive Streams Implementation (from )
 - Netty/Buffer (from http://netty.io/)
 - Netty/Buffer (from http://netty.io/)
@@ -341,6 +356,7 @@ The following software have components provided under the terms of this license:
 - Netty/Codec/HTTP (from )
 - Netty/Codec/HTTP (from )
 - Netty/Codec/HTTP2 (from )
+- Netty/Codec/HTTP2 (from )
 - Netty/Codec/Socks (from )
 - Netty/Common (from )
 - Netty/Common (from )
@@ -353,6 +369,7 @@ The following software have components provided under the terms of this license:
 - Netty/Transport (from http://netty.io/)
 - Netty/Transport (from http://netty.io/)
 - Netty/Transport/Native/Unix/Common (from )
+- Netty/Transport/Native/Unix/Common (from )
 - Nimbus Content Type (from https://bitbucket.org/connect2id/nimbus-content-type)
 - Nimbus Content Type (from https://bitbucket.org/connect2id/nimbus-content-type)
 - Nimbus JOSE+JWT (from https://bitbucket.org/connect2id/nimbus-jose-jwt)
@@ -415,6 +432,7 @@ The following software have components provided under the terms of this license:
 - Woodstox (from https://github.com/FasterXML/woodstox)
 - aalto-xml (from )
 - aggs-matrix-stats (from https://github.com/elastic/elasticsearch)
+- aws-ssm-java-caching-client (from https://github.com/awslabs/aws-ssm-java-caching-client)
 - cli (from https://github.com/elastic/elasticsearch)
 - com.google.api.grpc:grpc-google-cloud-pubsub-v1 (from https://github.com/googleapis/googleapis)
 - com.google.api.grpc:proto-google-cloud-logging-v2 (from https://github.com/googleapis/googleapis)
@@ -817,6 +835,7 @@ public-domain
 The following software have components provided under the terms of this license:
 
 - AOP alliance (from http://aopalliance.sourceforge.net)
+- AWS Java SDK :: SDK Core (from https://aws.amazon.com/sdkforjava)
 - AWS SDK for Java - Models (from https://aws.amazon.com/sdkforjava)
 - Asynchronous Http Client (from )
 - Guava: Google Core Libraries for Java (from https://github.com/google/guava.git)

deployments/scripts/aws/Token.py

@@ -19,7 +19,12 @@ import jwt;
 class AwsToken(object):
 
     def get_aws_id_token(self):
-      client = boto3.client('cognito-idp', region_name=os.environ["AWS_REGION"])
+      if os.getenv("AWS_COGNITO_REGION") is not None:      
+          region = os.environ["AWS_COGNITO_REGION"]
+      else:
+          region = os.environ["AWS_REGION"]
+
+      client = boto3.client('cognito-idp', region_name=region)
 
       userAuth = client.initiate_auth(
           ClientId= os.environ['AWS_COGNITO_CLIENT_ID'],

deployments/scripts/aws/bootstrap.sh

@@ -16,9 +16,12 @@ pip3 install -r $AWS_DEPLOYMENTS_SUBDIR/requirements.txt
 
 echo $AWS_BASE_URL
 export AWS_SCHEMA_SERVICE_URL=$AWS_BASE_URL/api/schema-service/v1/schema
-BEARER_TOKEN=`python $AWS_DEPLOYMENTS_SUBDIR/Token.py`
-echo $BEARER_TOKEN
+
+if [ -z "$BEARER_TOKEN" ];
+then BEARER_TOKEN=`python3 $AWS_DEPLOYMENTS_SUBDIR/Token.py`;
 export BEARER_TOKEN=$BEARER_TOKEN
+fi
+echo Bearer Token: $BEARER_TOKEN
 export APP_KEY=""
 export DATA_PARTITION=common
 python3 $DEPLOYMENTS_BASE_DIR/scripts/DeploySharedSchemas.py -l load_sequence.1.0.0.json -u $AWS_SCHEMA_SERVICE_URL

deployments/scripts/aws/prepare-dist.sh

+# Copyright © 2021 Amazon Web Services
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# This script prepares the dist directory for the integration tests.
+# Must be run from the root of the repostiory
+
+# THIS SCRIPT MUST BE RUN FROM THE ROOT FOLDER OF THE SCHEMA SERVICE 
+
+set -e
+
+OUTPUT_DIR="${OUTPUT_DIR:-dist}"
+
+echo "--Copying Schema Boostrap Scripts to ${OUTPUT_DIR}--"
+
+rm -rf "${OUTPUT_DIR}/deployments"
+
+mkdir -p "${OUTPUT_DIR}/deployments"
+
+rsync deployments/* "${OUTPUT_DIR}/deployments/"
+
+cp -r deployments/shared-schemas/ "${OUTPUT_DIR}/deployments/shared-schemas/"
+
+mkdir -p "${OUTPUT_DIR}/deployments/scripts/"
+rsync deployments/scripts/* "${OUTPUT_DIR}/deployments/scripts/"
+cp -r deployments/scripts/templates/ "${OUTPUT_DIR}/deployments/scripts/templates/"
+cp -r deployments/scripts/aws/ "${OUTPUT_DIR}/deployments/scripts/aws/"
\ No newline at end of file

devops/aws/bootstrap.yaml

@@ -2,6 +2,9 @@ aws_bootstrap:
   stage: bootstrap
   image: python:3.8
   needs: ["aws-update-ecs"]
+  extends:
+    - .aws
+    - .aws_variables
   script:
     - pip install -r $AWS_DEPLOYMENTS_SUBDIR/requirements.txt
     - chmod +x $AWS_DEPLOYMENTS_SUBDIR/bootstrap.sh

provider/schema-aws/build-aws/Dockerfile

@@ -16,7 +16,14 @@
 FROM amazoncorretto:8
 
 ARG JAR_FILE=provider/schema-aws/target/*-spring-boot.jar
+
+#Default to using self signed generated TLS cert
+ENV USE_SELF_SIGNED_SSL_CERT true
+
 WORKDIR /
 COPY ${JAR_FILE} app.jar
+COPY /provider/schema-aws/build-aws/ssl.sh /ssl.sh
+COPY /provider/schema-aws/build-aws/entrypoint.sh /entrypoint.sh
 EXPOSE 8080
-ENTRYPOINT java $JAVA_OPTS -jar /app.jar
+
+ENTRYPOINT ["/bin/sh", "-c", ". /entrypoint.sh"]

provider/schema-aws/build-aws/buildspec.yaml

@@ -27,6 +27,8 @@ phases:
     runtime-versions:
       java: corretto8
     commands:
+      # fix error noted here: https://github.com/yarnpkg/yarn/issues/7866
+      - curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -
       - if [ $(echo $CODEBUILD_SOURCE_VERSION | grep -c  ^refs/heads.*) -eq 1 ]; then echo "Branch name found"; else echo "This build only supports branch builds" && exit 1; fi
       - apt-get update -y
       - apt-get install -y maven
@@ -58,7 +60,7 @@ phases:
 
       - echo "Building primary service assemblies..."
       - mvn install -N # required for this service to install the parent pom so that the integration tests will find it
-      - mvn -B test install -pl schema-core,provider/schema-aws -Ddeployment.environment=prod
+      - mvn -B test install -pl schema-core,provider/schema-aws -Ddeployment.environment=prod 
 
 #      Suspended until further notice
 #      - echo "Copying assemblies to dist..."
@@ -70,6 +72,10 @@ phases:
       - chmod +x ./testing/schema-test-aws/build-aws/prepare-dist.sh
       - ./testing/schema-test-aws/build-aws/prepare-dist.sh
 
+      #Copy R3 data def bootstrap scripts to dist
+      - chmod +x ./deployments/scripts/aws/prepare-dist.sh
+      - ./deployments/scripts/aws/prepare-dist.sh
+
       - echo "Logging into Docker Hub..."
       - docker login -u ${DOCKER_USERNAME} -p ${DOCKER_PASSWORD}
 

provider/schema-aws/build-aws/entrypoint.sh

+
+
+if [ -n $USE_SELF_SIGNED_SSL_CERT ];
+then    
+    export SSL_KEY_PASSWORD=$RANDOM$RANDOM$RANDOM;
+    export SSL_KEY_STORE_PASSWORD=$SSL_KEY_PASSWORD;
+    export SSL_KEY_STORE_DIR=/tmp/certs;
+    export SSL_KEY_STORE_NAME=osduonaws.p12;
+    export SSL_KEY_STORE_PATH=$SSL_KEY_STORE_DIR/$SSL_KEY_STORE_NAME;
+    export SSL_KEY_ALIAS=osduonaws;
+    
+    ./ssl.sh;
+fi
+
+java $JAVA_OPTS -jar /app.jar
\ No newline at end of file

provider/schema-aws/build-aws/ssl.sh

+# Copyright © 2021 Amazon Web Services
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/usr/bin/env bash
+
+#Future: Support for using Amazon Cert Manager
+# if [ "$1" == "webserver" ] && [ -n $ACM_CERTIFICATE_ARN ];
+# then
+
+#   aws acm export-certificate --certificate-arn $ACM_CERTIFICATE_ARN --passphrase $(echo -n 'aws123' | openssl base64 -e) | jq -r '"\(.PrivateKey)"' > ${SSL_KEY_PATH}.enc
+#   openssl rsa -in ${SSL_KEY_PATH}.enc -out $SSL_KEY_PATH -passin pass:aws123
+#   aws acm get-certificate --certificate-arn $ACM_CERTIFICATE_ARN | jq -r '"\(.CertificateChain)"' > $SSL_CERT_PATH
+#   aws acm get-certificate --certificate-arn $ACM_CERTIFICATE_ARN | jq -r '"\(.Certificate)"' >> $SSL_CERT_PATH
+
+# fi
+
+if [ -n $USE_SELF_SIGNED_SSL_CERT ];
+then
+    mkdir -p $SSL_KEY_STORE_DIR
+    pushd $SSL_KEY_STORE_DIR
+    keytool -genkeypair -alias $SSL_KEY_ALIAS -keyalg RSA -keysize 2048 -storetype PKCS12 -keystore $SSL_KEY_STORE_NAME -validity 3650 -keypass $SSL_KEY_PASSWORD -storepass $SSL_KEY_PASSWORD -dname "CN=localhost, OU=AWS, O=Energy, L=Houston, ST=TX, C=US"
+    popd
+fi

provider/schema-aws/pom.xml

@@ -67,7 +67,7 @@
         <dependency>
             <groupId>org.opengroup.osdu.core.aws</groupId>
             <artifactId>os-core-lib-aws</artifactId>
-            <version>0.3.7</version>
+            <version>0.3.17</version>
         </dependency>
 
         <dependency>

provider/schema-aws/src/main/resources/application-unittest.properties

+# Copyright � 2020 Amazon Web Services
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+server.servlet.context-path=/api/schema-service/v1
+logging.level.org.springframework.web=${LOG_LEVEL:INFO}
+server.port=${APPLICATION_PORT:8080}
+aws.resource.prefix=${ENVIRONMENT}
+aws.ssm=${SSM_ENABLED:True}
+AUTHORIZE_API=${ENTITLEMENTS_BASE_URL}/api/entitlements/v1
+AUTHORIZE_API_KEY=
+
+LOG_PREFIX=schema
+
+# s3 data bucket
+aws.s3.endpoint=s3.${AWS_REGION}.amazonaws.com
+aws.s3.bucket=${S3_BUCKET:""}
+
+## AWS DynamoDB configuration
+# needed for tenantfactory
+aws.dynamodb.key=kind
+aws.dynamodb.table.prefix=${aws.resource.prefix}-
+aws.region=${AWS_REGION}
+aws.dynamodb.endpoint=dynamodb.${AWS_REGION}.amazonaws.com
+
+
+# if this is turned on then the service tries to connect to elastic search
+management.health.elasticsearch.enabled=false
+
+# Use this property to name your shared tenant name
+shared.tenant.name=common
+
+server.ssl.enabled=false
\ No newline at end of file

provider/schema-aws/src/main/resources/application.properties

-# Copyright  2020 Amazon Web Services
+# Copyright � 2020 Amazon Web Services
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -39,3 +39,10 @@ management.health.elasticsearch.enabled=false
 
 # Use this property to name your shared tenant name
 shared.tenant.name=common
+
+server.ssl.enabled=${SSL_ENABLED:true}
+server.ssl.key-store-type=PKCS12
+server.ssl.key-store=${SSL_KEY_STORE_PATH:/certs/osduonaws.p12}
+server.ssl.key-alias=${SSL_KEY_ALIAS:osduonaws}
+server.ssl.key-password=${SSL_KEY_PASSWORD:}
+server.ssl.key-store-password=${SSL_KEY_STORE_PASSWORD:}
\ No newline at end of file

testing/schema-test-aws/build-aws/prepare-dist.sh

@@ -34,9 +34,9 @@ echo $INTEGRATION_TEST_OUTPUT_BIN_DIR
 rm -rf "$INTEGRATION_TEST_OUTPUT_DIR"
 mkdir -p "$INTEGRATION_TEST_OUTPUT_DIR" && mkdir -p "$INTEGRATION_TEST_OUTPUT_BIN_DIR"
 echo "Building integration testing assemblies and gathering artifacts..."
-mvn install dependency:copy-dependencies -DskipTests -f "$INTEGRATION_TEST_SOURCE_DIR_CORE"/pom.xml -DincludeGroupIds=org.opengroup.osdu -Dmdep.copyPom
+mvn -ntp install dependency:copy-dependencies -DskipItTests=true -f "$INTEGRATION_TEST_SOURCE_DIR_CORE"/pom.xml -DincludeGroupIds=org.opengroup.osdu -Dmdep.copyPom
 cp "$INTEGRATION_TEST_SOURCE_DIR_CORE"/target/dependency/* "${INTEGRATION_TEST_OUTPUT_BIN_DIR}"
-(cd "${INTEGRATION_TEST_OUTPUT_BIN_DIR}" && ls *.jar | sed -e 's/\.jar$//' | xargs -I {} echo mvn install:install-file -Dfile={}.jar -DpomFile={}.pom >> install-deps.sh)
+(cd "${INTEGRATION_TEST_OUTPUT_BIN_DIR}" && ls *.jar | sed -e 's/\.jar$//' | xargs -I {} echo mvn -ntp install:install-file -Dfile={}.jar -DpomFile={}.pom >> install-deps.sh)
 chmod +x "${INTEGRATION_TEST_OUTPUT_BIN_DIR}"/install-deps.sh
 #mvn clean -f "$INTEGRATION_TEST_SOURCE_DIR_CORE"/pom.xml
 cp -R "$INTEGRATION_TEST_SOURCE_DIR_CORE"/* "${INTEGRATION_TEST_OUTPUT_DIR}"/

testing/schema-test-core/pom.xml

@@ -85,7 +85,7 @@
 		<dependency>
 			<groupId>org.opengroup.osdu.core.aws</groupId>
 			<artifactId>os-core-lib-aws</artifactId>
-			<version>0.3.7</version>
+			<version>0.3.16</version>
 		</dependency>
 
 		<dependency>