Commit 9c61218a authored by Matt Wise's avatar Matt Wise
Browse files

Merge branch 'aws-integration-only' into 'master'

Aws integration only

See merge request !87
parents a1c526ca 16a90893
Pipeline #28861 failed with stages
in 70 minutes and 55 seconds
......@@ -46,3 +46,6 @@ dist/
# Environment configuration
*.env
.envrc
# Python gen files
*.pyc
\ No newline at end of file
......@@ -23,6 +23,20 @@ The following software have components provided under the terms of this license:
- ASM Core (from )
- ASM based accessors helper used by json-smart (from )
- AWS Event Stream (from https://github.com/awslabs/aws-eventstream-java)
- AWS Java SDK :: AWS Core (from https://aws.amazon.com/sdkforjava)
- AWS Java SDK :: Annotations (from )
- AWS Java SDK :: Auth (from https://aws.amazon.com/sdkforjava)
- AWS Java SDK :: Core :: Protocols :: AWS Json Protocol (from https://aws.amazon.com/sdkforjava)
- AWS Java SDK :: Core :: Protocols :: Protocol Core (from https://aws.amazon.com/sdkforjava)
- AWS Java SDK :: HTTP Client Interface (from )
- AWS Java SDK :: HTTP Clients :: Apache (from )
- AWS Java SDK :: HTTP Clients :: Netty Non-Blocking I/O (from )
- AWS Java SDK :: Profiles (from https://aws.amazon.com/sdkforjava)
- AWS Java SDK :: Regions (from )
- AWS Java SDK :: SDK Core (from https://aws.amazon.com/sdkforjava)
- AWS Java SDK :: Services :: AWS Simple Systems Management (SSM) (from https://aws.amazon.com/sdkforjava)
- AWS Java SDK :: Utilities (from )
- AWS Java SDK for AWS Amplify (from https://aws.amazon.com/sdkforjava)
- AWS Java SDK for AWS App Mesh (from https://aws.amazon.com/sdkforjava)
- AWS Java SDK for AWS AppSync (from https://aws.amazon.com/sdkforjava)
......@@ -331,8 +345,9 @@ The following software have components provided under the terms of this license:
- Microsoft Application Insights Log4j 2 Appender (from https://github.com/Microsoft/ApplicationInsights-Java)
- Microsoft Azure Netty HTTP Client Library (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure SDK for SQL API of Azure Cosmos DB Service (from https://github.com/Azure/azure-sdk-for-java)
- Mockito (from http://mockito.org)
- Mockito (from http://www.mockito.org)
- Mockito (from http://mockito.org)
- Netty Reactive Streams HTTP support (from )
- Netty Reactive Streams Implementation (from )
- Netty/Buffer (from http://netty.io/)
- Netty/Buffer (from http://netty.io/)
......@@ -341,6 +356,7 @@ The following software have components provided under the terms of this license:
- Netty/Codec/HTTP (from )
- Netty/Codec/HTTP (from )
- Netty/Codec/HTTP2 (from )
- Netty/Codec/HTTP2 (from )
- Netty/Codec/Socks (from )
- Netty/Common (from )
- Netty/Common (from )
......@@ -353,6 +369,7 @@ The following software have components provided under the terms of this license:
- Netty/Transport (from http://netty.io/)
- Netty/Transport (from http://netty.io/)
- Netty/Transport/Native/Unix/Common (from )
- Netty/Transport/Native/Unix/Common (from )
- Nimbus Content Type (from https://bitbucket.org/connect2id/nimbus-content-type)
- Nimbus Content Type (from https://bitbucket.org/connect2id/nimbus-content-type)
- Nimbus JOSE+JWT (from https://bitbucket.org/connect2id/nimbus-jose-jwt)
......@@ -415,6 +432,7 @@ The following software have components provided under the terms of this license:
- Woodstox (from https://github.com/FasterXML/woodstox)
- aalto-xml (from )
- aggs-matrix-stats (from https://github.com/elastic/elasticsearch)
- aws-ssm-java-caching-client (from https://github.com/awslabs/aws-ssm-java-caching-client)
- cli (from https://github.com/elastic/elasticsearch)
- com.google.api.grpc:grpc-google-cloud-pubsub-v1 (from https://github.com/googleapis/googleapis)
- com.google.api.grpc:proto-google-cloud-logging-v2 (from https://github.com/googleapis/googleapis)
......@@ -817,6 +835,7 @@ public-domain
The following software have components provided under the terms of this license:
- AOP alliance (from http://aopalliance.sourceforge.net)
- AWS Java SDK :: SDK Core (from https://aws.amazon.com/sdkforjava)
- AWS SDK for Java - Models (from https://aws.amazon.com/sdkforjava)
- Asynchronous Http Client (from )
- Guava: Google Core Libraries for Java (from https://github.com/google/guava.git)
......
......@@ -19,7 +19,12 @@ import jwt;
class AwsToken(object):
def get_aws_id_token(self):
client = boto3.client('cognito-idp', region_name=os.environ["AWS_REGION"])
if os.getenv("AWS_COGNITO_REGION") is not None:
region = os.environ["AWS_COGNITO_REGION"]
else:
region = os.environ["AWS_REGION"]
client = boto3.client('cognito-idp', region_name=region)
userAuth = client.initiate_auth(
ClientId= os.environ['AWS_COGNITO_CLIENT_ID'],
......
......@@ -16,9 +16,12 @@ pip3 install -r $AWS_DEPLOYMENTS_SUBDIR/requirements.txt
echo $AWS_BASE_URL
export AWS_SCHEMA_SERVICE_URL=$AWS_BASE_URL/api/schema-service/v1/schema
BEARER_TOKEN=`python $AWS_DEPLOYMENTS_SUBDIR/Token.py`
echo $BEARER_TOKEN
if [ -z "$BEARER_TOKEN" ];
then BEARER_TOKEN=`python3 $AWS_DEPLOYMENTS_SUBDIR/Token.py`;
export BEARER_TOKEN=$BEARER_TOKEN
fi
echo Bearer Token: $BEARER_TOKEN
export APP_KEY=""
export DATA_PARTITION=common
python3 $DEPLOYMENTS_BASE_DIR/scripts/DeploySharedSchemas.py -l load_sequence.1.0.0.json -u $AWS_SCHEMA_SERVICE_URL
# Copyright © 2021 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This script prepares the dist directory for the integration tests.
# Must be run from the root of the repostiory
# THIS SCRIPT MUST BE RUN FROM THE ROOT FOLDER OF THE SCHEMA SERVICE
set -e
OUTPUT_DIR="${OUTPUT_DIR:-dist}"
echo "--Copying Schema Boostrap Scripts to ${OUTPUT_DIR}--"
rm -rf "${OUTPUT_DIR}/deployments"
mkdir -p "${OUTPUT_DIR}/deployments"
rsync deployments/* "${OUTPUT_DIR}/deployments/"
cp -r deployments/shared-schemas/ "${OUTPUT_DIR}/deployments/shared-schemas/"
mkdir -p "${OUTPUT_DIR}/deployments/scripts/"
rsync deployments/scripts/* "${OUTPUT_DIR}/deployments/scripts/"
cp -r deployments/scripts/templates/ "${OUTPUT_DIR}/deployments/scripts/templates/"
cp -r deployments/scripts/aws/ "${OUTPUT_DIR}/deployments/scripts/aws/"
\ No newline at end of file
......@@ -2,6 +2,9 @@ aws_bootstrap:
stage: bootstrap
image: python:3.8
needs: ["aws-update-ecs"]
extends:
- .aws
- .aws_variables
script:
- pip install -r $AWS_DEPLOYMENTS_SUBDIR/requirements.txt
- chmod +x $AWS_DEPLOYMENTS_SUBDIR/bootstrap.sh
......
......@@ -16,7 +16,14 @@
FROM amazoncorretto:8
ARG JAR_FILE=provider/schema-aws/target/*-spring-boot.jar
#Default to using self signed generated TLS cert
ENV USE_SELF_SIGNED_SSL_CERT true
WORKDIR /
COPY ${JAR_FILE} app.jar
COPY /provider/schema-aws/build-aws/ssl.sh /ssl.sh
COPY /provider/schema-aws/build-aws/entrypoint.sh /entrypoint.sh
EXPOSE 8080
ENTRYPOINT java $JAVA_OPTS -jar /app.jar
ENTRYPOINT ["/bin/sh", "-c", ". /entrypoint.sh"]
......@@ -27,6 +27,8 @@ phases:
runtime-versions:
java: corretto8
commands:
# fix error noted here: https://github.com/yarnpkg/yarn/issues/7866
- curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -
- if [ $(echo $CODEBUILD_SOURCE_VERSION | grep -c ^refs/heads.*) -eq 1 ]; then echo "Branch name found"; else echo "This build only supports branch builds" && exit 1; fi
- apt-get update -y
- apt-get install -y maven
......@@ -58,7 +60,7 @@ phases:
- echo "Building primary service assemblies..."
- mvn install -N # required for this service to install the parent pom so that the integration tests will find it
- mvn -B test install -pl schema-core,provider/schema-aws -Ddeployment.environment=prod
- mvn -B test install -pl schema-core,provider/schema-aws -Ddeployment.environment=prod
# Suspended until further notice
# - echo "Copying assemblies to dist..."
......@@ -70,6 +72,10 @@ phases:
- chmod +x ./testing/schema-test-aws/build-aws/prepare-dist.sh
- ./testing/schema-test-aws/build-aws/prepare-dist.sh
#Copy R3 data def bootstrap scripts to dist
- chmod +x ./deployments/scripts/aws/prepare-dist.sh
- ./deployments/scripts/aws/prepare-dist.sh
- echo "Logging into Docker Hub..."
- docker login -u ${DOCKER_USERNAME} -p ${DOCKER_PASSWORD}
......
if [ -n $USE_SELF_SIGNED_SSL_CERT ];
then
export SSL_KEY_PASSWORD=$RANDOM$RANDOM$RANDOM;
export SSL_KEY_STORE_PASSWORD=$SSL_KEY_PASSWORD;
export SSL_KEY_STORE_DIR=/tmp/certs;
export SSL_KEY_STORE_NAME=osduonaws.p12;
export SSL_KEY_STORE_PATH=$SSL_KEY_STORE_DIR/$SSL_KEY_STORE_NAME;
export SSL_KEY_ALIAS=osduonaws;
./ssl.sh;
fi
java $JAVA_OPTS -jar /app.jar
\ No newline at end of file
# Copyright © 2021 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#!/usr/bin/env bash
#Future: Support for using Amazon Cert Manager
# if [ "$1" == "webserver" ] && [ -n $ACM_CERTIFICATE_ARN ];
# then
# aws acm export-certificate --certificate-arn $ACM_CERTIFICATE_ARN --passphrase $(echo -n 'aws123' | openssl base64 -e) | jq -r '"\(.PrivateKey)"' > ${SSL_KEY_PATH}.enc
# openssl rsa -in ${SSL_KEY_PATH}.enc -out $SSL_KEY_PATH -passin pass:aws123
# aws acm get-certificate --certificate-arn $ACM_CERTIFICATE_ARN | jq -r '"\(.CertificateChain)"' > $SSL_CERT_PATH
# aws acm get-certificate --certificate-arn $ACM_CERTIFICATE_ARN | jq -r '"\(.Certificate)"' >> $SSL_CERT_PATH
# fi
if [ -n $USE_SELF_SIGNED_SSL_CERT ];
then
mkdir -p $SSL_KEY_STORE_DIR
pushd $SSL_KEY_STORE_DIR
keytool -genkeypair -alias $SSL_KEY_ALIAS -keyalg RSA -keysize 2048 -storetype PKCS12 -keystore $SSL_KEY_STORE_NAME -validity 3650 -keypass $SSL_KEY_PASSWORD -storepass $SSL_KEY_PASSWORD -dname "CN=localhost, OU=AWS, O=Energy, L=Houston, ST=TX, C=US"
popd
fi
......@@ -67,7 +67,7 @@
<dependency>
<groupId>org.opengroup.osdu.core.aws</groupId>
<artifactId>os-core-lib-aws</artifactId>
<version>0.3.7</version>
<version>0.3.17</version>
</dependency>
<dependency>
......
# Copyright � 2020 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
server.servlet.context-path=/api/schema-service/v1
logging.level.org.springframework.web=${LOG_LEVEL:INFO}
server.port=${APPLICATION_PORT:8080}
aws.resource.prefix=${ENVIRONMENT}
aws.ssm=${SSM_ENABLED:True}
AUTHORIZE_API=${ENTITLEMENTS_BASE_URL}/api/entitlements/v1
AUTHORIZE_API_KEY=
LOG_PREFIX=schema
# s3 data bucket
aws.s3.endpoint=s3.${AWS_REGION}.amazonaws.com
aws.s3.bucket=${S3_BUCKET:""}
## AWS DynamoDB configuration
# needed for tenantfactory
aws.dynamodb.key=kind
aws.dynamodb.table.prefix=${aws.resource.prefix}-
aws.region=${AWS_REGION}
aws.dynamodb.endpoint=dynamodb.${AWS_REGION}.amazonaws.com
# if this is turned on then the service tries to connect to elastic search
management.health.elasticsearch.enabled=false
# Use this property to name your shared tenant name
shared.tenant.name=common
server.ssl.enabled=false
\ No newline at end of file
# Copyright 2020 Amazon Web Services
# Copyright 2020 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
......@@ -39,3 +39,10 @@ management.health.elasticsearch.enabled=false
# Use this property to name your shared tenant name
shared.tenant.name=common
server.ssl.enabled=${SSL_ENABLED:true}
server.ssl.key-store-type=PKCS12
server.ssl.key-store=${SSL_KEY_STORE_PATH:/certs/osduonaws.p12}
server.ssl.key-alias=${SSL_KEY_ALIAS:osduonaws}
server.ssl.key-password=${SSL_KEY_PASSWORD:}
server.ssl.key-store-password=${SSL_KEY_STORE_PASSWORD:}
\ No newline at end of file
......@@ -34,9 +34,9 @@ echo $INTEGRATION_TEST_OUTPUT_BIN_DIR
rm -rf "$INTEGRATION_TEST_OUTPUT_DIR"
mkdir -p "$INTEGRATION_TEST_OUTPUT_DIR" && mkdir -p "$INTEGRATION_TEST_OUTPUT_BIN_DIR"
echo "Building integration testing assemblies and gathering artifacts..."
mvn install dependency:copy-dependencies -DskipTests -f "$INTEGRATION_TEST_SOURCE_DIR_CORE"/pom.xml -DincludeGroupIds=org.opengroup.osdu -Dmdep.copyPom
mvn -ntp install dependency:copy-dependencies -DskipItTests=true -f "$INTEGRATION_TEST_SOURCE_DIR_CORE"/pom.xml -DincludeGroupIds=org.opengroup.osdu -Dmdep.copyPom
cp "$INTEGRATION_TEST_SOURCE_DIR_CORE"/target/dependency/* "${INTEGRATION_TEST_OUTPUT_BIN_DIR}"
(cd "${INTEGRATION_TEST_OUTPUT_BIN_DIR}" && ls *.jar | sed -e 's/\.jar$//' | xargs -I {} echo mvn install:install-file -Dfile={}.jar -DpomFile={}.pom >> install-deps.sh)
(cd "${INTEGRATION_TEST_OUTPUT_BIN_DIR}" && ls *.jar | sed -e 's/\.jar$//' | xargs -I {} echo mvn -ntp install:install-file -Dfile={}.jar -DpomFile={}.pom >> install-deps.sh)
chmod +x "${INTEGRATION_TEST_OUTPUT_BIN_DIR}"/install-deps.sh
#mvn clean -f "$INTEGRATION_TEST_SOURCE_DIR_CORE"/pom.xml
cp -R "$INTEGRATION_TEST_SOURCE_DIR_CORE"/* "${INTEGRATION_TEST_OUTPUT_DIR}"/
......
......@@ -85,7 +85,7 @@
<dependency>
<groupId>org.opengroup.osdu.core.aws</groupId>
<artifactId>os-core-lib-aws</artifactId>
<version>0.3.7</version>
<version>0.3.16</version>
</dependency>
<dependency>
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment