Merge branch 'awswellboreddms' into 'master'

aws wellbore domain service implementation

See merge request !58

.gitlab-ci.yml

@@ -22,6 +22,12 @@ variables:
   AZURE_DOCKER_SUBDIR: build/Dockerfile
   AZURE_TEST_SUBDIR: tests/integration
   AZURE_TEST_TYPE: python
+
+  AWS_SERVICE: wellbore-ddms
+  AWS_ENVIRONMENT: dev
+  AWS_BUILD_SUBDIR: provider/os-wellbore-ddms-aws/build-aws
+  AWS_TEST_SUBDIR: tests/aws-test/build-aws
+  AWS_INT_TEST_TYPE: python
  
 include:
   - project: "osdu/platform/ci-cd-pipelines"
@@ -43,6 +49,10 @@ include:
     file: 'cloud-providers/ibm-wellbore.yml'
 
   - local: "/devops/azure/azure-wellbore.yml"
+
+  - project: 'osdu/platform/ci-cd-pipelines'
+    file: 'cloud-providers/aws.yml'
+
 # --------------------------------------------------------------------------------
 
 containerize:

app/bulk_persistence/tenant_provider.py

@@ -39,6 +39,12 @@ async def resolve_tenant(data_partition_id: str) -> Tenant:
             bucket_name='logstore-osdu-ibm'
         )
 
+    if Config.cloud_provider.value == 'aws':
+        return Tenant(
+            data_partition_id=data_partition_id,
+            project_id='',
+            bucket_name=f'{data_partition_id}-logstore-osdu' #folder name
+        )
     return Tenant(
             data_partition_id=data_partition_id,
             project_id='undefined',

app/conf.py

@@ -97,10 +97,10 @@ class ConfigurationContainer:
 
     cloud_provider: EnvVar = EnvVar(
         key='CLOUD_PROVIDER',
-        description='Short name of the current cloud provider environment, must be "gcp" or "az" or "ibm',
+        description='Short name of the current cloud provider environment, must be "aws" or "gcp" or "az" or "ibm',
         default=None,
         is_mandatory=True,
-        allowed_values=['gcp', 'az', 'local', 'ibm'],
+        allowed_values=['aws', 'gcp', 'az', 'local', 'ibm'],
         factory=lambda x: x.lower()
     )
 
@@ -287,7 +287,19 @@ def cloud_provider_additional_environment(config: ConfigurationContainer):
                             default='logstore-ibm',
                             is_mandatory=True,
                             override=True)
-
+    if provider == 'aws':
+        config.add_from_env(attribute_name='aws_region',
+                            env_var_key='AWS_REGION',
+                            description='AWS data tenant ID',
+                            default='us-east-1',
+                            is_mandatory=True,
+                            override=True)
+        config.add_from_env(attribute_name='aws_env',
+                            env_var_key='ENVIRONMENT',
+                            description='AWS ResourcePrefix',
+                            default='osdu-',
+                            is_mandatory=True,
+                            override=True)
 # Global config instance
 Config = ConfigurationContainer.with_load_all(contextual_loader=cloud_provider_additional_environment)
 

app/injector/aws_injector.py

+# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from osdu.core.api.storage.blob_storage_base import BlobStorageBase
+from osdu_aws.storage.storage_aws import AwsStorage
+from .app_injector import AppInjector, AppInjectorModule
+from app.conf import Config
+
+class AwsInjector(AppInjectorModule):
+    def configure(self, app_injector: AppInjector):
+        app_injector.register(BlobStorageBase, AwsInjector.build_aws_storage)
+
+    @staticmethod
+    async def build_aws_storage() -> BlobStorageBase:
+        return AwsStorage(
+            session=None,
+            service_account_file=f'{Config.aws_region.value}$${Config.aws_env.value}'
+        )

app/injector/main_injector.py

@@ -17,6 +17,7 @@ from app.conf import *
 from .app_injector import AppInjector, AppInjectorModule, WithLifeTime
 
 from app.injector.az_injector import AzureInjector
+from app.injector.aws_injector import AwsInjector
 from app.injector.gcp_injector import GCPInjector
 from app.clients import StorageRecordServiceClient
 from app.clients.storage_service_blob_storage import StorageRecordServiceBlobStorage
@@ -61,6 +62,9 @@ class MainInjector(AppInjectorModule):
             logger.info('using ibm injector')
             IBMInjector().configure(app_injector)
 
+        if Config.cloud_provider.value == 'aws':
+            logger.info('using aws injector')
+            AwsInjector().configure(app_injector)
         # run overriders
         self.overriders(app_injector)
 

provider/os-wellbore-ddms-aws/build-aws/Dockerfile

+# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM python:3.7-slim-buster
+
+COPY requirements.txt requirements_dev.txt ./
+
+RUN pip install -r requirements.txt
+RUN pip install -r requirements_dev.txt
+
+COPY ./app /app
+COPY provider/os-wellbore-ddms-aws/build-aws/entrypoint.sh .
+COPY provider/os-wellbore-ddms-aws/build-aws/ssl.sh .
+ENV PYTHONPATH=./
+
+#Default to using self signed generated TLS cert
+ENV USE_SELF_SIGNED_SSL_CERT "true"
+ENV SSL_CERT_PATH "./aws/certs/cert.crt" 
+ENV SSL_KEY_PATH "./aws/certs/cert.key"
+ENV SSL_ENABLED "true"
+
+
+EXPOSE ${APPLICATION_PORT}
+WORKDIR ./
+
+ENTRYPOINT ["/bin/sh", "-c", "/entrypoint.sh"]
\ No newline at end of file

provider/os-wellbore-ddms-aws/build-aws/build-info.py

+# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import boto3
+import json
+import os
+import argparse
+
+# Create the build-info.json
+parser = argparse.ArgumentParser(description="")
+
+# env - CODEBUILD_SOURCE_VERSION
+parser.add_argument("--branch", type=str, help="")
+
+# env - CODEBUILD_RESOLVED_SOURCE_VERSION
+parser.add_argument("--commit", type=str,  help="")
+
+# env - CODEBUILD_BUILD_ID
+parser.add_argument("--buildid", type=str,  help="")
+
+# env - CODEBUILD_BUILD_NUMBER
+parser.add_argument("--buildnumber", type=str,  help="")
+
+# Get from directory name
+parser.add_argument("--reponame", type=str,  help="")
+
+# env OUTPUT_DIR
+parser.add_argument("--outdir", type=str,  help="")
+
+# full ecr image and tag, and any other artifacts
+parser.add_argument("--artifact", type=str, action="append", help="")
+
+
+
+args = parser.parse_args()
+
+branch = args.branch
+commitId = args.commit
+buildId = args.buildid
+buildNumber = args.buildnumber
+repoName = args.reponame
+outputDir = args.outdir
+artifacts = args.artifact
+
+buildInfoFilePath = os.path.join(".", outputDir, "build-info.json")
+
+print(buildInfoFilePath)
+
+commitArgs = {
+    "repositoryName": repoName,
+    "commitId": commitId
+}
+
+commitDetail = {
+    "commit": ""
+}
+
+# get the commit detail
+try:
+    codecommit = boto3.client("codecommit")
+    commitDetail = codecommit.get_commit(**commitArgs)
+except Exception as e:
+    print("Getting commit information from codecommit failed")
+
+buildInfo = {
+    "branch": branch,
+    "build-id": buildId,
+    "build-number": buildNumber,
+    "repo": repoName,
+    "artifacts": artifacts,
+    "commit": commitDetail["commit"]
+}
+print(json.dumps(buildInfo, sort_keys=True, indent=4))
+
+# write the build.json file to dist
+f = open(buildInfoFilePath, "w")
+f.write(json.dumps(buildInfo, sort_keys=True, indent=4))
+f.close()

provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml

+# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: 0.2
+
+env:
+  secrets-manager:
+    DOCKER_USERNAME: /osdu/devops/docker_credentials:username
+    DOCKER_PASSWORD: /osdu/devops/docker_credentials:password
+
+phases:
+  install:
+    commands:
+      - apt-get update -y
+  
+  pre_build:
+    commands:
+      - echo "Logging in to Amazon ECR..."
+      - $(aws ecr get-login --no-include-email --region $AWS_REGION) # authenticate with ECR via the AWS CLI
+  build:
+    commands:
+      - export OUTPUT_DIR="dist"
+      - mkdir ${OUTPUT_DIR}
+      - export REPO_NAME=${PWD##*/}
+      - export BRANCH_NAME=`echo ${CODEBUILD_SOURCE_VERSION} | awk '{gsub("refs/heads/","");gsub("\\.","-");gsub("[[:space:]]","-")}1' | sed 's/\//-/g' | awk '{print tolower($0)}'`
+      - export ECR_TAG=`echo build.${BRANCH_NAME}.${CODEBUILD_BUILD_NUMBER}.${CODEBUILD_RESOLVED_SOURCE_VERSION} | cut -c 1-120`
+      - export ECR_IMAGE=${ECR_REGISTRY}:${ECR_TAG}
+      - export ECR_IMAGE_BRANCH_LATEST=${ECR_REGISTRY}:${BRANCH_NAME}
+
+      - echo "Placeholder" >> ${OUTPUT_DIR}/build-info.json # touched so that the output directory has some content incase the build fails so that testing reports are uploaded
+
+      - echo "Building integration testing assemblies and gathering artifacts..."
+      - chmod +x ./tests/aws-test/build-aws/prepare-dist.sh
+      - ./tests/aws-test/build-aws/prepare-dist.sh
+
+      - pip install -r ./requirements.txt
+      - pip install -r ./requirements_dev.txt
+
+      - echo "Logging into Docker Hub..."
+      - docker login -u ${DOCKER_USERNAME} -p ${DOCKER_PASSWORD}
+
+      - echo "Building docker image..."
+      - docker build -f provider/os-wellbore-ddms-aws/build-aws/Dockerfile -t ${ECR_IMAGE} .
+      - docker tag  ${ECR_IMAGE} ${ECR_IMAGE_BRANCH_LATEST}
+      - echo "Pushing docker image..."
+      - docker push ${ECR_IMAGE}
+      - docker push ${ECR_IMAGE_BRANCH_LATEST}
+     
+      - python -m pytest --junit-xml=unit_tests_report.xml --cov=app --cov-report=html --cov-report=xml ./tests/unit
+
+      - echo "Generate build-info.json"
+      - |
+        python provider/os-wellbore-ddms-aws/build-aws/build-info.py --branch ${CODEBUILD_SOURCE_VERSION} --commit ${CODEBUILD_RESOLVED_SOURCE_VERSION} \
+          --buildid ${CODEBUILD_BUILD_ID} --buildnumber ${CODEBUILD_BUILD_NUMBER} --reponame ${REPO_NAME} --outdir ${OUTPUT_DIR} \
+          --artifact ${ECR_IMAGE}
+artifacts:
+  files:
+    - "**/*"
+  base-directory: "dist"
+  name: ${REPO_NAME}_${BRANCH_NAME}_$(date +%F)_${CODEBUILD_BUILD_NUMBER}.zip

provider/os-wellbore-ddms-aws/build-aws/entrypoint.sh

+./ssl.sh;
+
+if [ ${APPLICATION_PORT} -ne 443 ]
+then
+    uvicorn app.wdms_app:base_app --host 0.0.0.0 --port ${APPLICATION_PORT}
+else
+    uvicorn app.wdms_app:base_app --host 0.0.0.0 --port ${APPLICATION_PORT} --ssl-certfile ${SSL_CERT_PATH} --ssl-keyfile ${SSL_KEY_PATH}
+fi
\ No newline at end of file

provider/os-wellbore-ddms-aws/build-aws/ssl.sh

+#!/usr/bin/env bash
+
+#Future: Support for using Amazon Cert Manager
+# if [ "$1" == "webserver" ] && [ -n $ACM_CERTIFICATE_ARN ];
+# then
+
+#   if [ -z $SSL_CERT_PATH ] || [ -z $SSL_KEY_PATH ];
+#   then
+#     echo "SSL_CERT_PATH and SSL_KEY_PATH must be set as environment variables when using ACM_CERTIFICATE_ARN"
+#     exit 1
+#   fi
+
+#   aws acm export-certificate --certificate-arn $ACM_CERTIFICATE_ARN --passphrase $(echo -n 'aws123' | openssl base64 -e) | jq -r '"\(.PrivateKey)"' > ${SSL_KEY_PATH}.enc
+#   openssl rsa -in ${SSL_KEY_PATH}.enc -out $SSL_KEY_PATH -passin pass:aws123
+#   aws acm get-certificate --certificate-arn $ACM_CERTIFICATE_ARN | jq -r '"\(.CertificateChain)"' > $SSL_CERT_PATH
+#   aws acm get-certificate --certificate-arn $ACM_CERTIFICATE_ARN | jq -r '"\(.Certificate)"' >> $SSL_CERT_PATH
+
+# fi
+
+if [ "$APPLICATION_PORT" -eq 443 ]
+then
+    if [ -z $SSL_CERT_PATH ] || [ -z $SSL_KEY_PATH ]
+    then
+      echo "SSL_CERT_PATH and SSL_KEY_PATH must be set as environment variables when using USE_SELF_SIGNED_SSL_CERT"
+      exit 1
+    fi
+
+    mkdir -p $(dirname "$SSL_CERT_PATH")
+    mkdir -p $(dirname "$SSL_KEY_PATH")
+
+    hostname="localhost"
+    subject="/CN=${hostname}"
+
+    #new versions of openssl support this:
+    # openssl req \
+    #     -newkey rsa:2048  -nodes  -keyout ${SSL_KEY_PATH} \
+    #     -new -x509 -sha256 -days 365 -out ${SSL_CERT_PATH} \
+    #     -subj "${subject}" \
+    #     -addext "subjectAltName = DNS:${hostname}" \
+    #     -addext "extendedKeyUsage = serverAuth"
+
+    #old version of openssl use this:
+    confdir=$(openssl version -d | awk -F'"' '{print $2}')
+    
+    openssl req \
+        -newkey rsa:2048  -nodes  -keyout ${SSL_KEY_PATH} \
+        -new -x509 -sha256 -days 365 -out ${SSL_CERT_PATH} \
+        -subj "${subject}" \
+        -extensions SAN -reqexts SAN \
+        -config <(cat ${confdir}/openssl.cnf;printf "[SAN]\nsubjectAltName=DNS:${hostname}\nextendedKeyUsage=serverAuth")
+fi

requirements.txt

@@ -28,5 +28,6 @@ osdu-data-ecosystem-search>=0.3.2, <0.4
 osdu-core-lib-python-ibm>=0.0.1, <0.1
 osdu-core-lib-python-gcp>=0.3.0, <0.4
 osdu-core-lib-python-azure~=0.2.0
+osdu-core-lib-python-aws>=0.0.1, <0.1
 osdu-core-lib-python>=0.4.0, <0.5
 

tests/aws-test/build-aws/aws_jwt_client.py

+import os;
+import boto3;
+import jwt;
+
+def get_id_token():
+    client = boto3.client('cognito-idp', region_name=os.environ["AWS_REGION"])
+    userAuth = client.initiate_auth(
+        ClientId= os.environ['AWS_COGNITO_CLIENT_ID'],
+        AuthFlow= os.environ['AWS_COGNITO_AUTH_FLOW'],
+        AuthParameters= {
+            "USERNAME": os.environ['AWS_COGNITO_AUTH_PARAMS_USER'],
+            "PASSWORD": os.environ['AWS_COGNITO_AUTH_PARAMS_PASSWORD']
+        })
+
+    print(userAuth['AuthenticationResult']['AccessToken'])
+
+
+def get_invalid_token():
+    #generate a dummy jwt
+    return jwt.encode({'some': 'payload'}, 'secret', algorithm='HS256').decode("utf-8")
+
+if __name__ == '__main__':
+    get_id_token()

tests/aws-test/build-aws/prepare-dist.sh

+# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script prepares the dist directory for the integration tests.
+# Must be run from the root of the repostiory
+
+set -e
+
+OUTPUT_DIR="${OUTPUT_DIR:-dist}"
+
+INTEGRATION_TEST_OUTPUT_DIR=${INTEGRATION_TEST_OUTPUT_DIR:-$OUTPUT_DIR}/testing
+
+rm -rf "$INTEGRATION_TEST_OUTPUT_DIR"
+mkdir -p "$INTEGRATION_TEST_OUTPUT_DIR"
+
+if [ ! -e requirements_dev.txt ]; then
+    echo "File requirements_dev.txt does not exist!"
+else
+    cp requirements_dev.txt tests/aws-test/build-aws/requirements.txt
+fi
+cp  -r tests/aws-test "${INTEGRATION_TEST_OUTPUT_DIR}"
+cp  -r tests/integration "${INTEGRATION_TEST_OUTPUT_DIR}"
+cp  -r schema/indexation "${INTEGRATION_TEST_OUTPUT_DIR}"
\ No newline at end of file

tests/aws-test/build-aws/run-integration-tests.sh

+#!/usr/bin/env bash
+# from tests/aws-test/build-aws/ goes up to the tests dir.
+cd ../../../
+
+# Install venv for python3
+which apt-get && sudo apt-get install -y python3 python3-pip python3-venv || echo "Not Ubuntu, skipping"
+which yum && sudo yum install -y python3 python3-pip python3-venv || echo "Not RHEL, skipping"
+
+python3 -m venv env
+source env/bin/activate
+python3 -m pip install --upgrade pip
+python3 -m pip install -r requirements.txt
+python3 -m pip install -r requirements_dev.txt
+python3 -m pip install wheel pytest pytest-cov
+
+svc_url=$WELLBORE_DDMS_URL
+tenant='opendes'
+acl_domain='testing.com' 
+legal_tag='opendes-sdmstestlegaltag'
+svctoken=$(python tests/aws-test/build-aws/aws_jwt_client.py)
+
+cd tests/integration
+
+python3 gen_postman_env.py --token $svctoken --base_url $svc_url --cloud_provider "aws" --acl_domain $acl_domain --legal_tag $legal_tag --data_partition $tenant
+
+pytest ./functional --environment="./generated/postman_environment.json" --filter-tag=basic
\ No newline at end of file

tests/aws-test/build-aws/run-tests.sh

+# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script executes the test and copies reports to the provided output directory
+# To call this script from the service working directory
+# ./dist/testing/integration/build-aws/run-tests.sh "./reports/"
+echo '********* Running Wellbore DDMS integration tests  *********'
+
+echo $(pwd)
+
+AWS_COGNITO_PWD=$ADMIN_PASSWORD
+AWS_COGNITO_USER=$ADMIN_USER
+client_id=$AWS_COGNITO_CLIENT_ID
+svc_url=$WELLBORE_DDMS_URL
+tenant='opendes'
+acl_domain='testing.com' 
+legal_tag='opendes-sdmstestlegaltag'
+
+
+#### RUN INTEGRATION TEST #########################################################################
+
+echo 'Generating token...'
+token=$(aws cognito-idp initiate-auth --auth-flow USER_PASSWORD_AUTH --client-id $client_id --auth-parameters USERNAME=$AWS_COGNITO_USER,PASSWORD=$AWS_COGNITO_PWD --output=text --query AuthenticationResult.{AccessToken:AccessToken})
+
+#### RUN INTEGRATION TEST #########################################################################
+
+
+cd deployment/osdu-core/os-wellbore-domain-services/testing
+python3 -m venv env
+source env/bin/activate
+pip install -r ./aws-test/build-aws/requirements.txt
+rm -rf test-reports/
+mkdir test-reports
+
+cd integration
+
+acl_domain='testing.com' 
+legal_tag='opendes-sdmstestlegaltag'
+
+python gen_postman_env.py --token $token --base_url $svc_url --cloud_provider "aws" --acl_domain $acl_domain --legal_tag $legal_tag --data_partition $tenant
+
+pytest ./functional --environment="./generated/postman_environment.json" --filter-tag=basic
+
+TEST_EXIT_CODE=$?
+deactivate
+exit $TEST_EXIT_CODE
\ No newline at end of file

tests/unit/routers/about_test.py

@@ -56,7 +56,7 @@ def test_about_contains_build_n_version(client):
     assert response_json['version']
 
 
-@pytest.mark.parametrize("cloud_provider", ['Azure', 'gcp', 'unknown', None])
+@pytest.mark.parametrize("cloud_provider", ['Azure', 'gcp', 'unknown', 'aws', None])
 def test_about_with_cloud_provider(client, cloud_provider):
 
     Config.cloud_provider.value = cloud_provider