Commit 750cf7a8 authored by Yunhua Koglin's avatar Yunhua Koglin
Browse files

aws wellbore domain service implementation

commit 21e78ab4 
Author: Yunhua Koglin <kogliny@amazon.com> 
Date: Wed Mar 17 2021 15:17:31 GMT-0500 (Central Daylight Time) 

    remove account info
parent be0dcb73
......@@ -39,6 +39,12 @@ async def resolve_tenant(data_partition_id: str) -> Tenant:
bucket_name='logstore-osdu-ibm'
)
if Config.cloud_provider.value == 'aws':
return Tenant(
data_partition_id=data_partition_id,
project_id='',
bucket_name=f'{data_partition_id}-logstore-osdu' #folder name
)
return Tenant(
data_partition_id=data_partition_id,
project_id='undefined',
......
......@@ -97,10 +97,10 @@ class ConfigurationContainer:
cloud_provider: EnvVar = EnvVar(
key='CLOUD_PROVIDER',
description='Short name of the current cloud provider environment, must be "gcp" or "az" or "ibm',
description='Short name of the current cloud provider environment, must be "aws" or "gcp" or "az" or "ibm',
default=None,
is_mandatory=True,
allowed_values=['gcp', 'az', 'local', 'ibm'],
allowed_values=['aws', 'gcp', 'az', 'local', 'ibm'],
factory=lambda x: x.lower()
)
......@@ -287,7 +287,19 @@ def cloud_provider_additional_environment(config: ConfigurationContainer):
default='logstore-ibm',
is_mandatory=True,
override=True)
if provider == 'aws':
config.add_from_env(attribute_name='aws_region',
env_var_key='AWS_REGION',
description='AWS data tenant ID',
default='us-east-1',
is_mandatory=True,
override=True)
config.add_from_env(attribute_name='aws_env',
env_var_key='ENVIRONMENT',
description='AWS ResourcePrefix',
default='osdu-',
is_mandatory=True,
override=True)
# Global config instance
Config = ConfigurationContainer.with_load_all(contextual_loader=cloud_provider_additional_environment)
......
# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http:#www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from osdu.core.api.storage.blob_storage_base import BlobStorageBase
from osdu_aws.storage.storage_aws import AwsStorage
from .app_injector import AppInjector, AppInjectorModule
from app.conf import Config
class AwsInjector(AppInjectorModule):
def configure(self, app_injector: AppInjector):
app_injector.register(BlobStorageBase, AwsInjector.build_aws_storage)
@staticmethod
async def build_aws_storage() -> BlobStorageBase:
return AwsStorage(
session=None,
service_account_file=f'{Config.aws_region.value}$${Config.aws_env.value}'
)
......@@ -17,6 +17,7 @@ from app.conf import *
from .app_injector import AppInjector, AppInjectorModule, WithLifeTime
from app.injector.az_injector import AzureInjector
from app.injector.aws_injector import AwsInjector
from app.injector.gcp_injector import GCPInjector
from app.clients import StorageRecordServiceClient
from app.clients.storage_service_blob_storage import StorageRecordServiceBlobStorage
......@@ -61,6 +62,9 @@ class MainInjector(AppInjectorModule):
logger.info('using ibm injector')
IBMInjector().configure(app_injector)
if Config.cloud_provider.value == 'aws':
logger.info('using aws injector')
AwsInjector().configure(app_injector)
# run overriders
self.overriders(app_injector)
......
# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http:#www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import boto3
import json
import os
import argparse
# Create the build-info.json
parser = argparse.ArgumentParser(description="")
# env - CODEBUILD_SOURCE_VERSION
parser.add_argument("--branch", type=str, help="")
# env - CODEBUILD_RESOLVED_SOURCE_VERSION
parser.add_argument("--commit", type=str, help="")
# env - CODEBUILD_BUILD_ID
parser.add_argument("--buildid", type=str, help="")
# env - CODEBUILD_BUILD_NUMBER
parser.add_argument("--buildnumber", type=str, help="")
# Get from directory name
parser.add_argument("--reponame", type=str, help="")
# env OUTPUT_DIR
parser.add_argument("--outdir", type=str, help="")
# full ecr image and tag, and any other artifacts
parser.add_argument("--artifact", type=str, action="append", help="")
args = parser.parse_args()
branch = args.branch
commitId = args.commit
buildId = args.buildid
buildNumber = args.buildnumber
repoName = args.reponame
outputDir = args.outdir
artifacts = args.artifact
buildInfoFilePath = os.path.join(".", outputDir, "build-info.json")
print(buildInfoFilePath)
commitArgs = {
"repositoryName": repoName,
"commitId": commitId
}
commitDetail = {
"commit": ""
}
# get the commit detail
try:
codecommit = boto3.client("codecommit")
commitDetail = codecommit.get_commit(**commitArgs)
except Exception as e:
print("Getting commit information from codecommit failed")
buildInfo = {
"branch": branch,
"build-id": buildId,
"build-number": buildNumber,
"repo": repoName,
"artifacts": artifacts,
"commit": commitDetail["commit"]
}
print(json.dumps(buildInfo, sort_keys=True, indent=4))
# write the build.json file to dist
f = open(buildInfoFilePath, "w")
f.write(json.dumps(buildInfo, sort_keys=True, indent=4))
f.close()
# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http:#www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
FROM python:3.7-slim-buster
ARG aws_account_url="dev"
COPY provider/os-wellbore-ddms-aws/build-aws/requirements.txt .
COPY requirements_dev.txt .
RUN pip install awscli
ARG token="token"
RUN pip config set global.index-url https://aws:${token}@${aws_account_url}
RUN pip install -r requirements.txt
RUN pip install -r requirements_dev.txt
COPY ./app /app
COPY provider/os-wellbore-ddms-aws/build-aws/entrypoint.sh .
COPY provider/os-wellbore-ddms-aws/build-aws/ssl.sh .
ENV PYTHONPATH=./
#Default to using self signed generated TLS cert
ENV USE_SELF_SIGNED_SSL_CERT "true"
ENV SSL_CERT_PATH "./aws/certs/cert.crt"
ENV SSL_KEY_PATH "./aws/certs/cert.key"
ENV SSL_ENABLED "true"
EXPOSE ${APPLICATION_PORT}
WORKDIR ./
ENTRYPOINT ["/bin/sh", "-c", "/entrypoint.sh"]
\ No newline at end of file
# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http:#www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
version: 0.2
env:
secrets-manager:
DOCKER_USERNAME: /osdu/devops/docker_credentials:username
DOCKER_PASSWORD: /osdu/devops/docker_credentials:password
phases:
install:
commands:
- apt-get update -y
- echo "Installing requirements_dev.txt"
- pip install -r ./requirements_dev.txt
pre_build:
commands:
- echo "Logging in to Amazon ECR..."
- $(aws ecr get-login --no-include-email --region $AWS_REGION) # authenticate with ECR via the AWS CLI
build:
commands:
- export OUTPUT_DIR="dist"
- mkdir ${OUTPUT_DIR}
- export REPO_NAME=${PWD##*/}
- export BRANCH_NAME=`echo ${CODEBUILD_SOURCE_VERSION} | awk '{gsub("refs/heads/","");gsub("\\.","-");gsub("[[:space:]]","-")}1' | sed 's/\//-/g' | awk '{print tolower($0)}'`
- export ECR_TAG=`echo build.${BRANCH_NAME}.${CODEBUILD_BUILD_NUMBER}.${CODEBUILD_RESOLVED_SOURCE_VERSION} | cut -c 1-120`
- export ECR_IMAGE=${ECR_REGISTRY}:${ECR_TAG}
- export ECR_IMAGE_BRANCH_LATEST=${ECR_REGISTRY}:${BRANCH_NAME}
- echo "Placeholder" >> ${OUTPUT_DIR}/build-info.json # touched so that the output directory has some content incase the build fails so that testing reports are uploaded
- echo "Building integration testing assemblies and gathering artifacts..."
- chmod +x ./tests/aws-test/build-aws/prepare-dist.sh
- ./tests/aws-test/build-aws/prepare-dist.sh
- echo "Installing requirements.txt"
- export AWS_ACCOUNT_ID=`aws sts get-caller-identity | grep Account | cut -d':' -f 2 | cut -d'"' -f 2`
- aws codeartifact login --tool pip --domain osdu-dev --domain-owner $AWS_ACCOUNT_ID --repository osdu-python --region $AWS_REGION
- pip install -r ./provider/os-wellbore-ddms-aws/build-aws/requirements.txt
- echo "Logging into Docker Hub..."
- docker login -u ${DOCKER_USERNAME} -p ${DOCKER_PASSWORD}
- echo "Building docker image..."
# Set PIP_EXTRA_URL
- export PIP_EXTRA_URL=https://community.opengroup.org/groups/osdu/platform/domain-data-mgmt-services/wellbore/-/packages
- export CODEARTIFACT_TOKEN=`aws codeartifact get-authorization-token --domain osdu-dev --domain-owner $AWS_ACCOUNT_ID --query authorizationToken --output text`
- docker build -f provider/os-wellbore-ddms-aws/build-aws/builder.Dockerfile --build-arg aws_account_url=${AWS_OSDU_DEV_PYPI_URL} --build-arg token=${CODEARTIFACT_TOKEN} --build-arg PIP_EXTRA_URL=$PIP_EXTRA_URL --build-arg PIP_WHEEL_DIR=python-packages -t ${ECR_IMAGE} .
- docker tag ${ECR_IMAGE} ${ECR_IMAGE_BRANCH_LATEST}
- echo "Pushing docker image..."
- docker push ${ECR_IMAGE}
- docker push ${ECR_IMAGE_BRANCH_LATEST}
- python -m pytest --junit-xml=unit_tests_report.xml --cov=app --cov-report=html --cov-report=xml ./tests/unit
- echo "Generate build-info.json"
- |
python provider/os-wellbore-ddms-aws/build-aws/build-info.py --branch ${CODEBUILD_SOURCE_VERSION} --commit ${CODEBUILD_RESOLVED_SOURCE_VERSION} \
--buildid ${CODEBUILD_BUILD_ID} --buildnumber ${CODEBUILD_BUILD_NUMBER} --reponame ${REPO_NAME} --outdir ${OUTPUT_DIR} \
--artifact ${ECR_IMAGE}
artifacts:
files:
- "**/*"
base-directory: "dist"
name: ${REPO_NAME}_${BRANCH_NAME}_$(date +%F)_${CODEBUILD_BUILD_NUMBER}.zip
./ssl.sh;
if [ ${APPLICATION_PORT} -ne 443 ]
then
uvicorn app.wdms_app:base_app --host 0.0.0.0 --port ${APPLICATION_PORT}
else
uvicorn app.wdms_app:base_app --host 0.0.0.0 --port ${APPLICATION_PORT} --ssl-certfile ${SSL_CERT_PATH} --ssl-keyfile ${SSL_KEY_PATH}
fi
\ No newline at end of file
fastapi>=0.59.0
cachetools==3.1.1
aiohttp==3.6.2
cryptography>=2.7
pyarrow>=0.15.0
pandas==1.1.2
uvicorn
Click
structlog
python-rapidjson
python-multipart
jsonpath-ng # maintenance of 'jsonpath-rw' lib it's bit abandoned
opencensus
opencensus-ext-stackdriver
opencensus-ext-azure
opencensus-ext-ocagent
opencensus-ext-logging
--extra-index-url \
https://community.opengroup.org/api/v4/projects/465/packages/pypi/simple/
osdu-log-recognition-lib>=0.0.9
osdu-data-ecosystem-storage~=1.1.0
osdu-data-ecosystem-search>=0.3.2, <0.4
osdu-core-lib-python-ibm>=0.0.1, <0.1
osdu-core-lib-python-gcp>=0.3.0, <0.4
osdu-core-lib-python-azure~=0.2.0
osdu-core-lib-python>=0.4.0, <0.5
osdu-core-lib-python-aws
#!/usr/bin/env bash
#Future: Support for using Amazon Cert Manager
# if [ "$1" == "webserver" ] && [ -n $ACM_CERTIFICATE_ARN ];
# then
# if [ -z $SSL_CERT_PATH ] || [ -z $SSL_KEY_PATH ];
# then
# echo "SSL_CERT_PATH and SSL_KEY_PATH must be set as environment variables when using ACM_CERTIFICATE_ARN"
# exit 1
# fi
# aws acm export-certificate --certificate-arn $ACM_CERTIFICATE_ARN --passphrase $(echo -n 'aws123' | openssl base64 -e) | jq -r '"\(.PrivateKey)"' > ${SSL_KEY_PATH}.enc
# openssl rsa -in ${SSL_KEY_PATH}.enc -out $SSL_KEY_PATH -passin pass:aws123
# aws acm get-certificate --certificate-arn $ACM_CERTIFICATE_ARN | jq -r '"\(.CertificateChain)"' > $SSL_CERT_PATH
# aws acm get-certificate --certificate-arn $ACM_CERTIFICATE_ARN | jq -r '"\(.Certificate)"' >> $SSL_CERT_PATH
# fi
if [ "$APPLICATION_PORT" -eq 443 ]
then
if [ -z $SSL_CERT_PATH ] || [ -z $SSL_KEY_PATH ]
then
echo "SSL_CERT_PATH and SSL_KEY_PATH must be set as environment variables when using USE_SELF_SIGNED_SSL_CERT"
exit 1
fi
mkdir -p $(dirname "$SSL_CERT_PATH")
mkdir -p $(dirname "$SSL_KEY_PATH")
hostname="localhost"
subject="/CN=${hostname}"
#new versions of openssl support this:
# openssl req \
# -newkey rsa:2048 -nodes -keyout ${SSL_KEY_PATH} \
# -new -x509 -sha256 -days 365 -out ${SSL_CERT_PATH} \
# -subj "${subject}" \
# -addext "subjectAltName = DNS:${hostname}" \
# -addext "extendedKeyUsage = serverAuth"
#old version of openssl use this:
confdir=$(openssl version -d | awk -F'"' '{print $2}')
openssl req \
-newkey rsa:2048 -nodes -keyout ${SSL_KEY_PATH} \
-new -x509 -sha256 -days 365 -out ${SSL_CERT_PATH} \
-subj "${subject}" \
-extensions SAN -reqexts SAN \
-config <(cat ${confdir}/openssl.cnf;printf "[SAN]\nsubjectAltName=DNS:${hostname}\nextendedKeyUsage=serverAuth")
fi
# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http:#www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This script prepares the dist directory for the integration tests.
# Must be run from the root of the repostiory
set -e
OUTPUT_DIR="${OUTPUT_DIR:-dist}"
INTEGRATION_TEST_OUTPUT_DIR=${INTEGRATION_TEST_OUTPUT_DIR:-$OUTPUT_DIR}/testing
rm -rf "$INTEGRATION_TEST_OUTPUT_DIR"
mkdir -p "$INTEGRATION_TEST_OUTPUT_DIR"
cp -r tests/aws-test "${INTEGRATION_TEST_OUTPUT_DIR}"
cp -r tests/integration "${INTEGRATION_TEST_OUTPUT_DIR}"
cp -r schema/indexation "${INTEGRATION_TEST_OUTPUT_DIR}"
\ No newline at end of file
pytest
pytest-asyncio
pytest-cov
pytest-mock
pytest-httpx
httpx
# Note since 3.8 includes Mock 4.0+.
mock>=4.0
requests # used by starlette.TestClient for testing
pyjwt
cryptography
# the following are used in functional integration tests
pytest-dependency
munch
jsonschema
# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http:#www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This script executes the test and copies reports to the provided output directory
# To call this script from the service working directory
# ./dist/testing/integration/build-aws/run-tests.sh "./reports/"
echo '********* Running Wellbore DDMS integration tests *********'
echo $(pwd)
AWS_COGNITO_PWD=$ADMIN_PASSWORD
AWS_COGNITO_USER=$ADMIN_USER
client_id=$AWS_COGNITO_CLIENT_ID
svc_url=$WELLBORE_DDMS_URL
tenant='opendes'
acl_domain='testing.com'
legal_tag='opendes-sdmstestlegaltag'
#### RUN INTEGRATION TEST #########################################################################
echo 'Generating token...'
token=$(aws cognito-idp initiate-auth --auth-flow USER_PASSWORD_AUTH --client-id $client_id --auth-parameters USERNAME=$AWS_COGNITO_USER,PASSWORD=$AWS_COGNITO_PWD --output=text --query AuthenticationResult.{AccessToken:AccessToken})
#### RUN INTEGRATION TEST #########################################################################
cd deployment/osdu-core/os-wellbore-domain-services/testing
pip install -r ./aws-test/build-aws/requirements.txt
rm -rf test-reports/
mkdir test-reports
cd indexation
schemaFiles=$(ls *.json)
for schemaFile in $schemaFiles
do
echo "loading $schemaFile: "
schema=$(sed "s/DATA_PARTITION_TAG/${tenant}/" ${schemaFile})
echo $schema | head -c 100
echo "..."
curl \
--location \
--request POST "${AWS_BASE_URL}/api/storage/v2/schemas" \
--header "Content-Type: application/json" \
--header "data-partition-id: ${tenant}" \
--header "Authorization: Bearer ${token}" \
--data-raw "${schema}"
echo ""
echo "---"
done
cd ..
cd integration
acl_domain='testing.com'
legal_tag='opendes-sdmstestlegaltag'
python gen_postman_env.py --token $token --base_url $svc_url --cloud_provider "aws" --acl_domain $acl_domain --legal_tag $legal_tag --data_partition $tenant
pytest ./functional --environment="./generated/postman_environment.json" --filter-tag=basic
TEST_EXIT_CODE=$?
exit $TEST_EXIT_CODE
\ No newline at end of file
......@@ -56,7 +56,7 @@ def test_about_contains_build_n_version(client):
assert response_json['version']
@pytest.mark.parametrize("cloud_provider", ['Azure', 'gcp', 'unknown', None])
@pytest.mark.parametrize("cloud_provider", ['Azure', 'gcp', 'unknown', 'aws', None])
def test_about_with_cloud_provider(client, cloud_provider):
Config.cloud_provider.value = cloud_provider
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment