Commit 93875ddc authored by Yunhua Koglin's avatar Yunhua Koglin
Browse files

Merge branch 'aws-imp' into 'master'

Aws implementation for wellbore DDMS lib

See merge request !1
parents 5ee6d141 faf0f0c0
Pipeline #52677 passed with stage
in 33 seconds
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# ide
.idea/
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
.hypothesis/
.pytest_cache/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# Jupyter Notebook
.ipynb_checkpoints
# pyenv
.python-version
# celery beat schedule file
celerybeat-schedule
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
.venv*
env/
venv/
venv*/
ENV/
env.bak/
venv.bak/
.virtualenv/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
\ No newline at end of file
default:
image: python:3.7-slim-buster
include:
- project: "osdu/platform/domain-data-mgmt-services/wellbore/lib/lib-registry"
file: "registry-setup.yml"
stages:
- test
- deploy
build:
stage: test
script:
- echo ---- ---- ---- BUILD IMAGE ---- ---- ----
- pip3 install -r requirements.txt
- pip3 install -r requirements_opengroup.txt
- pip3 install -r requirements_dev.txt
- echo ---- ---- ---- UNIT TESTS ---- ---- ----
- pytest tests --junitxml=report.xml
artifacts:
when: always
reports:
junit: report.xml
# This job only runs on master, and it creates the lib and push it to the feed
deploylib:
stage: deploy
script:
- echo ---- ---- ---- BUILD IMAGE ---- ---- ----
- pip3 install -r requirements.txt
- pip3 install -r requirements_opengroup.txt
- pip3 install twine
- python3 setup.py sdist bdist_wheel
- TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=${CI_REGISTRY_USER} python -m twine upload --repository-url ${CI_API_V4_URL}/projects/${WDMS_LIB_REGISTRY}/packages/pypi dist/*
rules:
- if: $CI_COMMIT_BRANCH == 'master'
# Pushes merge request builds to the feed
deploydev:
stage: deploy
script:
- echo ---- ---- ---- SET DEVELOPMENT VERSION ---- ---- ----
- PACKAGE_VERSION_FILE='osdu_aws/__init__.py'
- PACKAGE_NAME=$(python setup.py --name)
- PACKAGE_LIB_VERSION=$(python setup.py --version)
- PACKAGE_DEV_VERSION=${PACKAGE_LIB_VERSION}.dev${CI_JOB_ID}
- sed -i "s/${PACKAGE_LIB_VERSION}/${PACKAGE_DEV_VERSION}/g" ${PACKAGE_VERSION_FILE}
- echo ---- ---- ---- BUILD IMAGE ---- ---- ----
- pip3 install -r requirements.txt
- pip3 install -r requirements_opengroup.txt
- pip3 install twine
- python3 setup.py sdist bdist_wheel
- echo ---- ---- ---- PUSHING DEVELOPMENT PACKAGE ${PACKAGE_DEV_VERSION} TO PROJECT ${WDMS_LIB_REGISTRY} REGISTRY ---- ---- ----
- TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=${CI_REGISTRY_USER} python -m twine upload --repository-url ${CI_API_V4_URL}/projects/${WDMS_LIB_REGISTRY}/packages/pypi dist/*
only:
- merge_requests
......@@ -174,4 +174,4 @@
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
END OF TERMS AND CONDITIONS
\ No newline at end of file
# Wellbore-aws-lib
# osdu-core-lib-python-aws
This is the python package for osdu AWS cloud implementations.
## testing
Access details to run this module:
```bash
export AWS_REGION='xxxx'
export ENVIRONMENT='osdu-'
```
# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# https://docs.aws.amazon.com/codebuild/latest/userguide/build-spec-ref.html
# https://docs.aws.amazon.com/codebuild/latest/userguide/build-env-ref-env-vars.html
version: 0.2
phases:
install:
commands:
- echo "Installing requirements.txt"
- pip3 install -r requirements.txt
- pip3 install -r requirements_opengroup.txt
- pip3 install -r requirements_dev.txt
- pip3 install pylint
- pip3 install twine
build:
commands:
- export REPO_NAME=${PWD##*/}
- export BRANCH_NAME=`echo ${CODEBUILD_SOURCE_VERSION} | awk '{gsub("refs/heads/","");gsub("\\.","-");gsub("[[:space:]]","-")}1' | sed 's/\//-/g' | awk '{print tolower($0)}'`
- pylint ./osdu_aws/storage/*.py --disable=F0001 --errors-only
# run basic build
- python setup.py build
- echo ---- ---- ---- UNIT TESTS ---- ---- ----
- pytest tests --junitxml=report.xml
# publish new artifact to code artifact
- export AWS_ACCOUNT_ID=`aws sts get-caller-identity | grep Account | cut -d':' -f 2 | cut -d'"' -f 2`
- aws codeartifact login --tool twine --domain osdu-dev --domain-owner $AWS_ACCOUNT_ID --repository osdu-python --region $AWS_REGION
- python setup.py sdist bdist_wheel
- twine upload --skip-existing --repository codeartifact dist/osdu-core-lib*.tar.gz
artifacts:
files:
- report.xml
name: ${REPO_NAME}_${BRANCH_NAME}_$(date +%F)_${CODEBUILD_BUILD_NUMBER}.zip
# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
__version__ = '0.0.1'
# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
\ No newline at end of file
# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import io
import os
from typing import Any, Optional, Tuple, List
from urllib.parse import quote
from osdu.core.api.storage.blob_storage_base import BlobStorageBase
from osdu.core.api.storage.blob import Blob
from osdu.core.api.storage.tenant import Tenant
import boto3
try:
import ujson as json
except ImportError:
import json # type: ignore
def set_default(obj):
""" convert set to list """
if isinstance(obj, set):
return list(obj)
raise TypeError
class AwsStorage(BlobStorageBase):
def __init__(self, session=None, service_account_file: Optional[str] = None):
self._session = session
if(service_account_file):
self._region, self._aws_env = service_account_file.split("$$")
else:
self._region = os.getenv("AWS_REGION", "us-east-1")
self._aws_env = os.getenv("ENVIRONMENT", "")
self._access_key = ""
self._secret_key = ""
self._session_token = ""
self._bucket_name =""
def getPolicy(self, bucket: str, keypath:str):
""" policy for access keypath folder in bucket """
# Statement 1: Allow Listing files at the file location
# Statement 2: Allow Listing files under the file location
# Statement 3: Allow Uploading files at the file location
# Statement 4: Allow Uploading files under the file location
UploadPolicy = {
'Version': '2012-10-17',
'Statement': [{
'Sid': 'One',
'Effect': 'Allow',
'Action': [
's3:ListBucketVersions',
's3:ListBucket'
],
'Resource': ["arn:aws:s3:::%s" % bucket ],
'Condition': {
'StringEquals': {
"s3:prefix": "%s/" % keypath
}
}
},
{
'Sid': 'Two',
'Effect': 'Allow',
'Action': [
"s3:*"
],
'Resource': ["arn:aws:s3:::%s" % bucket ],
'Condition': {
'StringLike': {
"s3:prefix": "%s/*" % keypath
}
}
},
{
'Sid': "Three",
'Effect': "Allow",
'Action': [
"s3:PutObject",
"s3:DeleteObject",
"s3:GetObject",
"s3:HeadObject",
"s3:ListObjects",
"s3:ListBucketMultipartUploads",
"s3:AbortMultipartUpload",
"s3:ListMultipartUploadParts"
],
'Resource': [
"arn:aws:s3:::%s/%s/" % (bucket, keypath)
]
},
{
'Sid': "Four",
'Effect': "Allow",
'Action': [
"s3:PutObject",
"s3:DeleteObject",
"s3:GetObject",
"s3:HeadObject",
"s3:ListObjects",
"s3:ListBucketMultipartUploads",
"s3:AbortMultipartUpload",
"s3:ListMultipartUploadParts"
],
'Resource': [
"arn:aws:s3:::%s/%s/*" % (bucket, keypath)
]
}
]}
return UploadPolicy
async def getCredentials(self, tenant: Tenant):
""" get credentials to access s3 bucket for tenant's folder """
# all tenants in one bucket(self._bucket_name). Each tenant.bucket_name is in fact, a folder in self._bucket_name
folder = tenant.bucket_name
ssm = boto3.client('ssm', region_name=self._region)
if(self._bucket_name == ""):
# get bucket name from ssm
response = ssm.get_parameter(Name='/osdu/'+self._aws_env+'/wellbore-ddms/wellbore-ddms-s3-bucket-name',WithDecryption=True)
self._bucket_name = response['Parameter']['Value']
bucket_policy = self.getPolicy(self._bucket_name, folder)
response =ssm.get_parameter(Name='/osdu/' + self._aws_env + '/wellbore-ddms/iam/s3-access-role-arn')
roleArn = response['Parameter']['Value']
response = ssm.get_parameter(Name='/osdu/' + self._aws_env + '/wellbore-ddms/temp-cred-expiration-duration')
exp = response['Parameter']['Value']
sts_client = boto3.client('sts', region_name=self._region)
assumed_role_object=sts_client.assume_role(
RoleArn=roleArn,
RoleSessionName="AssumeRoleSession1",
Policy=json.dumps(bucket_policy, default=set_default),
ExternalId="OSDUAWS",
DurationSeconds= int(exp) )
credentials=assumed_role_object['Credentials']
self._access_key = credentials['AccessKeyId']
self._secret_key = credentials['SecretAccessKey']
self._session_token = credentials['SessionToken']
async def upload(self, tenant: Tenant, object_name: str, file_data: Any, *,
auth: Optional = None, content_type: str = None, metadata: dict = None,
timeout: int = 30, **kwargs):
bucket = tenant.bucket_name
await self.getCredentials(tenant)
object_name = f"{bucket}/{object_name}"
s3_client = boto3.client(
's3',
region_name=self._region,
aws_access_key_id=self._access_key,
aws_secret_access_key=self._secret_key,
aws_session_token=self._session_token
)
extra: dict = {}
if(content_type != None):
extra['ContentType'] = content_type
if(metadata != None):
extra['Metadata'] = metadata
if(auth!= None):
extra['ACL'] = auth
if len(extra)==0:
response = s3_client.upload_fileobj(file_data, self._bucket_name,object_name)
else:
response = s3_client.upload_fileobj(file_data, self._bucket_name, object_name,ExtraArgs=extra)
return response
async def delete(self, tenant: Tenant, object_name: str,
*, auth: Optional = None, timeout: int = 10, **kwargs):
encoded_object_name = quote(object_name, safe='')
bucket = tenant.bucket_name
await self.getCredentials(tenant)
s3_client = boto3.client(
's3',
region_name=self._region,
aws_access_key_id=self._access_key,
aws_secret_access_key=self._secret_key,
aws_session_token=self._session_token
)
response = s3_client.delete_object(
Bucket=self._bucket_name,
Key=encoded_object_name,
)
return response
async def download(self, tenant: Tenant, object_name: str,
*, auth: Optional = None, timeout: int = 10, **kwargs) -> bytes:
await self.getCredentials(tenant)
bucket = tenant.bucket_name
object_name = f"{bucket}/{object_name}"
s3_client = boto3.client(
's3',
region_name=self._region,
aws_access_key_id=self._access_key,
aws_secret_access_key=self._secret_key,
aws_session_token=self._session_token
)
obj = s3_client.get_object(Bucket=self._bucket_name, Key=object_name)
return obj['Body'].read()
async def download_metadata(self, tenant: Tenant, object_name: str,
*, auth: Optional = None, timeout: int = 10, ** kwargs) -> Blob:
await self.getCredentials(tenant)
bucket = tenant.bucket_name
object_name = f"{bucket}/{object_name}"
s3_client = boto3.client(
's3',
region_name=self._region,
aws_access_key_id=self._access_key,
aws_secret_access_key=self._secret_key,
aws_session_token=self._session_token
)
metadata = s3_client.head_object(
Bucket=self._bucket_name,
Key=object_name,
)
return Blob(identifier=metadata.get('ETag', None),
bucket=bucket,
name=metadata.get('name', object_name),
metadata=metadata,
acl=metadata.get('acl', None),
content_type=metadata.get('ContentType', None),
time_created=metadata.get('timeCreated', None),
time_updated=metadata.get('LastModified', None),
size=metadata.get('ContentLength', -1)
)
async def list_objects(self, tenant: Tenant, *args,
auth: Optional = None, prefix: str = '', page_token: Optional[str] = None,
max_result: Optional[int] = None, timeout: int = 10, **kwargs) -> List[str]:
await self.getCredentials(tenant)
bucket = tenant.bucket_name
kwargs = {'Bucket': self._bucket_name}
if prefix is not None:
kwargs['Prefix'] = prefix
if max_result is not None:
kwargs['MaxKeys'] = max_result
s3_client = boto3.client(
's3',
region_name=self._region,
aws_access_key_id=self._access_key,
aws_secret_access_key=self._secret_key,
aws_session_token=self._session_token
)
response = s3_client.list_objects_v2(
**kwargs
)
objects_list = []
if 'Contents' in response.keys():
for key in response['Contents']:
objects_list.append(key['Key'])
return objects_list
pytest>=3
pytest-asyncio
pytest-cov
# osdu core lib main python
--extra-index-url \
https://community.opengroup.org/api/v4/projects/465/packages/pypi/simple/
osdu-core-lib-python>=0.4.0, <0.5
# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import re
from setuptools import setup, find_packages
from osdu_aws import __version__ as osdu_aws_version
setup(
name='osdu-core-lib-python-aws',
version=osdu_aws_version,
packages=find_packages(),
author="aws wellbore DDMS",
author_email="aws@aws.com",
description="osdu python core lib - aws cloud implementation",
license="Apache License, Version 2.0",
# We need py 3.7 for some of the tools used here
python_requires='>=3.7',
)
# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#