From cf4513ae05fb3474c6f57e05cd233c65b6ab8aa3 Mon Sep 17 00:00:00 2001 From: Yunhua Koglin Date: Mon, 16 Aug 2021 13:08:49 -0500 Subject: [PATCH 01/36] update --- tests/aws-test/build-aws/prepare-dist.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/aws-test/build-aws/prepare-dist.sh b/tests/aws-test/build-aws/prepare-dist.sh index 1ee26d90..b05bbe8c 100644 --- a/tests/aws-test/build-aws/prepare-dist.sh +++ b/tests/aws-test/build-aws/prepare-dist.sh @@ -32,3 +32,6 @@ fi cp -r tests/aws-test "${INTEGRATION_TEST_OUTPUT_DIR}" cp -r tests/integration "${INTEGRATION_TEST_OUTPUT_DIR}" cp -r schema/indexation "${INTEGRATION_TEST_OUTPUT_DIR}" +cp -r tests/dependencies "${INTEGRATION_TEST_OUTPUT_DIR}" +cp -r tests/performace "${INTEGRATION_TEST_OUTPUT_DIR}" +cp -r tests/unit "${INTEGRATION_TEST_OUTPUT_DIR}" \ No newline at end of file -- GitLab From e3dd7afce96b2fb7bbc716fd70dfade4dad393fc Mon Sep 17 00:00:00 2001 From: Yunhua Koglin Date: Mon, 16 Aug 2021 16:00:32 -0500 Subject: [PATCH 02/36] update --- tests/aws-test/build-aws/run-tests.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/aws-test/build-aws/run-tests.sh b/tests/aws-test/build-aws/run-tests.sh index aa661d0c..670fae79 100644 --- a/tests/aws-test/build-aws/run-tests.sh +++ b/tests/aws-test/build-aws/run-tests.sh @@ -42,9 +42,11 @@ source env/bin/activate python3 -m pip install -r ./aws-test/build-aws/requirements.txt rm -rf test-reports/ mkdir test-reports +echo $(pwd) +echo 'cd to integration\n' cd integration - +echo $(pwd) acl_domain='example.com' legal_tag='opendes-sdmstestlegaltag' -- GitLab From f7f5290f28617e22c9746f0675af2eff075cf2d7 Mon Sep 17 00:00:00 2001 From: Yunhua Koglin Date: Mon, 16 Aug 2021 16:11:10 -0500 Subject: [PATCH 03/36] update --- tests/aws-test/build-aws/prepare-dist.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/aws-test/build-aws/prepare-dist.sh b/tests/aws-test/build-aws/prepare-dist.sh index b05bbe8c..db80ace9 100644 --- a/tests/aws-test/build-aws/prepare-dist.sh +++ b/tests/aws-test/build-aws/prepare-dist.sh @@ -33,5 +33,5 @@ cp -r tests/aws-test "${INTEGRATION_TEST_OUTPUT_DIR}" cp -r tests/integration "${INTEGRATION_TEST_OUTPUT_DIR}" cp -r schema/indexation "${INTEGRATION_TEST_OUTPUT_DIR}" cp -r tests/dependencies "${INTEGRATION_TEST_OUTPUT_DIR}" -cp -r tests/performace "${INTEGRATION_TEST_OUTPUT_DIR}" +cp -r tests/performance "${INTEGRATION_TEST_OUTPUT_DIR}" cp -r tests/unit "${INTEGRATION_TEST_OUTPUT_DIR}" \ No newline at end of file -- GitLab From 6ec900f89311ca6c951c73d9977c36d0b4006bb2 Mon Sep 17 00:00:00 2001 From: Yunhua Koglin Date: Wed, 18 Aug 2021 09:47:57 -0500 Subject: [PATCH 04/36] update --- tests/aws-test/build-aws/run-integration-tests.sh | 2 +- tests/aws-test/build-aws/run-tests.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/aws-test/build-aws/run-integration-tests.sh b/tests/aws-test/build-aws/run-integration-tests.sh index 20e286db..0fcba959 100644 --- a/tests/aws-test/build-aws/run-integration-tests.sh +++ b/tests/aws-test/build-aws/run-integration-tests.sh @@ -23,4 +23,4 @@ cd tests/integration python3 gen_postman_env.py --token $svctoken --base_url $svc_url --cloud_provider "aws" --acl_domain $acl_domain --legal_tag $legal_tag --data_partition $tenant -pytest ./functional --environment="./generated/postman_environment.json" --filter-tag=basic +pytest ./functional --environment="./generated/postman_environment.json" --filter-tag=!search diff --git a/tests/aws-test/build-aws/run-tests.sh b/tests/aws-test/build-aws/run-tests.sh index 670fae79..d33b74ad 100644 --- a/tests/aws-test/build-aws/run-tests.sh +++ b/tests/aws-test/build-aws/run-tests.sh @@ -52,7 +52,7 @@ legal_tag='opendes-sdmstestlegaltag' python3 gen_postman_env.py --token $token --base_url $svc_url --cloud_provider "aws" --acl_domain $acl_domain --legal_tag $legal_tag --data_partition $tenant -pytest ./functional --environment="./generated/postman_environment.json" --filter-tag=basic +pytest ./functional --environment="./generated/postman_environment.json" --filter-tag=!search TEST_EXIT_CODE=$? deactivate -- GitLab From 7e203c03b0343640f9f579edf9984953a1fb556f Mon Sep 17 00:00:00 2001 From: Yunhua Koglin Date: Fri, 20 Aug 2021 08:02:27 -0500 Subject: [PATCH 05/36] update --- app/injector/aws_injector.py | 7 ++++- requirements.in | 2 +- requirements.txt | 61 +++++++++++++++++------------------- 3 files changed, 35 insertions(+), 35 deletions(-) diff --git a/app/injector/aws_injector.py b/app/injector/aws_injector.py index 27876189..a84a71b5 100644 --- a/app/injector/aws_injector.py +++ b/app/injector/aws_injector.py @@ -14,6 +14,7 @@ from osdu.core.api.storage.blob_storage_base import BlobStorageBase from osdu_aws.storage.storage_aws import AwsStorage +from osdu_aws.storage.dask_storage_parameters import get_dask_storage_parameters as aws_parameters from app.bulk_persistence.dask.dask_bulk_storage import DaskBulkStorage from .app_injector import AppInjector, AppInjectorModule @@ -34,4 +35,8 @@ class AwsInjector(AppInjectorModule): @staticmethod async def build_aws_dask_blob_storage() -> DaskBulkStorage: - raise NotImplementedError() + ctx: Context = Context.current() + tenant = await resolve_tenant(ctx.partition_id) + service_account_file=f'{Config.aws_region.value}$${Config.aws_env.value}' + params = await aws_parameters(tenant, service_account_file) + return await DaskBulkStorage.create(params) diff --git a/requirements.in b/requirements.in index 4f266dff..4a2a86c0 100644 --- a/requirements.in +++ b/requirements.in @@ -35,5 +35,5 @@ osdu-data-ecosystem-search>=0.3.2, <0.4 osdu-core-lib-python-ibm~=1.1.0 osdu-core-lib-python-gcp~=1.1.0 osdu-core-lib-python-azure~=1.2.1 -osdu-core-lib-python-aws>=0.0.1, <0.1 +osdu-core-lib-python-aws>=0.0.2, <0.1 osdu-core-lib-python~=1.1.0 diff --git a/requirements.txt b/requirements.txt index 79576d66..50ba6da3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.7 +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # # pip-compile requirements.in @@ -8,7 +8,7 @@ adal==1.2.7 # via azure-datalake-store -adlfs==0.7.7 +adlfs==2021.8.2 # via osdu-core-lib-python-azure aiohttp==3.7.4.post0 # via @@ -17,9 +17,9 @@ aiohttp==3.7.4.post0 # gcsfs # osdu-core-lib-python # osdu-core-lib-python-gcp -aioredis==1.3.1 +aioredis==2.0.0 # via osdu-core-lib-python -anyio==3.2.1 +anyio==3.3.0 # via httpcore asgiref==3.4.1 # via uvicorn @@ -36,7 +36,7 @@ azure-common==1.1.27 # azure-keyvault-certificates # azure-keyvault-keys # azure-keyvault-secrets -azure-core==1.16.0 +azure-core==1.17.0 # via # adlfs # azure-identity @@ -46,7 +46,7 @@ azure-core==1.16.0 # azure-storage-blob azure-datalake-store==0.0.52 # via adlfs -azure-identity==1.6.0 +azure-identity==1.6.1 # via # adlfs # osdu-core-lib-python-azure @@ -64,9 +64,9 @@ azure-storage-blob==12.8.1 # osdu-core-lib-python-azure backoff==1.11.1 # via -r requirements.in -boto3==1.18.1 +boto3==1.18.25 # via osdu-core-lib-python-ibm -botocore==1.21.1 +botocore==1.21.25 # via # boto3 # osdu-core-lib-python-ibm @@ -86,8 +86,10 @@ cffi==1.14.6 # cryptography chardet==4.0.0 # via aiohttp -charset-normalizer==2.0.3 - # via requests +charset-normalizer==2.0.4 + # via + # httpx + # requests click==8.0.1 # via # -r requirements.in @@ -132,29 +134,29 @@ fsspec==2021.7.0 # osdu-core-lib-python-azure gcsfs==2021.7.0 # via osdu-core-lib-python-gcp -google-api-core[grpc]==1.31.0 +google-api-core[grpc]==1.31.2 # via # google-cloud-core # google-cloud-monitoring # google-cloud-trace # opencensus -google-auth==1.33.0 +google-auth==1.35.0 # via # gcsfs # google-api-core # google-auth-oauthlib # google-cloud-core -google-auth-oauthlib==0.4.4 +google-auth-oauthlib==0.4.5 # via gcsfs -google-cloud-core==1.7.1 +google-cloud-core==1.7.2 # via google-cloud-trace -google-cloud-monitoring==0.36.0 +google-cloud-monitoring==2.4.2 # via opencensus-ext-stackdriver google-cloud-trace==0.24.0 # via opencensus-ext-stackdriver googleapis-common-protos==1.53.0 # via google-api-core -grpcio==1.38.1 +grpcio==1.39.0 # via # google-api-core # opencensus-ext-ocagent @@ -165,11 +167,9 @@ h11==0.12.0 # uvicorn heapdict==1.0.1 # via zict -hiredis==2.0.0 - # via aioredis httpcore==0.13.6 # via httpx -httpx==0.18.2 +httpx==0.19.0 # via # osdu-data-ecosystem-search # osdu-data-ecosystem-storage @@ -179,8 +179,6 @@ idna==3.2 # requests # rfc3986 # yarl -importlib-metadata==4.6.3 - # via click isodate==0.6.0 # via msrest jmespath==0.10.0 @@ -193,7 +191,7 @@ jsonpath-ng==1.5.3 # via -r requirements.in locket==0.2.1 # via partd -msal==1.12.0 +msal==1.13.0 # via # azure-identity # msal-extensions @@ -213,7 +211,7 @@ multidict==5.1.0 # yarl natsort==7.1.1 # via -r requirements.in -numpy==1.21.1 +numpy==1.21.2 # via # pandas # pyarrow @@ -234,7 +232,7 @@ opencensus-ext-logging==0.1.0 # via -r requirements.in opencensus-ext-ocagent==0.7.1 # via -r requirements.in -opencensus-ext-stackdriver==0.7.4 +opencensus-ext-stackdriver==0.8.0 # via -r requirements.in opencensus-proto==0.1.0 # via opencensus-ext-ocagent @@ -258,6 +256,7 @@ packaging==21.0 # via # dask # google-api-core + # google-cloud-monitoring pandas==1.2.4 # via -r requirements.in partd==1.2.0 @@ -266,10 +265,13 @@ ply==3.11 # via jsonpath-ng portalocker==1.7.1 # via msal-extensions +proto-plus==1.19.0 + # via google-cloud-monitoring protobuf==3.17.3 # via # google-api-core # googleapis-common-protos + # proto-plus psutil==5.8.0 # via # distributed @@ -374,20 +376,15 @@ tornado==6.1 typing-extensions==3.10.0.0 # via # aiohttp - # anyio - # asgiref - # importlib-metadata + # aioredis # osdu-data-ecosystem-search # osdu-data-ecosystem-storage # pydantic - # structlog - # uvicorn - # yarl urllib3==1.26.6 # via # botocore # requests -uvicorn==0.14.0 +uvicorn==0.15.0 # via -r requirements.in xmltodict==0.11.0 # via osdu-log-recognition-lib @@ -395,8 +392,6 @@ yarl==1.6.3 # via aiohttp zict==2.0.0 # via distributed -zipp==3.5.0 - # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools -- GitLab From c62122396bb8db7c5a1004acb3fd3f464ce47a8c Mon Sep 17 00:00:00 2001 From: Yunhua Koglin Date: Fri, 20 Aug 2021 09:47:22 -0500 Subject: [PATCH 06/36] update --- app/injector/aws_injector.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/injector/aws_injector.py b/app/injector/aws_injector.py index a84a71b5..425c8cb6 100644 --- a/app/injector/aws_injector.py +++ b/app/injector/aws_injector.py @@ -15,7 +15,8 @@ from osdu.core.api.storage.blob_storage_base import BlobStorageBase from osdu_aws.storage.storage_aws import AwsStorage from osdu_aws.storage.dask_storage_parameters import get_dask_storage_parameters as aws_parameters - +from app.utils import Context +from app.bulk_persistence import resolve_tenant from app.bulk_persistence.dask.dask_bulk_storage import DaskBulkStorage from .app_injector import AppInjector, AppInjectorModule from app.conf import Config -- GitLab From 6738ca12d2808aab74e21d7a7cf447463b52d571 Mon Sep 17 00:00:00 2001 From: Yunhua Koglin Date: Fri, 20 Aug 2021 10:47:13 -0500 Subject: [PATCH 07/36] update --- app/injector/ibm_injector.py | 46 +++++++++++++++++++----------------- requirements.in | 3 ++- requirements.txt | 27 ++++++++++----------- 3 files changed, 39 insertions(+), 37 deletions(-) diff --git a/app/injector/ibm_injector.py b/app/injector/ibm_injector.py index 72a6ab92..eb71adee 100644 --- a/app/injector/ibm_injector.py +++ b/app/injector/ibm_injector.py @@ -1,33 +1,35 @@ -from osdu.core.api.storage.blob_storage_base import BlobStorageBase -from osdu_ibm.storage.blob_storage_ibm import IBMObjectStorage +# from osdu.core.api.storage.blob_storage_base import BlobStorageBase +# from osdu_ibm.storage.blob_storage_ibm import IBMObjectStorage -from app.utils import get_http_client_session -from app.utils import Context -from .app_injector import AppInjector, AppInjectorModule -from app.bulk_persistence import resolve_tenant -from app.bulk_persistence.dask.dask_bulk_storage import DaskBulkStorage -from osdu_ibm.storage.dask_storage_parameters import get_dask_storage_parameters as ibm_parameters +# from app.utils import get_http_client_session +# from app.utils import Context +# from .app_injector import AppInjector, AppInjectorModule +# from app.bulk_persistence import resolve_tenant +# from app.bulk_persistence.dask.dask_bulk_storage import DaskBulkStorage +# from osdu_ibm.storage.dask_storage_parameters import get_dask_storage_parameters as ibm_parameters class IBMInjector(AppInjectorModule): def configure(self, app_injector: AppInjector): - app_injector.register(BlobStorageBase, IBMInjector.build_ibm_blob_storage) - app_injector.register(DaskBulkStorage, IBMInjector.build_ibm_dask_blob_storage) + raise NotImplementedError() + # app_injector.register(BlobStorageBase, IBMInjector.build_ibm_blob_storage) + # app_injector.register(DaskBulkStorage, IBMInjector.build_ibm_dask_blob_storage) @staticmethod async def build_ibm_blob_storage(*args, **kwargs) -> BlobStorageBase: - ctx: Context = Context.current() - # TODO to be reviewed - tenant = await resolve_tenant(ctx.partition_id) - return IBMObjectStorage( - session=get_http_client_session(), - service_account_file=tenant.credentials - ) + raise NotImplementedError() + # ctx: Context = Context.current() + # # TODO to be reviewed + # tenant = await resolve_tenant(ctx.partition_id) + # return IBMObjectStorage( + # session=get_http_client_session(), + # service_account_file=tenant.credentials + # ) @staticmethod async def build_ibm_dask_blob_storage() -> DaskBulkStorage: - # raise NotImplementedError() - ctx: Context = Context.current() - tenant = await resolve_tenant(ctx.partition_id) - params = await ibm_parameters(tenant) - return await DaskBulkStorage.create(params) + raise NotImplementedError() + # ctx: Context = Context.current() + # tenant = await resolve_tenant(ctx.partition_id) + # params = await ibm_parameters(tenant) + # return await DaskBulkStorage.create(params) diff --git a/requirements.in b/requirements.in index 4a2a86c0..1967d6c7 100644 --- a/requirements.in +++ b/requirements.in @@ -32,8 +32,9 @@ osdu-log-recognition-lib>=0.0.9 osdu-data-ecosystem-storage~=1.2.0 osdu-data-ecosystem-search>=0.3.2, <0.4 -osdu-core-lib-python-ibm~=1.1.0 +# osdu-core-lib-python-ibm~=1.1.0 osdu-core-lib-python-gcp~=1.1.0 osdu-core-lib-python-azure~=1.2.1 osdu-core-lib-python-aws>=0.0.2, <0.1 osdu-core-lib-python~=1.1.0 +s3fs \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 50ba6da3..d4cfef41 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,13 +10,18 @@ adal==1.2.7 # via azure-datalake-store adlfs==2021.8.2 # via osdu-core-lib-python-azure +aiobotocore==1.4.0 + # via s3fs aiohttp==3.7.4.post0 # via # -r requirements.in # adlfs + # aiobotocore # gcsfs # osdu-core-lib-python # osdu-core-lib-python-gcp +aioitertools==0.8.0 + # via aiobotocore aioredis==2.0.0 # via osdu-core-lib-python anyio==3.3.0 @@ -64,13 +69,8 @@ azure-storage-blob==12.8.1 # osdu-core-lib-python-azure backoff==1.11.1 # via -r requirements.in -boto3==1.18.25 - # via osdu-core-lib-python-ibm -botocore==1.21.25 - # via - # boto3 - # osdu-core-lib-python-ibm - # s3transfer +botocore==1.20.106 + # via aiobotocore cachetools==4.2.2 # via # google-auth @@ -132,6 +132,7 @@ fsspec==2021.7.0 # dask # gcsfs # osdu-core-lib-python-azure + # s3fs gcsfs==2021.7.0 # via osdu-core-lib-python-gcp google-api-core[grpc]==1.31.2 @@ -182,9 +183,7 @@ idna==3.2 isodate==0.6.0 # via msrest jmespath==0.10.0 - # via - # boto3 - # botocore + # via botocore jsonbender==0.9.3 # via -r requirements.in jsonpath-ng==1.5.3 @@ -244,8 +243,6 @@ osdu-core-lib-python-azure==1.2.1 # via -r requirements.in osdu-core-lib-python-gcp==1.1.0 # via -r requirements.in -osdu-core-lib-python-ibm==1.1.0 - # via -r requirements.in osdu-data-ecosystem-search==0.3.3 # via -r requirements.in osdu-data-ecosystem-storage==1.2.0 @@ -337,8 +334,8 @@ rfc3986[idna2008]==1.5.0 # via httpx rsa==4.7.2 # via google-auth -s3transfer==0.5.0 - # via boto3 +s3fs==2021.7.0 + # via -r requirements.in six==1.16.0 # via # azure-core @@ -386,6 +383,8 @@ urllib3==1.26.6 # requests uvicorn==0.15.0 # via -r requirements.in +wrapt==1.12.1 + # via aiobotocore xmltodict==0.11.0 # via osdu-log-recognition-lib yarl==1.6.3 -- GitLab From 77d6fe9519a02f66c7ff1ff005414cc8fb250ed8 Mon Sep 17 00:00:00 2001 From: Yunhua Koglin Date: Fri, 20 Aug 2021 11:14:46 -0500 Subject: [PATCH 08/36] update --- provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml b/provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml index c80cc448..04b515ef 100644 --- a/provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml +++ b/provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml @@ -79,7 +79,7 @@ phases: - docker push ${ECR_IMAGE} - docker push ${ECR_IMAGE_BRANCH_LATEST} - - python -m pytest --junit-xml=unit_tests_report.xml --cov=app --cov-report=html --cov-report=xml ./tests/unit + # - python -m pytest --junit-xml=unit_tests_report.xml --cov=app --cov-report=html --cov-report=xml ./tests/unit - echo "Generate build-info.json" - | -- GitLab From 62d2adc3fd85a12683d754ecedf28e13e7486423 Mon Sep 17 00:00:00 2001 From: Yunhua Koglin Date: Fri, 20 Aug 2021 13:54:48 -0500 Subject: [PATCH 09/36] update --- requirements.in | 2 + requirements.txt | 430 ++++------------------------------------------- 2 files changed, 39 insertions(+), 393 deletions(-) diff --git a/requirements.in b/requirements.in index 1967d6c7..548e9c82 100644 --- a/requirements.in +++ b/requirements.in @@ -32,6 +32,8 @@ osdu-log-recognition-lib>=0.0.9 osdu-data-ecosystem-storage~=1.2.0 osdu-data-ecosystem-search>=0.3.2, <0.4 +boto3 +botocore # osdu-core-lib-python-ibm~=1.1.0 osdu-core-lib-python-gcp~=1.1.0 osdu-core-lib-python-azure~=1.2.1 diff --git a/requirements.txt b/requirements.txt index d4cfef41..23a0ec69 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,396 +1,40 @@ -# -# This file is autogenerated by pip-compile with python 3.9 -# To update, run: -# -# pip-compile requirements.in -# ---extra-index-url https://community.opengroup.org/api/v4/projects/465/packages/pypi/simple/ - -adal==1.2.7 - # via azure-datalake-store -adlfs==2021.8.2 - # via osdu-core-lib-python-azure -aiobotocore==1.4.0 - # via s3fs +fastapi>=0.65.1 aiohttp==3.7.4.post0 - # via - # -r requirements.in - # adlfs - # aiobotocore - # gcsfs - # osdu-core-lib-python - # osdu-core-lib-python-gcp -aioitertools==0.8.0 - # via aiobotocore -aioredis==2.0.0 - # via osdu-core-lib-python -anyio==3.3.0 - # via httpcore -asgiref==3.4.1 - # via uvicorn -async-timeout==3.0.1 - # via - # aiohttp - # aioredis -attrs==21.2.0 - # via - # aiohttp - # osdu-core-lib-python -azure-common==1.1.27 - # via - # azure-keyvault-certificates - # azure-keyvault-keys - # azure-keyvault-secrets -azure-core==1.17.0 - # via - # adlfs - # azure-identity - # azure-keyvault-certificates - # azure-keyvault-keys - # azure-keyvault-secrets - # azure-storage-blob -azure-datalake-store==0.0.52 - # via adlfs -azure-identity==1.6.1 - # via - # adlfs - # osdu-core-lib-python-azure -azure-keyvault==4.1.0 - # via osdu-core-lib-python-azure -azure-keyvault-certificates==4.3.0 - # via azure-keyvault -azure-keyvault-keys==4.4.0 - # via azure-keyvault -azure-keyvault-secrets==4.3.0 - # via azure-keyvault -azure-storage-blob==12.8.1 - # via - # adlfs - # osdu-core-lib-python-azure -backoff==1.11.1 - # via -r requirements.in -botocore==1.20.106 - # via aiobotocore -cachetools==4.2.2 - # via - # google-auth - # osdu-core-lib-python-azure -certifi==2021.5.30 - # via - # httpx - # msrest - # requests -cffi==1.14.6 - # via - # azure-datalake-store - # cryptography -chardet==4.0.0 - # via aiohttp -charset-normalizer==2.0.4 - # via - # httpx - # requests -click==8.0.1 - # via - # -r requirements.in - # distributed - # uvicorn -cloudpickle==1.6.0 - # via - # dask - # distributed -cryptography==3.4.7 - # via - # -r requirements.in - # adal - # azure-identity - # azure-keyvault-keys - # azure-storage-blob - # msal - # osdu-core-lib-python - # osdu-core-lib-python-gcp - # pyjwt -dask[distributed]==2021.7.2 - # via - # -r requirements.in - # distributed -decorator==5.0.9 - # via - # gcsfs - # jsonpath-ng -distributed==2021.07.2 - # via dask -fastapi==0.65.1 - # via - # -r requirements.in - # osdu-data-ecosystem-search - # osdu-data-ecosystem-storage -fsspec==2021.7.0 - # via - # -r requirements.in - # adlfs - # dask - # gcsfs - # osdu-core-lib-python-azure - # s3fs -gcsfs==2021.7.0 - # via osdu-core-lib-python-gcp -google-api-core[grpc]==1.31.2 - # via - # google-cloud-core - # google-cloud-monitoring - # google-cloud-trace - # opencensus -google-auth==1.35.0 - # via - # gcsfs - # google-api-core - # google-auth-oauthlib - # google-cloud-core -google-auth-oauthlib==0.4.5 - # via gcsfs -google-cloud-core==1.7.2 - # via google-cloud-trace -google-cloud-monitoring==2.4.2 - # via opencensus-ext-stackdriver -google-cloud-trace==0.24.0 - # via opencensus-ext-stackdriver -googleapis-common-protos==1.53.0 - # via google-api-core -grpcio==1.39.0 - # via - # google-api-core - # opencensus-ext-ocagent - # opencensus-proto -h11==0.12.0 - # via - # httpcore - # uvicorn -heapdict==1.0.1 - # via zict -httpcore==0.13.6 - # via httpx -httpx==0.19.0 - # via - # osdu-data-ecosystem-search - # osdu-data-ecosystem-storage -idna==3.2 - # via - # anyio - # requests - # rfc3986 - # yarl -isodate==0.6.0 - # via msrest -jmespath==0.10.0 - # via botocore -jsonbender==0.9.3 - # via -r requirements.in -jsonpath-ng==1.5.3 - # via -r requirements.in -locket==0.2.1 - # via partd -msal==1.13.0 - # via - # azure-identity - # msal-extensions -msal-extensions==0.3.0 - # via azure-identity -msgpack==1.0.2 - # via distributed -msrest==0.6.21 - # via - # azure-keyvault-certificates - # azure-keyvault-keys - # azure-keyvault-secrets - # azure-storage-blob -multidict==5.1.0 - # via - # aiohttp - # yarl -natsort==7.1.1 - # via -r requirements.in -numpy==1.21.2 - # via - # pandas - # pyarrow -oauthlib==3.1.1 - # via requests-oauthlib -opencensus==0.7.13 - # via - # -r requirements.in - # opencensus-ext-azure - # opencensus-ext-logging - # opencensus-ext-ocagent - # opencensus-ext-stackdriver -opencensus-context==0.1.2 - # via opencensus -opencensus-ext-azure==1.0.8 - # via -r requirements.in -opencensus-ext-logging==0.1.0 - # via -r requirements.in -opencensus-ext-ocagent==0.7.1 - # via -r requirements.in -opencensus-ext-stackdriver==0.8.0 - # via -r requirements.in -opencensus-proto==0.1.0 - # via opencensus-ext-ocagent -osdu-core-lib-python==1.1.0 - # via -r requirements.in -osdu-core-lib-python-aws==0.0.2 - # via -r requirements.in -osdu-core-lib-python-azure==1.2.1 - # via -r requirements.in -osdu-core-lib-python-gcp==1.1.0 - # via -r requirements.in -osdu-data-ecosystem-search==0.3.3 - # via -r requirements.in -osdu-data-ecosystem-storage==1.2.0 - # via -r requirements.in -osdu-log-recognition-lib==0.0.9 - # via -r requirements.in -packaging==21.0 - # via - # dask - # google-api-core - # google-cloud-monitoring +cryptography>=2.7 +pyarrow>=0.15.0 pandas==1.2.4 - # via -r requirements.in -partd==1.2.0 - # via dask -ply==3.11 - # via jsonpath-ng -portalocker==1.7.1 - # via msal-extensions -proto-plus==1.19.0 - # via google-cloud-monitoring -protobuf==3.17.3 - # via - # google-api-core - # googleapis-common-protos - # proto-plus -psutil==5.8.0 - # via - # distributed - # opencensus-ext-azure -pyarrow==4.0.1 - # via -r requirements.in -pyasn1==0.4.8 - # via - # pyasn1-modules - # rsa -pyasn1-modules==0.2.8 - # via google-auth -pycparser==2.20 - # via cffi -pydantic==1.8.2 - # via - # fastapi - # osdu-data-ecosystem-search - # osdu-data-ecosystem-storage -pyjwt[crypto]==2.1.0 - # via - # adal - # msal - # osdu-core-lib-python - # osdu-core-lib-python-gcp -pyparsing==2.4.7 - # via packaging -python-dateutil==2.8.2 - # via - # adal - # botocore - # pandas -python-multipart==0.0.5 - # via -r requirements.in -python-rapidjson==1.4 - # via -r requirements.in -python-ulid==1.0.3 - # via -r requirements.in -pytz==2021.1 - # via - # google-api-core - # pandas -pyyaml==5.4.1 - # via - # dask - # distributed -requests==2.26.0 - # via - # adal - # azure-core - # azure-datalake-store - # gcsfs - # google-api-core - # msal - # msrest - # opencensus-ext-azure - # requests-oauthlib -requests-oauthlib==1.3.0 - # via - # google-auth-oauthlib - # msrest -rfc3986[idna2008]==1.5.0 - # via httpx -rsa==4.7.2 - # via google-auth -s3fs==2021.7.0 - # via -r requirements.in -six==1.16.0 - # via - # azure-core - # azure-identity - # azure-keyvault-keys - # google-api-core - # google-auth - # google-cloud-core - # grpcio - # isodate - # jsonpath-ng - # protobuf - # python-dateutil - # python-multipart -sniffio==1.2.0 - # via - # anyio - # httpcore - # httpx -sortedcontainers==2.4.0 - # via distributed -starlette==0.14.2 - # via fastapi -structlog==21.1.0 - # via -r requirements.in -tblib==1.7.0 - # via distributed -toolz==0.11.1 - # via - # dask - # distributed - # partd -tornado==6.1 - # via distributed -typing-extensions==3.10.0.0 - # via - # aiohttp - # aioredis - # osdu-data-ecosystem-search - # osdu-data-ecosystem-storage - # pydantic -urllib3==1.26.6 - # via - # botocore - # requests -uvicorn==0.15.0 - # via -r requirements.in -wrapt==1.12.1 - # via aiobotocore -xmltodict==0.11.0 - # via osdu-log-recognition-lib -yarl==1.6.3 - # via aiohttp -zict==2.0.0 - # via distributed +uvicorn +Click +backoff +structlog +python-rapidjson +python-multipart +jsonpath-ng # TODO: remove when bulk_path no longer supported +jsonbender==0.9.3 +natsort + +opencensus +opencensus-ext-stackdriver +opencensus-ext-azure +opencensus-ext-ocagent +opencensus-ext-logging + +# for chunking feature +dask[distributed]==2021.6.2 +fsspec +python-ulid + +--extra-index-url \ +https://community.opengroup.org/api/v4/projects/465/packages/pypi/simple/ + +osdu-log-recognition-lib>=0.0.9 + +osdu-data-ecosystem-storage~=1.2.0 +osdu-data-ecosystem-search>=0.3.2, <0.4 -# The following packages are considered to be unsafe in a requirements file: -# setuptools +osdu-core-lib-python-ibm~=1.1.0 +osdu-core-lib-python-gcp~=1.1.0 +osdu-core-lib-python-azure~=1.2.1 +osdu-core-lib-python-aws>=0.0.1, <0.1 +osdu-core-lib-python~=1.1.0 +s3fs \ No newline at end of file -- GitLab From 0e73c53115776a9f6ec77976c4140893b690dff6 Mon Sep 17 00:00:00 2001 From: Yunhua Koglin Date: Fri, 20 Aug 2021 18:59:45 -0500 Subject: [PATCH 10/36] update --- requirements.in | 6 +- requirements.txt | 440 +++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 406 insertions(+), 40 deletions(-) diff --git a/requirements.in b/requirements.in index 548e9c82..1390dbdf 100644 --- a/requirements.in +++ b/requirements.in @@ -32,11 +32,11 @@ osdu-log-recognition-lib>=0.0.9 osdu-data-ecosystem-storage~=1.2.0 osdu-data-ecosystem-search>=0.3.2, <0.4 -boto3 -botocore # osdu-core-lib-python-ibm~=1.1.0 osdu-core-lib-python-gcp~=1.1.0 osdu-core-lib-python-azure~=1.2.1 osdu-core-lib-python-aws>=0.0.2, <0.1 osdu-core-lib-python~=1.1.0 -s3fs \ No newline at end of file +boto3==1.17.106 +botocore==1.20.106 +s3fs==2021.8.0 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 23a0ec69..fb26c6fa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,40 +1,406 @@ -fastapi>=0.65.1 +# +# This file is autogenerated by pip-compile with python 3.9 +# To update, run: +# +# pip-compile requirements.in +# +--extra-index-url https://community.opengroup.org/api/v4/projects/465/packages/pypi/simple/ + +adal==1.2.7 + # via azure-datalake-store +adlfs==2021.8.2 + # via osdu-core-lib-python-azure +aiobotocore==1.4.0 + # via s3fs aiohttp==3.7.4.post0 -cryptography>=2.7 -pyarrow>=0.15.0 -pandas==1.2.4 -uvicorn -Click -backoff -structlog -python-rapidjson -python-multipart -jsonpath-ng # TODO: remove when bulk_path no longer supported + # via + # -r requirements.in + # adlfs + # aiobotocore + # gcsfs + # osdu-core-lib-python + # osdu-core-lib-python-gcp +aioitertools==0.8.0 + # via aiobotocore +aioredis==2.0.0 + # via osdu-core-lib-python +anyio==3.3.0 + # via httpcore +asgiref==3.4.1 + # via uvicorn +async-timeout==3.0.1 + # via + # aiohttp + # aioredis +attrs==21.2.0 + # via + # aiohttp + # osdu-core-lib-python +azure-common==1.1.27 + # via + # azure-keyvault-certificates + # azure-keyvault-keys + # azure-keyvault-secrets +azure-core==1.17.0 + # via + # adlfs + # azure-identity + # azure-keyvault-certificates + # azure-keyvault-keys + # azure-keyvault-secrets + # azure-storage-blob +azure-datalake-store==0.0.52 + # via adlfs +azure-identity==1.6.1 + # via + # adlfs + # osdu-core-lib-python-azure +azure-keyvault==4.1.0 + # via osdu-core-lib-python-azure +azure-keyvault-certificates==4.3.0 + # via azure-keyvault +azure-keyvault-keys==4.4.0 + # via azure-keyvault +azure-keyvault-secrets==4.3.0 + # via azure-keyvault +azure-storage-blob==12.8.1 + # via + # adlfs + # osdu-core-lib-python-azure +backoff==1.11.1 + # via -r requirements.in +boto3==1.17.106 + # via -r requirements.in +botocore==1.20.106 + # via + # -r requirements.in + # aiobotocore + # boto3 + # s3transfer +cachetools==4.2.2 + # via + # google-auth + # osdu-core-lib-python-azure +certifi==2021.5.30 + # via + # httpx + # msrest + # requests +cffi==1.14.6 + # via + # azure-datalake-store + # cryptography +chardet==4.0.0 + # via aiohttp +charset-normalizer==2.0.4 + # via + # httpx + # requests +click==8.0.1 + # via + # -r requirements.in + # distributed + # uvicorn +cloudpickle==1.6.0 + # via + # dask + # distributed +cryptography==3.4.7 + # via + # -r requirements.in + # adal + # azure-identity + # azure-keyvault-keys + # azure-storage-blob + # msal + # osdu-core-lib-python + # osdu-core-lib-python-gcp + # pyjwt +dask[distributed]==2021.7.2 + # via + # -r requirements.in + # distributed +decorator==5.0.9 + # via + # gcsfs + # jsonpath-ng +distributed==2021.07.2 + # via dask +fastapi==0.65.1 + # via + # -r requirements.in + # osdu-data-ecosystem-search + # osdu-data-ecosystem-storage +fsspec==2021.7.0 + # via + # -r requirements.in + # adlfs + # dask + # gcsfs + # osdu-core-lib-python-azure + # s3fs +gcsfs==2021.7.0 + # via osdu-core-lib-python-gcp +google-api-core[grpc]==1.31.2 + # via + # google-cloud-core + # google-cloud-monitoring + # google-cloud-trace + # opencensus +google-auth==1.35.0 + # via + # gcsfs + # google-api-core + # google-auth-oauthlib + # google-cloud-core +google-auth-oauthlib==0.4.5 + # via gcsfs +google-cloud-core==1.7.2 + # via google-cloud-trace +google-cloud-monitoring==2.4.2 + # via opencensus-ext-stackdriver +google-cloud-trace==0.24.0 + # via opencensus-ext-stackdriver +googleapis-common-protos==1.53.0 + # via google-api-core +grpcio==1.39.0 + # via + # google-api-core + # opencensus-ext-ocagent + # opencensus-proto +h11==0.12.0 + # via + # httpcore + # uvicorn +heapdict==1.0.1 + # via zict +httpcore==0.13.6 + # via httpx +httpx==0.19.0 + # via + # osdu-data-ecosystem-search + # osdu-data-ecosystem-storage +idna==3.2 + # via + # anyio + # requests + # rfc3986 + # yarl +isodate==0.6.0 + # via msrest +jmespath==0.10.0 + # via + # boto3 + # botocore jsonbender==0.9.3 -natsort - -opencensus -opencensus-ext-stackdriver -opencensus-ext-azure -opencensus-ext-ocagent -opencensus-ext-logging - -# for chunking feature -dask[distributed]==2021.6.2 -fsspec -python-ulid - ---extra-index-url \ -https://community.opengroup.org/api/v4/projects/465/packages/pypi/simple/ - -osdu-log-recognition-lib>=0.0.9 - -osdu-data-ecosystem-storage~=1.2.0 -osdu-data-ecosystem-search>=0.3.2, <0.4 + # via -r requirements.in +jsonpath-ng==1.5.3 + # via -r requirements.in +locket==0.2.1 + # via partd +msal==1.13.0 + # via + # azure-identity + # msal-extensions +msal-extensions==0.3.0 + # via azure-identity +msgpack==1.0.2 + # via distributed +msrest==0.6.21 + # via + # azure-keyvault-certificates + # azure-keyvault-keys + # azure-keyvault-secrets + # azure-storage-blob +multidict==5.1.0 + # via + # aiohttp + # yarl +natsort==7.1.1 + # via -r requirements.in +numpy==1.21.2 + # via + # pandas + # pyarrow +oauthlib==3.1.1 + # via requests-oauthlib +opencensus==0.7.13 + # via + # -r requirements.in + # opencensus-ext-azure + # opencensus-ext-logging + # opencensus-ext-ocagent + # opencensus-ext-stackdriver +opencensus-context==0.1.2 + # via opencensus +opencensus-ext-azure==1.0.8 + # via -r requirements.in +opencensus-ext-logging==0.1.0 + # via -r requirements.in +opencensus-ext-ocagent==0.7.1 + # via -r requirements.in +opencensus-ext-stackdriver==0.8.0 + # via -r requirements.in +opencensus-proto==0.1.0 + # via opencensus-ext-ocagent +osdu-core-lib-python==1.1.0 + # via -r requirements.in +osdu-core-lib-python-aws==0.0.2 + # via -r requirements.in +osdu-core-lib-python-azure==1.2.1 + # via -r requirements.in +osdu-core-lib-python-gcp==1.1.0 + # via -r requirements.in +osdu-data-ecosystem-search==0.3.3 + # via -r requirements.in +osdu-data-ecosystem-storage==1.2.0 + # via -r requirements.in +osdu-log-recognition-lib==0.0.9 + # via -r requirements.in +packaging==21.0 + # via + # dask + # google-api-core + # google-cloud-monitoring +pandas==1.2.4 + # via -r requirements.in +partd==1.2.0 + # via dask +ply==3.11 + # via jsonpath-ng +portalocker==1.7.1 + # via msal-extensions +proto-plus==1.19.0 + # via google-cloud-monitoring +protobuf==3.17.3 + # via + # google-api-core + # googleapis-common-protos + # proto-plus +psutil==5.8.0 + # via + # distributed + # opencensus-ext-azure +pyarrow==4.0.1 + # via -r requirements.in +pyasn1==0.4.8 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.2.8 + # via google-auth +pycparser==2.20 + # via cffi +pydantic==1.8.2 + # via + # fastapi + # osdu-data-ecosystem-search + # osdu-data-ecosystem-storage +pyjwt[crypto]==2.1.0 + # via + # adal + # msal + # osdu-core-lib-python + # osdu-core-lib-python-gcp +pyparsing==2.4.7 + # via packaging +python-dateutil==2.8.2 + # via + # adal + # botocore + # pandas +python-multipart==0.0.5 + # via -r requirements.in +python-rapidjson==1.4 + # via -r requirements.in +python-ulid==1.0.3 + # via -r requirements.in +pytz==2021.1 + # via + # google-api-core + # pandas +pyyaml==5.4.1 + # via + # dask + # distributed +requests==2.26.0 + # via + # adal + # azure-core + # azure-datalake-store + # gcsfs + # google-api-core + # msal + # msrest + # opencensus-ext-azure + # requests-oauthlib +requests-oauthlib==1.3.0 + # via + # google-auth-oauthlib + # msrest +rfc3986[idna2008]==1.5.0 + # via httpx +rsa==4.7.2 + # via google-auth +s3fs==2021.8.0 + # via -r requirements.in +s3transfer==0.4.2 + # via boto3 +six==1.16.0 + # via + # azure-core + # azure-identity + # azure-keyvault-keys + # google-api-core + # google-auth + # google-cloud-core + # grpcio + # isodate + # jsonpath-ng + # protobuf + # python-dateutil + # python-multipart +sniffio==1.2.0 + # via + # anyio + # httpcore + # httpx +sortedcontainers==2.4.0 + # via distributed +starlette==0.14.2 + # via fastapi +structlog==21.1.0 + # via -r requirements.in +tblib==1.7.0 + # via distributed +toolz==0.11.1 + # via + # dask + # distributed + # partd +tornado==6.1 + # via distributed +typing-extensions==3.10.0.0 + # via + # aiohttp + # aioredis + # osdu-data-ecosystem-search + # osdu-data-ecosystem-storage + # pydantic +urllib3==1.26.6 + # via + # botocore + # requests +uvicorn==0.15.0 + # via -r requirements.in +wrapt==1.12.1 + # via aiobotocore +xmltodict==0.11.0 + # via osdu-log-recognition-lib +yarl==1.6.3 + # via aiohttp +zict==2.0.0 + # via distributed -osdu-core-lib-python-ibm~=1.1.0 -osdu-core-lib-python-gcp~=1.1.0 -osdu-core-lib-python-azure~=1.2.1 -osdu-core-lib-python-aws>=0.0.1, <0.1 -osdu-core-lib-python~=1.1.0 -s3fs \ No newline at end of file +# The following packages are considered to be unsafe in a requirements file: +# setuptools -- GitLab From acf78b326c0fe3ce5997eeddbc8a6b5f6c324eb3 Mon Sep 17 00:00:00 2001 From: Yunhua Koglin Date: Fri, 20 Aug 2021 20:58:59 -0500 Subject: [PATCH 11/36] restore ibm --- app/injector/ibm_injector.py | 46 +++++++++---------- .../build-aws/buildspec.yaml | 2 +- 2 files changed, 23 insertions(+), 25 deletions(-) diff --git a/app/injector/ibm_injector.py b/app/injector/ibm_injector.py index eb71adee..72a6ab92 100644 --- a/app/injector/ibm_injector.py +++ b/app/injector/ibm_injector.py @@ -1,35 +1,33 @@ -# from osdu.core.api.storage.blob_storage_base import BlobStorageBase -# from osdu_ibm.storage.blob_storage_ibm import IBMObjectStorage +from osdu.core.api.storage.blob_storage_base import BlobStorageBase +from osdu_ibm.storage.blob_storage_ibm import IBMObjectStorage -# from app.utils import get_http_client_session -# from app.utils import Context -# from .app_injector import AppInjector, AppInjectorModule -# from app.bulk_persistence import resolve_tenant -# from app.bulk_persistence.dask.dask_bulk_storage import DaskBulkStorage -# from osdu_ibm.storage.dask_storage_parameters import get_dask_storage_parameters as ibm_parameters +from app.utils import get_http_client_session +from app.utils import Context +from .app_injector import AppInjector, AppInjectorModule +from app.bulk_persistence import resolve_tenant +from app.bulk_persistence.dask.dask_bulk_storage import DaskBulkStorage +from osdu_ibm.storage.dask_storage_parameters import get_dask_storage_parameters as ibm_parameters class IBMInjector(AppInjectorModule): def configure(self, app_injector: AppInjector): - raise NotImplementedError() - # app_injector.register(BlobStorageBase, IBMInjector.build_ibm_blob_storage) - # app_injector.register(DaskBulkStorage, IBMInjector.build_ibm_dask_blob_storage) + app_injector.register(BlobStorageBase, IBMInjector.build_ibm_blob_storage) + app_injector.register(DaskBulkStorage, IBMInjector.build_ibm_dask_blob_storage) @staticmethod async def build_ibm_blob_storage(*args, **kwargs) -> BlobStorageBase: - raise NotImplementedError() - # ctx: Context = Context.current() - # # TODO to be reviewed - # tenant = await resolve_tenant(ctx.partition_id) - # return IBMObjectStorage( - # session=get_http_client_session(), - # service_account_file=tenant.credentials - # ) + ctx: Context = Context.current() + # TODO to be reviewed + tenant = await resolve_tenant(ctx.partition_id) + return IBMObjectStorage( + session=get_http_client_session(), + service_account_file=tenant.credentials + ) @staticmethod async def build_ibm_dask_blob_storage() -> DaskBulkStorage: - raise NotImplementedError() - # ctx: Context = Context.current() - # tenant = await resolve_tenant(ctx.partition_id) - # params = await ibm_parameters(tenant) - # return await DaskBulkStorage.create(params) + # raise NotImplementedError() + ctx: Context = Context.current() + tenant = await resolve_tenant(ctx.partition_id) + params = await ibm_parameters(tenant) + return await DaskBulkStorage.create(params) diff --git a/provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml b/provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml index 04b515ef..c80cc448 100644 --- a/provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml +++ b/provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml @@ -79,7 +79,7 @@ phases: - docker push ${ECR_IMAGE} - docker push ${ECR_IMAGE_BRANCH_LATEST} - # - python -m pytest --junit-xml=unit_tests_report.xml --cov=app --cov-report=html --cov-report=xml ./tests/unit + - python -m pytest --junit-xml=unit_tests_report.xml --cov=app --cov-report=html --cov-report=xml ./tests/unit - echo "Generate build-info.json" - | -- GitLab From 8fe010811bc380a5ab391ac4aca8c7c33afb2bd2 Mon Sep 17 00:00:00 2001 From: Yunhua Koglin Date: Fri, 20 Aug 2021 21:06:44 -0500 Subject: [PATCH 12/36] add libs --- requirements.in | 11 +++++------ requirements.txt | 8 +++++++- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/requirements.in b/requirements.in index 1390dbdf..6c763d94 100644 --- a/requirements.in +++ b/requirements.in @@ -23,7 +23,9 @@ opencensus-ext-logging dask[distributed]==2021.7.2 fsspec python-ulid - +boto3==1.17.106 +botocore==1.20.106 +s3fs==2021.8.0 --extra-index-url \ https://community.opengroup.org/api/v4/projects/465/packages/pypi/simple/ @@ -32,11 +34,8 @@ osdu-log-recognition-lib>=0.0.9 osdu-data-ecosystem-storage~=1.2.0 osdu-data-ecosystem-search>=0.3.2, <0.4 -# osdu-core-lib-python-ibm~=1.1.0 +osdu-core-lib-python-ibm~=1.1.0 osdu-core-lib-python-gcp~=1.1.0 osdu-core-lib-python-azure~=1.2.1 osdu-core-lib-python-aws>=0.0.2, <0.1 -osdu-core-lib-python~=1.1.0 -boto3==1.17.106 -botocore==1.20.106 -s3fs==2021.8.0 \ No newline at end of file +osdu-core-lib-python~=1.1.0 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index fb26c6fa..a53959a2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,7 @@ # # pip-compile requirements.in # +--index-url https://aws:eyJ2ZXIiOjEsImlzdSI6MTYyOTQ5NDA5MSwiZW5jIjoiQTEyOEdDTSIsInRhZyI6Ik0xV2EzcUg2UUlRVGpTUW9sSTUtMEEiLCJleHAiOjE2Mjk1MzcyOTEsImFsZyI6IkExMjhHQ01LVyIsIml2IjoiaEdRcXlqQkh5bnRLaWpfayJ9.vT3xN0hCevtjezwjg4tktQ.pVTY81mIAV04Mzm2.kfmGoNB3HUJAkk6G4d-MS-UkmSWDKPmsKmmpA24Mqi_YIH89OuazDvgmGdLlqGVCgcxKbZNrKIa1UMfPWqWFVVU-WEea32aIfbXkZkBurYk4blQRJEKHZ86UZPsVDNIlSJz0WxIbWT9CsCP27Gin9rYqw7YeX4B1F20baDQBbziVf9oU8zPlWb0QRE4DsXnMFuzMbXJUuY-wchKBk62YMqAEyoWN7GEE5iWR08z6rxtE9hce7OSLIPx3HyOX7ubwIiMNTtdPbjuvqxpOlnj_sCXMxrRAMZR0FyJMnjc4U_x8EUU7Xs205TxJGd39AhpWnVfCOY5QhG2OJlBe_OqAhGGKfcBUrE6I3O5DNnyDEJ9dbPTOa3FZ4Rx8Q4rWgZhm1iD3MlIUHF8Q_5a89lrn5KxYhitThmQRhzx5AtzcLxKjbyhL4Ssd8mlMVMyCmkKpKl2cG4VMmwGLqCEWQAj_JEM0W5QoF2moPgCtEhJYvNqdsMhuhUSoVnx57Puyri1b8bTK4fJkuC3ROkeBlNr6ub0IIiUv8_834-wZhA9DGx9bFg6RTU2arjMnHZdp1Wno4hWxQ7ptsAUsy3Bj-Dh2pl-w8FVB8RJvAtAvB7Pp8bDvozzCjM0FOezOkgtvysWZPMLqhuPFMKy-6aoUibQwZQQprAJCN5sl95IYGqMFi68KMP5oRxQ5hNf99xZYE8V-QKUv3ENDS1ZW2ygo0u6jEZwpqAVNbmr2Je72ohMMAWlHvHTl5a8qgvjJ_WMQ3ei_uOkISy6SA9KA1m_sv2hnd0FojV9dACuPbPHvp3972CJekKWL48QnPxhIEn285uBAIBjtjWXlFvIBrDPspNyu2LUQ__jOAOHxuGzBOfH0-Q5JXMCgae9dhW5o-1bOW7o_onqwFXWQmyDxINi3IXIa0z1bi_UbKiSt7z8MBc8A8pq0fdnRqxHmZUY86z-gx2-H9pehBVoUU2BtpygZYHc8zijO--GqvsYpPDZCAbksle3u_R6aYRLNkyq6ga3Eg-x5iDsdVL1fAU4xUj36mNKzBR7Ev271CGK1grR9sbvYf_bklimSsoT2cK_6RRma3Z_cbmrLvKUiMDuOfHmPvuYKITO-XUUcPMCo8tpLZnSI8Yg1KvC4xzGAhH1b-nQLQY8BPtupyJNXW90RHM-_z4Rt4zJFvdbBVoAubWqBdZn6tuPh1Y8dW-AGJOafG4FPXXCier5CCeSrTbgkDsRMDalqT7iTMc7pg_BC67xQ3wA3TRH-FabJxSi4jXmWYBrYOFNF9Md5ffolMuZhDcKG5ym0ZBz9iCGVvrhoaMen09G07Q6a1-98MZKvZw86AoNDCRzJjSJZZ3AfV1raIhIlV2UwoLVWaR-KDgqJVQ.576N9F8aR2u1kzius_Dlag@osdu-dev-888733619319.d.codeartifact.us-east-1.amazonaws.com/pypi/osdu-python/simple/ --extra-index-url https://community.opengroup.org/api/v4/projects/465/packages/pypi/simple/ adal==1.2.7 @@ -70,12 +71,15 @@ azure-storage-blob==12.8.1 backoff==1.11.1 # via -r requirements.in boto3==1.17.106 - # via -r requirements.in + # via + # -r requirements.in + # osdu-core-lib-python-ibm botocore==1.20.106 # via # -r requirements.in # aiobotocore # boto3 + # osdu-core-lib-python-ibm # s3transfer cachetools==4.2.2 # via @@ -251,6 +255,8 @@ osdu-core-lib-python-azure==1.2.1 # via -r requirements.in osdu-core-lib-python-gcp==1.1.0 # via -r requirements.in +osdu-core-lib-python-ibm==1.1.0 + # via -r requirements.in osdu-data-ecosystem-search==0.3.3 # via -r requirements.in osdu-data-ecosystem-storage==1.2.0 -- GitLab From b8e6eacf63d93fd52f02cb6e20d1066db492f606 Mon Sep 17 00:00:00 2001 From: Yunhua Koglin Date: Fri, 27 Aug 2021 20:57:37 -0500 Subject: [PATCH 13/36] update requirements.txt --- requirements.txt | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/requirements.txt b/requirements.txt index deaddb36..bf228c2e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,14 +4,14 @@ # # pip-compile requirements.in # ---index-url https://aws:eyJ2ZXIiOjEsImlzdSI6MTYyOTQ5NDA5MSwiZW5jIjoiQTEyOEdDTSIsInRhZyI6Ik0xV2EzcUg2UUlRVGpTUW9sSTUtMEEiLCJleHAiOjE2Mjk1MzcyOTEsImFsZyI6IkExMjhHQ01LVyIsIml2IjoiaEdRcXlqQkh5bnRLaWpfayJ9.vT3xN0hCevtjezwjg4tktQ.pVTY81mIAV04Mzm2.kfmGoNB3HUJAkk6G4d-MS-UkmSWDKPmsKmmpA24Mqi_YIH89OuazDvgmGdLlqGVCgcxKbZNrKIa1UMfPWqWFVVU-WEea32aIfbXkZkBurYk4blQRJEKHZ86UZPsVDNIlSJz0WxIbWT9CsCP27Gin9rYqw7YeX4B1F20baDQBbziVf9oU8zPlWb0QRE4DsXnMFuzMbXJUuY-wchKBk62YMqAEyoWN7GEE5iWR08z6rxtE9hce7OSLIPx3HyOX7ubwIiMNTtdPbjuvqxpOlnj_sCXMxrRAMZR0FyJMnjc4U_x8EUU7Xs205TxJGd39AhpWnVfCOY5QhG2OJlBe_OqAhGGKfcBUrE6I3O5DNnyDEJ9dbPTOa3FZ4Rx8Q4rWgZhm1iD3MlIUHF8Q_5a89lrn5KxYhitThmQRhzx5AtzcLxKjbyhL4Ssd8mlMVMyCmkKpKl2cG4VMmwGLqCEWQAj_JEM0W5QoF2moPgCtEhJYvNqdsMhuhUSoVnx57Puyri1b8bTK4fJkuC3ROkeBlNr6ub0IIiUv8_834-wZhA9DGx9bFg6RTU2arjMnHZdp1Wno4hWxQ7ptsAUsy3Bj-Dh2pl-w8FVB8RJvAtAvB7Pp8bDvozzCjM0FOezOkgtvysWZPMLqhuPFMKy-6aoUibQwZQQprAJCN5sl95IYGqMFi68KMP5oRxQ5hNf99xZYE8V-QKUv3ENDS1ZW2ygo0u6jEZwpqAVNbmr2Je72ohMMAWlHvHTl5a8qgvjJ_WMQ3ei_uOkISy6SA9KA1m_sv2hnd0FojV9dACuPbPHvp3972CJekKWL48QnPxhIEn285uBAIBjtjWXlFvIBrDPspNyu2LUQ__jOAOHxuGzBOfH0-Q5JXMCgae9dhW5o-1bOW7o_onqwFXWQmyDxINi3IXIa0z1bi_UbKiSt7z8MBc8A8pq0fdnRqxHmZUY86z-gx2-H9pehBVoUU2BtpygZYHc8zijO--GqvsYpPDZCAbksle3u_R6aYRLNkyq6ga3Eg-x5iDsdVL1fAU4xUj36mNKzBR7Ev271CGK1grR9sbvYf_bklimSsoT2cK_6RRma3Z_cbmrLvKUiMDuOfHmPvuYKITO-XUUcPMCo8tpLZnSI8Yg1KvC4xzGAhH1b-nQLQY8BPtupyJNXW90RHM-_z4Rt4zJFvdbBVoAubWqBdZn6tuPh1Y8dW-AGJOafG4FPXXCier5CCeSrTbgkDsRMDalqT7iTMc7pg_BC67xQ3wA3TRH-FabJxSi4jXmWYBrYOFNF9Md5ffolMuZhDcKG5ym0ZBz9iCGVvrhoaMen09G07Q6a1-98MZKvZw86AoNDCRzJjSJZZ3AfV1raIhIlV2UwoLVWaR-KDgqJVQ.576N9F8aR2u1kzius_Dlag@osdu-dev-888733619319.d.codeartifact.us-east-1.amazonaws.com/pypi/osdu-python/simple/ +--index-url https://aws:eyJ2ZXIiOjEsImlzdSI6MTYzMDExNDkxNCwiZW5jIjoiQTEyOEdDTSIsInRhZyI6InBnSGNPamhIWk8teVdfaFI3TnNuV0EiLCJleHAiOjE2MzAxNTgxMTQsImFsZyI6IkExMjhHQ01LVyIsIml2IjoiWTBScnZabWRPNlBfUnNSVSJ9.43R-J82u8v3qAYoFe1_-aQ.y_p2h4tIaXNR3B9E.46VoUr4wBFrK3liMOCzGCOSfuAKh1BPOcl_MJnyjIwjTEOalMimXF3HjjttkHC8eHD1Na7eFsUDo2FFf6wMB425cTDYMgJlGaaXJlzc-83ORTnh2v7Asactf5K40GbJ4XLLz0YOHu8BIPKJFoLfltDj4tbZdJIxwxski9BwzklElUirS-6w5QCzM5_UDA_tAqQQTWscLyaMK4DJS5deeN_jw7WovW6lM1AqpyxZY5hGvnjV25i4rgiItksutg1SMwCsQC-LU7YO2w-IFSiKFSlO_9h6YEzeCXcwptDPAaalnhSscZRXHZUs2G7Nyii7kI7VATArqpTVo2YYI85A7yM-tRIgfNvlXSn_FvEXE41egUs25zMAyXDhpaS4KMLSTGGRRSZBN6aoS5YOh-skjFP72fej99F00KovfRR_ETpyTh9xoZxyiJvZMFLKpSgo3dWCGt5pKLHQtrlJnEpghl2ecwCdjh5sasEEOKbGtn_6H7fDiixnhP-zMyeO8ajixepkCkIhoxnPQolt865Q_JLr6nFu9LDu3Hecl91BojXqkE-kacIdVb9zWpjuM1PtNBQn3P13fovgGX81LriVbnhGjJWYB3ec80rSsodVQNI8s-_UURb6qucb8tgUjJh10PYjs4hkiagEAdAi5119FnkHqB-O1VJ96FWLmw7nvsp7M8__SMmj_nYBVbEQipfIDBGzI0rFBNDnL3oJjLrWAUVA9Zy1EhCZ0P6l9zs9th968bbVD8CUzcefBHAKCrD9_oLnFCy5UQaIbrS827pcalqthwdmxZjRJ-hjlBB9CGdz_SZAgn3hypvMxEMsAuRKgt6q8HCHK6ZttzCbb_p3RfOMMT7hdm9rHQ538lfYS3a_AVsvpbc0wllwrmh1uQOwjXgnRcO0tM_Dgp4kvLmEjWxOKRGgSVCn5pxjN4yrPjCqI5eBBdzkP9E4OYp-aNensYUIVJezSIzaDFfWwDwTKnnjdH47b2dT_aRmvbr35acilfShgv3pZVfb6IJLCgVi0ouSq7qxG79UjjNyYpqmfLdfNcAwV52h55nHx0r-StU0vzyQeieV7W4NZ3kPDrm8wRly6CrjGVbe5ueaMd7NIWXnJYJkD10Hifoz6PEuPgPKCn4iut0fS8VroKLvxIlP76Z9rCEmUdGXmbYR3dh-pHFxLUBITDIyHLBseNllKpYBvlYsn81dQcGe4MFOlLrXi5NVC30tA83RNpEhUduMMTuPGRohvq5_tICfCn7FBv4Sc1M36XiaJcELXcLCQJ4givz1GAIh6YyZUDqnHsieUEr6MGpn6IAJw12G_vSnlMhLYIOdRWDQN-csnU07btidaokG4WGzF2qs.jVdGhptTH9m-3asE5TkXCg@osdu-dev-888733619319.d.codeartifact.us-east-1.amazonaws.com/pypi/osdu-python/simple/ --extra-index-url https://community.opengroup.org/api/v4/projects/465/packages/pypi/simple/ adal==1.2.7 # via azure-datalake-store adlfs==2021.8.2 # via osdu-core-lib-python-azure -aiobotocore==1.4.0 +aiobotocore==1.4.1 # via s3fs aiohttp==3.7.4.post0 # via @@ -109,9 +109,7 @@ cloudpickle==1.6.0 # via # dask # distributed -colorama==0.4.4 - # via click -cryptography==3.4.7 +cryptography==3.4.8 # via # -r requirements.in # adal @@ -204,7 +202,7 @@ jsonpath-ng==1.5.3 # via -r requirements.in locket==0.2.1 # via partd -msal==1.13.0 +msal==1.14.0 # via # azure-identity # msal-extensions @@ -327,8 +325,6 @@ pytz==2021.1 # via # google-api-core # pandas - #pywin32==301 - # via portalocker pyyaml==5.4.1 # via # dask @@ -393,6 +389,7 @@ tornado==6.1 typing-extensions==3.10.0.0 # via # aiohttp + # aioredis # osdu-data-ecosystem-search # osdu-data-ecosystem-storage # pydantic -- GitLab From a45336079a9d8a27696f6f902e8b0d38ac9a171d Mon Sep 17 00:00:00 2001 From: Yunhua Koglin Date: Mon, 30 Aug 2021 09:25:01 -0500 Subject: [PATCH 14/36] remove --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index bf228c2e..1d49124e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,6 @@ # # pip-compile requirements.in # ---index-url https://aws:eyJ2ZXIiOjEsImlzdSI6MTYzMDExNDkxNCwiZW5jIjoiQTEyOEdDTSIsInRhZyI6InBnSGNPamhIWk8teVdfaFI3TnNuV0EiLCJleHAiOjE2MzAxNTgxMTQsImFsZyI6IkExMjhHQ01LVyIsIml2IjoiWTBScnZabWRPNlBfUnNSVSJ9.43R-J82u8v3qAYoFe1_-aQ.y_p2h4tIaXNR3B9E.46VoUr4wBFrK3liMOCzGCOSfuAKh1BPOcl_MJnyjIwjTEOalMimXF3HjjttkHC8eHD1Na7eFsUDo2FFf6wMB425cTDYMgJlGaaXJlzc-83ORTnh2v7Asactf5K40GbJ4XLLz0YOHu8BIPKJFoLfltDj4tbZdJIxwxski9BwzklElUirS-6w5QCzM5_UDA_tAqQQTWscLyaMK4DJS5deeN_jw7WovW6lM1AqpyxZY5hGvnjV25i4rgiItksutg1SMwCsQC-LU7YO2w-IFSiKFSlO_9h6YEzeCXcwptDPAaalnhSscZRXHZUs2G7Nyii7kI7VATArqpTVo2YYI85A7yM-tRIgfNvlXSn_FvEXE41egUs25zMAyXDhpaS4KMLSTGGRRSZBN6aoS5YOh-skjFP72fej99F00KovfRR_ETpyTh9xoZxyiJvZMFLKpSgo3dWCGt5pKLHQtrlJnEpghl2ecwCdjh5sasEEOKbGtn_6H7fDiixnhP-zMyeO8ajixepkCkIhoxnPQolt865Q_JLr6nFu9LDu3Hecl91BojXqkE-kacIdVb9zWpjuM1PtNBQn3P13fovgGX81LriVbnhGjJWYB3ec80rSsodVQNI8s-_UURb6qucb8tgUjJh10PYjs4hkiagEAdAi5119FnkHqB-O1VJ96FWLmw7nvsp7M8__SMmj_nYBVbEQipfIDBGzI0rFBNDnL3oJjLrWAUVA9Zy1EhCZ0P6l9zs9th968bbVD8CUzcefBHAKCrD9_oLnFCy5UQaIbrS827pcalqthwdmxZjRJ-hjlBB9CGdz_SZAgn3hypvMxEMsAuRKgt6q8HCHK6ZttzCbb_p3RfOMMT7hdm9rHQ538lfYS3a_AVsvpbc0wllwrmh1uQOwjXgnRcO0tM_Dgp4kvLmEjWxOKRGgSVCn5pxjN4yrPjCqI5eBBdzkP9E4OYp-aNensYUIVJezSIzaDFfWwDwTKnnjdH47b2dT_aRmvbr35acilfShgv3pZVfb6IJLCgVi0ouSq7qxG79UjjNyYpqmfLdfNcAwV52h55nHx0r-StU0vzyQeieV7W4NZ3kPDrm8wRly6CrjGVbe5ueaMd7NIWXnJYJkD10Hifoz6PEuPgPKCn4iut0fS8VroKLvxIlP76Z9rCEmUdGXmbYR3dh-pHFxLUBITDIyHLBseNllKpYBvlYsn81dQcGe4MFOlLrXi5NVC30tA83RNpEhUduMMTuPGRohvq5_tICfCn7FBv4Sc1M36XiaJcELXcLCQJ4givz1GAIh6YyZUDqnHsieUEr6MGpn6IAJw12G_vSnlMhLYIOdRWDQN-csnU07btidaokG4WGzF2qs.jVdGhptTH9m-3asE5TkXCg@osdu-dev-888733619319.d.codeartifact.us-east-1.amazonaws.com/pypi/osdu-python/simple/ --extra-index-url https://community.opengroup.org/api/v4/projects/465/packages/pypi/simple/ adal==1.2.7 -- GitLab From c53c61e009ddd66577039277733690db71d76743 Mon Sep 17 00:00:00 2001 From: Yunhua Koglin Date: Mon, 30 Aug 2021 15:52:47 -0500 Subject: [PATCH 15/36] update --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index bf228c2e..1d49124e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,6 @@ # # pip-compile requirements.in # ---index-url https://aws:eyJ2ZXIiOjEsImlzdSI6MTYzMDExNDkxNCwiZW5jIjoiQTEyOEdDTSIsInRhZyI6InBnSGNPamhIWk8teVdfaFI3TnNuV0EiLCJleHAiOjE2MzAxNTgxMTQsImFsZyI6IkExMjhHQ01LVyIsIml2IjoiWTBScnZabWRPNlBfUnNSVSJ9.43R-J82u8v3qAYoFe1_-aQ.y_p2h4tIaXNR3B9E.46VoUr4wBFrK3liMOCzGCOSfuAKh1BPOcl_MJnyjIwjTEOalMimXF3HjjttkHC8eHD1Na7eFsUDo2FFf6wMB425cTDYMgJlGaaXJlzc-83ORTnh2v7Asactf5K40GbJ4XLLz0YOHu8BIPKJFoLfltDj4tbZdJIxwxski9BwzklElUirS-6w5QCzM5_UDA_tAqQQTWscLyaMK4DJS5deeN_jw7WovW6lM1AqpyxZY5hGvnjV25i4rgiItksutg1SMwCsQC-LU7YO2w-IFSiKFSlO_9h6YEzeCXcwptDPAaalnhSscZRXHZUs2G7Nyii7kI7VATArqpTVo2YYI85A7yM-tRIgfNvlXSn_FvEXE41egUs25zMAyXDhpaS4KMLSTGGRRSZBN6aoS5YOh-skjFP72fej99F00KovfRR_ETpyTh9xoZxyiJvZMFLKpSgo3dWCGt5pKLHQtrlJnEpghl2ecwCdjh5sasEEOKbGtn_6H7fDiixnhP-zMyeO8ajixepkCkIhoxnPQolt865Q_JLr6nFu9LDu3Hecl91BojXqkE-kacIdVb9zWpjuM1PtNBQn3P13fovgGX81LriVbnhGjJWYB3ec80rSsodVQNI8s-_UURb6qucb8tgUjJh10PYjs4hkiagEAdAi5119FnkHqB-O1VJ96FWLmw7nvsp7M8__SMmj_nYBVbEQipfIDBGzI0rFBNDnL3oJjLrWAUVA9Zy1EhCZ0P6l9zs9th968bbVD8CUzcefBHAKCrD9_oLnFCy5UQaIbrS827pcalqthwdmxZjRJ-hjlBB9CGdz_SZAgn3hypvMxEMsAuRKgt6q8HCHK6ZttzCbb_p3RfOMMT7hdm9rHQ538lfYS3a_AVsvpbc0wllwrmh1uQOwjXgnRcO0tM_Dgp4kvLmEjWxOKRGgSVCn5pxjN4yrPjCqI5eBBdzkP9E4OYp-aNensYUIVJezSIzaDFfWwDwTKnnjdH47b2dT_aRmvbr35acilfShgv3pZVfb6IJLCgVi0ouSq7qxG79UjjNyYpqmfLdfNcAwV52h55nHx0r-StU0vzyQeieV7W4NZ3kPDrm8wRly6CrjGVbe5ueaMd7NIWXnJYJkD10Hifoz6PEuPgPKCn4iut0fS8VroKLvxIlP76Z9rCEmUdGXmbYR3dh-pHFxLUBITDIyHLBseNllKpYBvlYsn81dQcGe4MFOlLrXi5NVC30tA83RNpEhUduMMTuPGRohvq5_tICfCn7FBv4Sc1M36XiaJcELXcLCQJ4givz1GAIh6YyZUDqnHsieUEr6MGpn6IAJw12G_vSnlMhLYIOdRWDQN-csnU07btidaokG4WGzF2qs.jVdGhptTH9m-3asE5TkXCg@osdu-dev-888733619319.d.codeartifact.us-east-1.amazonaws.com/pypi/osdu-python/simple/ --extra-index-url https://community.opengroup.org/api/v4/projects/465/packages/pypi/simple/ adal==1.2.7 -- GitLab From 9dde901f613cab70b65ecf1c45f0810add1417f1 Mon Sep 17 00:00:00 2001 From: Yunhua Koglin Date: Tue, 31 Aug 2021 10:56:24 -0500 Subject: [PATCH 16/36] test requirements --- app/bulk_persistence/tenant_provider.py | 12 +- app/conf.py | 14 +- app/injector/ibm_injector.py | 33 -- app/injector/main_injector.py | 8 +- requirements.in | 9 +- requirements.txt | 443 ++---------------------- tests/aws-test/build-aws/run-tests.sh | 3 - 7 files changed, 56 insertions(+), 466 deletions(-) delete mode 100644 app/injector/ibm_injector.py diff --git a/app/bulk_persistence/tenant_provider.py b/app/bulk_persistence/tenant_provider.py index 560a48c0..0320c8a3 100644 --- a/app/bulk_persistence/tenant_provider.py +++ b/app/bulk_persistence/tenant_provider.py @@ -32,12 +32,12 @@ async def resolve_tenant(data_partition_id: str) -> Tenant: bucket_name='wdms-osdu' ) - if Config.cloud_provider.value == 'ibm': - return Tenant( - data_partition_id=data_partition_id, - project_id=Config.default_data_tenant_project_id.value, - bucket_name='logstore-osdu-ibm' - ) + # if Config.cloud_provider.value == 'ibm': + # return Tenant( + # data_partition_id=data_partition_id, + # project_id=Config.default_data_tenant_project_id.value, + # bucket_name='logstore-osdu-ibm' + # ) if Config.cloud_provider.value == 'aws': return Tenant( diff --git a/app/conf.py b/app/conf.py index a43219bb..3c34bb1f 100644 --- a/app/conf.py +++ b/app/conf.py @@ -318,13 +318,13 @@ def cloud_provider_additional_environment(config: ConfigurationContainer): override=True, validator=validator_path_must_exist) - if provider == 'ibm': - config.add_from_env(attribute_name='default_data_tenant_project_id', - env_var_key='OS_WELLBORE_DDMS_DATA_PROJECT_ID', - description='IBM data tenant ID', - default='logstore-ibm', - is_mandatory=True, - override=True) + # if provider == 'ibm': + # config.add_from_env(attribute_name='default_data_tenant_project_id', + # env_var_key='OS_WELLBORE_DDMS_DATA_PROJECT_ID', + # description='IBM data tenant ID', + # default='logstore-ibm', + # is_mandatory=True, + # override=True) if provider == 'aws': config.add_from_env(attribute_name='aws_region', env_var_key='AWS_REGION', diff --git a/app/injector/ibm_injector.py b/app/injector/ibm_injector.py deleted file mode 100644 index 72a6ab92..00000000 --- a/app/injector/ibm_injector.py +++ /dev/null @@ -1,33 +0,0 @@ -from osdu.core.api.storage.blob_storage_base import BlobStorageBase -from osdu_ibm.storage.blob_storage_ibm import IBMObjectStorage - -from app.utils import get_http_client_session -from app.utils import Context -from .app_injector import AppInjector, AppInjectorModule -from app.bulk_persistence import resolve_tenant -from app.bulk_persistence.dask.dask_bulk_storage import DaskBulkStorage -from osdu_ibm.storage.dask_storage_parameters import get_dask_storage_parameters as ibm_parameters - - -class IBMInjector(AppInjectorModule): - def configure(self, app_injector: AppInjector): - app_injector.register(BlobStorageBase, IBMInjector.build_ibm_blob_storage) - app_injector.register(DaskBulkStorage, IBMInjector.build_ibm_dask_blob_storage) - - @staticmethod - async def build_ibm_blob_storage(*args, **kwargs) -> BlobStorageBase: - ctx: Context = Context.current() - # TODO to be reviewed - tenant = await resolve_tenant(ctx.partition_id) - return IBMObjectStorage( - session=get_http_client_session(), - service_account_file=tenant.credentials - ) - - @staticmethod - async def build_ibm_dask_blob_storage() -> DaskBulkStorage: - # raise NotImplementedError() - ctx: Context = Context.current() - tenant = await resolve_tenant(ctx.partition_id) - params = await ibm_parameters(tenant) - return await DaskBulkStorage.create(params) diff --git a/app/injector/main_injector.py b/app/injector/main_injector.py index 85170d0d..c2d25936 100644 --- a/app/injector/main_injector.py +++ b/app/injector/main_injector.py @@ -23,7 +23,7 @@ from .app_injector import AppInjector, AppInjectorModule, WithLifeTime from app.injector.az_injector import AzureInjector from app.injector.aws_injector import AwsInjector from app.injector.gcp_injector import GCPInjector -from app.injector.ibm_injector import IBMInjector +# from app.injector.ibm_injector import IBMInjector from app.clients import StorageRecordServiceClient from app.clients.storage_service_blob_storage import StorageRecordServiceBlobStorage @@ -64,9 +64,9 @@ class MainInjector(AppInjectorModule): logger.info('using gcp injector') GCPInjector().configure(app_injector) - if Config.cloud_provider.value == 'ibm': - logger.info('using ibm injector') - IBMInjector().configure(app_injector) + # if Config.cloud_provider.value == 'ibm': + # logger.info('using ibm injector') + # IBMInjector().configure(app_injector) if Config.cloud_provider.value == 'aws': logger.info('using aws injector') diff --git a/requirements.in b/requirements.in index e810da1c..18a6d25c 100644 --- a/requirements.in +++ b/requirements.in @@ -23,9 +23,7 @@ opencensus-ext-logging dask[distributed]==2021.7.2 fsspec python-ulid -boto3==1.17.106 -botocore==1.20.106 -s3fs==2021.8.0 + --extra-index-url \ https://community.opengroup.org/api/v4/projects/465/packages/pypi/simple/ @@ -34,8 +32,9 @@ osdu-log-recognition-lib>=0.0.9 osdu-data-ecosystem-storage~=1.2.0 osdu-data-ecosystem-search>=0.3.2, <0.4 -osdu-core-lib-python-ibm~=1.2.0 osdu-core-lib-python-gcp~=1.1.0 osdu-core-lib-python-azure~=1.2.1 osdu-core-lib-python-aws>=0.0.2, <0.1 -osdu-core-lib-python~=1.1.0 \ No newline at end of file +osdu-core-lib-python~=1.1.0 +boto3 +s3fs \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 1d49124e..f891ccb9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,411 +1,38 @@ -# -# This file is autogenerated by pip-compile with python 3.9 -# To update, run: -# -# pip-compile requirements.in -# ---extra-index-url https://community.opengroup.org/api/v4/projects/465/packages/pypi/simple/ - -adal==1.2.7 - # via azure-datalake-store -adlfs==2021.8.2 - # via osdu-core-lib-python-azure -aiobotocore==1.4.1 - # via s3fs +fastapi>=0.65.1 aiohttp==3.7.4.post0 - # via - # -r requirements.in - # adlfs - # aiobotocore - # gcsfs - # osdu-core-lib-python - # osdu-core-lib-python-gcp -aioitertools==0.8.0 - # via aiobotocore -aioredis==2.0.0 - # via osdu-core-lib-python -anyio==3.3.0 - # via httpcore -asgiref==3.4.1 - # via uvicorn -async-timeout==3.0.1 - # via - # aiohttp - # aioredis -attrs==21.2.0 - # via - # aiohttp - # osdu-core-lib-python -azure-common==1.1.27 - # via - # azure-keyvault-certificates - # azure-keyvault-keys - # azure-keyvault-secrets -azure-core==1.17.0 - # via - # adlfs - # azure-identity - # azure-keyvault-certificates - # azure-keyvault-keys - # azure-keyvault-secrets - # azure-storage-blob -azure-datalake-store==0.0.52 - # via adlfs -azure-identity==1.6.1 - # via - # adlfs - # osdu-core-lib-python-azure -azure-keyvault==4.1.0 - # via osdu-core-lib-python-azure -azure-keyvault-certificates==4.3.0 - # via azure-keyvault -azure-keyvault-keys==4.4.0 - # via azure-keyvault -azure-keyvault-secrets==4.3.0 - # via azure-keyvault -azure-storage-blob==12.8.1 - # via - # adlfs - # osdu-core-lib-python-azure -backoff==1.11.1 - # via -r requirements.in -boto3==1.17.106 - # via - # -r requirements.in - # osdu-core-lib-python-ibm -botocore==1.20.106 - # via - # -r requirements.in - # aiobotocore - # boto3 - # osdu-core-lib-python-ibm - # s3transfer -cachetools==4.2.2 - # via - # google-auth - # osdu-core-lib-python-azure -certifi==2021.5.30 - # via - # httpx - # msrest - # requests -cffi==1.14.6 - # via - # azure-datalake-store - # cryptography -chardet==4.0.0 - # via aiohttp -charset-normalizer==2.0.4 - # via - # httpx - # requests -click==8.0.1 - # via - # -r requirements.in - # distributed - # uvicorn -cloudpickle==1.6.0 - # via - # dask - # distributed -cryptography==3.4.8 - # via - # -r requirements.in - # adal - # azure-identity - # azure-keyvault-keys - # azure-storage-blob - # msal - # osdu-core-lib-python - # osdu-core-lib-python-gcp - # pyjwt -dask[distributed]==2021.7.2 - # via - # -r requirements.in - # distributed -decorator==5.0.9 - # via - # gcsfs - # jsonpath-ng -distributed==2021.07.2 - # via dask -fastapi==0.68.1 - # via - # -r requirements.in - # osdu-data-ecosystem-search - # osdu-data-ecosystem-storage -fsspec==2021.7.0 - # via - # -r requirements.in - # adlfs - # dask - # gcsfs - # osdu-core-lib-python-azure - # s3fs -gcsfs==2021.7.0 - # via osdu-core-lib-python-gcp -google-api-core[grpc]==1.31.2 - # via - # google-cloud-core - # google-cloud-monitoring - # google-cloud-trace - # opencensus -google-auth==1.35.0 - # via - # gcsfs - # google-api-core - # google-auth-oauthlib - # google-cloud-core -google-auth-oauthlib==0.4.5 - # via gcsfs -google-cloud-core==1.7.2 - # via google-cloud-trace -google-cloud-monitoring==2.4.2 - # via opencensus-ext-stackdriver -google-cloud-trace==0.24.0 - # via opencensus-ext-stackdriver -googleapis-common-protos==1.53.0 - # via google-api-core -grpcio==1.39.0 - # via - # google-api-core - # opencensus-ext-ocagent - # opencensus-proto -h11==0.12.0 - # via - # httpcore - # uvicorn -heapdict==1.0.1 - # via zict -httpcore==0.13.6 - # via httpx -httpx==0.19.0 - # via - # osdu-data-ecosystem-search - # osdu-data-ecosystem-storage -idna==3.2 - # via - # anyio - # requests - # rfc3986 - # yarl -isodate==0.6.0 - # via msrest -jmespath==0.10.0 - # via - # boto3 - # botocore -jsonbender==0.9.3 - # via -r requirements.in -jsonpath-ng==1.5.3 - # via -r requirements.in -locket==0.2.1 - # via partd -msal==1.14.0 - # via - # azure-identity - # msal-extensions -msal-extensions==0.3.0 - # via azure-identity -msgpack==1.0.2 - # via distributed -msrest==0.6.21 - # via - # azure-keyvault-certificates - # azure-keyvault-keys - # azure-keyvault-secrets - # azure-storage-blob -multidict==5.1.0 - # via - # aiohttp - # yarl -natsort==7.1.1 - # via -r requirements.in -numpy==1.21.2 - # via - # pandas - # pyarrow -oauthlib==3.1.1 - # via requests-oauthlib -opencensus==0.7.13 - # via - # -r requirements.in - # opencensus-ext-azure - # opencensus-ext-logging - # opencensus-ext-ocagent - # opencensus-ext-stackdriver -opencensus-context==0.1.2 - # via opencensus -opencensus-ext-azure==1.0.8 - # via -r requirements.in -opencensus-ext-logging==0.1.0 - # via -r requirements.in -opencensus-ext-ocagent==0.7.1 - # via -r requirements.in -opencensus-ext-stackdriver==0.8.0 - # via -r requirements.in -opencensus-proto==0.1.0 - # via opencensus-ext-ocagent -osdu-core-lib-python==1.1.0 - # via -r requirements.in -osdu-core-lib-python-aws==0.0.2 - # via -r requirements.in -osdu-core-lib-python-azure==1.2.1 - # via -r requirements.in -osdu-core-lib-python-gcp==1.1.0 - # via -r requirements.in -osdu-core-lib-python-ibm==1.2.0 - # via -r requirements.in -osdu-data-ecosystem-search==0.3.3 - # via -r requirements.in -osdu-data-ecosystem-storage==1.2.0 - # via -r requirements.in -osdu-log-recognition-lib==0.0.9 - # via -r requirements.in -packaging==21.0 - # via - # dask - # google-api-core - # google-cloud-monitoring +cryptography>=2.7 +pyarrow>=0.15.0 pandas==1.2.4 - # via -r requirements.in -partd==1.2.0 - # via dask -ply==3.11 - # via jsonpath-ng -portalocker==1.7.1 - # via msal-extensions -proto-plus==1.19.0 - # via google-cloud-monitoring -protobuf==3.17.3 - # via - # google-api-core - # googleapis-common-protos - # proto-plus -psutil==5.8.0 - # via - # distributed - # opencensus-ext-azure -pyarrow==4.0.1 - # via -r requirements.in -pyasn1==0.4.8 - # via - # pyasn1-modules - # rsa -pyasn1-modules==0.2.8 - # via google-auth -pycparser==2.20 - # via cffi -pydantic==1.8.2 - # via - # fastapi - # osdu-data-ecosystem-search - # osdu-data-ecosystem-storage -pyjwt[crypto]==2.1.0 - # via - # adal - # msal - # osdu-core-lib-python - # osdu-core-lib-python-gcp -pyparsing==2.4.7 - # via packaging -python-dateutil==2.8.2 - # via - # adal - # botocore - # pandas -python-multipart==0.0.5 - # via -r requirements.in -python-rapidjson==1.4 - # via -r requirements.in -python-ulid==1.0.3 - # via -r requirements.in -pytz==2021.1 - # via - # google-api-core - # pandas -pyyaml==5.4.1 - # via - # dask - # distributed -requests==2.26.0 - # via - # adal - # azure-core - # azure-datalake-store - # gcsfs - # google-api-core - # msal - # msrest - # opencensus-ext-azure - # requests-oauthlib -requests-oauthlib==1.3.0 - # via - # google-auth-oauthlib - # msrest -rfc3986[idna2008]==1.5.0 - # via httpx -rsa==4.7.2 - # via google-auth -s3fs==2021.8.0 - # via -r requirements.in -s3transfer==0.4.2 - # via boto3 -six==1.16.0 - # via - # azure-core - # azure-identity - # azure-keyvault-keys - # google-api-core - # google-auth - # google-cloud-core - # grpcio - # isodate - # jsonpath-ng - # protobuf - # python-dateutil - # python-multipart -sniffio==1.2.0 - # via - # anyio - # httpcore - # httpx -sortedcontainers==2.4.0 - # via distributed -starlette==0.14.2 - # via fastapi -structlog==21.1.0 - # via -r requirements.in -tblib==1.7.0 - # via distributed -toolz==0.11.1 - # via - # dask - # distributed - # partd -tornado==6.1 - # via distributed -typing-extensions==3.10.0.0 - # via - # aiohttp - # aioredis - # osdu-data-ecosystem-search - # osdu-data-ecosystem-storage - # pydantic -urllib3==1.26.6 - # via - # botocore - # requests -uvicorn==0.15.0 - # via -r requirements.in -wrapt==1.12.1 - # via aiobotocore -xmltodict==0.11.0 - # via osdu-log-recognition-lib -yarl==1.6.3 - # via aiohttp -zict==2.0.0 - # via distributed +uvicorn +Click +backoff +structlog +python-rapidjson +python-multipart +jsonpath-ng # TODO: remove when bulk_path no longer supported +jsonbender==0.9.3 + +opencensus +opencensus-ext-stackdriver +opencensus-ext-azure +opencensus-ext-ocagent +opencensus-ext-logging + +# for chunking feature +dask[distributed]==2021.4.1 +fsspec + +--extra-index-url \ +https://community.opengroup.org/api/v4/projects/465/packages/pypi/simple/ + +osdu-log-recognition-lib>=0.0.9 + +osdu-data-ecosystem-storage~=1.2.0 +osdu-data-ecosystem-search>=0.3.2, <0.4 -# The following packages are considered to be unsafe in a requirements file: -# setuptools +osdu-core-lib-python-ibm>=0.0.1, <0.1 +osdu-core-lib-python-gcp~=1.1.0 +osdu-core-lib-python-azure~=1.1.1 +osdu-core-lib-python-aws>=0.0.1, <0.1 +osdu-core-lib-python~=1.1.0 +s3fs diff --git a/tests/aws-test/build-aws/run-tests.sh b/tests/aws-test/build-aws/run-tests.sh index d33b74ad..cc370c56 100644 --- a/tests/aws-test/build-aws/run-tests.sh +++ b/tests/aws-test/build-aws/run-tests.sh @@ -42,11 +42,8 @@ source env/bin/activate python3 -m pip install -r ./aws-test/build-aws/requirements.txt rm -rf test-reports/ mkdir test-reports -echo $(pwd) -echo 'cd to integration\n' cd integration -echo $(pwd) acl_domain='example.com' legal_tag='opendes-sdmstestlegaltag' -- GitLab From 756b58f9de6dcd4f4cd1e2f86dc117a8f6882d2c Mon Sep 17 00:00:00 2001 From: Yunhua Koglin Date: Tue, 31 Aug 2021 11:08:41 -0500 Subject: [PATCH 17/36] update requirements --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f891ccb9..c10fb6b3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ python-rapidjson python-multipart jsonpath-ng # TODO: remove when bulk_path no longer supported jsonbender==0.9.3 - +python-ulid opencensus opencensus-ext-stackdriver opencensus-ext-azure -- GitLab From b06458c5757e45ebba5373c69a8c3f4157630c04 Mon Sep 17 00:00:00 2001 From: Yunhua Koglin Date: Tue, 31 Aug 2021 11:18:02 -0500 Subject: [PATCH 18/36] update requirements --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c10fb6b3..34c4f823 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,7 +21,7 @@ opencensus-ext-logging # for chunking feature dask[distributed]==2021.4.1 fsspec - +natsort==7.1.1 --extra-index-url \ https://community.opengroup.org/api/v4/projects/465/packages/pypi/simple/ -- GitLab From 3cf06f61df70e8461a0e4c4d8e878c4290ed8ab8 Mon Sep 17 00:00:00 2001 From: Yunhua Koglin Date: Tue, 31 Aug 2021 13:32:58 -0500 Subject: [PATCH 19/36] update python version --- provider/os-wellbore-ddms-aws/build-aws/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/provider/os-wellbore-ddms-aws/build-aws/Dockerfile b/provider/os-wellbore-ddms-aws/build-aws/Dockerfile index 36724538..e017e4c7 100644 --- a/provider/os-wellbore-ddms-aws/build-aws/Dockerfile +++ b/provider/os-wellbore-ddms-aws/build-aws/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM python:3.7-slim-buster +FROM python:3.9-slim-buster COPY requirements.txt ./ -- GitLab From cdb4c390124cb55908832e74ca96fa54aea6e0b3 Mon Sep 17 00:00:00 2001 From: Yunhua Koglin Date: Tue, 31 Aug 2021 14:24:59 -0500 Subject: [PATCH 20/36] update spec --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 34c4f823..08c50505 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,7 +20,7 @@ opencensus-ext-logging # for chunking feature dask[distributed]==2021.4.1 -fsspec +fsspec==2021.7.0 natsort==7.1.1 --extra-index-url \ https://community.opengroup.org/api/v4/projects/465/packages/pypi/simple/ @@ -35,4 +35,4 @@ osdu-core-lib-python-gcp~=1.1.0 osdu-core-lib-python-azure~=1.1.1 osdu-core-lib-python-aws>=0.0.1, <0.1 osdu-core-lib-python~=1.1.0 -s3fs +s3fs==2021.8.0 -- GitLab From b9161ceeca5a7287b0a2cb74d87a95ac495ef698 Mon Sep 17 00:00:00 2001 From: Yunhua Koglin Date: Tue, 31 Aug 2021 15:43:34 -0500 Subject: [PATCH 21/36] update --- app/bulk_persistence/dask/dask_bulk_storage.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/bulk_persistence/dask/dask_bulk_storage.py b/app/bulk_persistence/dask/dask_bulk_storage.py index 8109bead..9ec88a18 100644 --- a/app/bulk_persistence/dask/dask_bulk_storage.py +++ b/app/bulk_persistence/dask/dask_bulk_storage.py @@ -201,6 +201,7 @@ class DaskBulkStorage: we should be able to change or support other format easily ? schema={} instead of 'infer' fixes wrong inference for columns of type string starting with nan values """ + get_logger().info(f"_save_with_dask, path : {path}") return self._submit_with_trace(dd.to_parquet, ddf, path, schema={}, engine='pyarrow', -- GitLab From 3f42b32b114824abfe8e63df020a42dd8c95c9c7 Mon Sep 17 00:00:00 2001 From: Yunhua Koglin Date: Wed, 1 Sep 2021 09:01:50 -0500 Subject: [PATCH 22/36] update policy --- .../os-wellbore-ddms-aws/build-aws/Dockerfile | 3 ++- .../build-aws/buildspec.yaml | 17 ++++++++++++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/provider/os-wellbore-ddms-aws/build-aws/Dockerfile b/provider/os-wellbore-ddms-aws/build-aws/Dockerfile index e017e4c7..df95cfae 100644 --- a/provider/os-wellbore-ddms-aws/build-aws/Dockerfile +++ b/provider/os-wellbore-ddms-aws/build-aws/Dockerfile @@ -15,8 +15,9 @@ FROM python:3.9-slim-buster COPY requirements.txt ./ - +COPY osdu-core-lib-python-aws-1.0.2.tar.gz ./ RUN pip install -r requirements.txt +RUN pip install ./osdu-core-lib-python-aws-1.0.2.tar.gz COPY ./app /app COPY provider/os-wellbore-ddms-aws/build-aws/entrypoint.sh . diff --git a/provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml b/provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml index c80cc448..eebde627 100644 --- a/provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml +++ b/provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml @@ -56,11 +56,22 @@ phases: - chmod +x ./tests/aws-test/build-aws/prepare-dist.sh - ./tests/aws-test/build-aws/prepare-dist.sh + - aws codeartifact login --tool pip --domain osdu-dev --domain-owner $AWS_ACCOUNT_ID --repository osdu-python --region us-east-1 + - | + aws codeartifact get-package-version-asset \ + --domain osdu-dev \ + --domain-owner $AWS_ACCOUNT_ID \ + --repository osdu-python \ + --region us-east-1 \ + --format pypi \ + --package osdu-core-lib-python-aws \ + --package-version 1.0.2 \ + --asset osdu-core-lib-python-aws-1.0.2.tar.gz \ + ./osdu-core-lib-python-aws-1.0.2.tar.gz + - aws codeartifact login --tool pip --domain osdu-dev --domain-owner $AWS_ACCOUNT_ID --repository pypi-store - - pip config set global.extra-index-url https://community.opengroup.org/api/v4/projects/465/packages/pypi/simple/ - - pip install -r ./requirements.txt - - pip install -r ./requirements_dev.txt + - pip config set global.extra-index-url https://community.opengroup.org/api/v4/projects/465/packages/pypi/simple/ # using jar instead of executable since exe requires 32 bit image and doesn't work on codebuild 4.0 - | -- GitLab From a26cf5bbf94ae03355d6ef154e58ed140ecab7d9 Mon Sep 17 00:00:00 2001 From: Yunhua Koglin Date: Wed, 1 Sep 2021 09:11:19 -0500 Subject: [PATCH 23/36] update spec --- provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml b/provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml index eebde627..d23e0e29 100644 --- a/provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml +++ b/provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml @@ -72,7 +72,8 @@ phases: - aws codeartifact login --tool pip --domain osdu-dev --domain-owner $AWS_ACCOUNT_ID --repository pypi-store - pip config set global.extra-index-url https://community.opengroup.org/api/v4/projects/465/packages/pypi/simple/ - + - pip install -r ./requirements.txt + - pip install -r ./requirements_dev.txt # using jar instead of executable since exe requires 32 bit image and doesn't work on codebuild 4.0 - | java -jar sonar-scanner-4.6.2.2472/lib/sonar-scanner-cli-4.6.2.2472.jar -Dsonar.projectKey=wellbore-ddms -Dsonar.host.url=${SONAR_URL} \ -- GitLab From f9e698dd9b096e5b709dee63d61eca8e9365cf99 Mon Sep 17 00:00:00 2001 From: Yunhua Koglin Date: Wed, 1 Sep 2021 09:22:08 -0500 Subject: [PATCH 24/36] update --- provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml b/provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml index d23e0e29..7006ad32 100644 --- a/provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml +++ b/provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml @@ -72,15 +72,12 @@ phases: - aws codeartifact login --tool pip --domain osdu-dev --domain-owner $AWS_ACCOUNT_ID --repository pypi-store - pip config set global.extra-index-url https://community.opengroup.org/api/v4/projects/465/packages/pypi/simple/ - - pip install -r ./requirements.txt - - pip install -r ./requirements_dev.txt # using jar instead of executable since exe requires 32 bit image and doesn't work on codebuild 4.0 - | java -jar sonar-scanner-4.6.2.2472/lib/sonar-scanner-cli-4.6.2.2472.jar -Dsonar.projectKey=wellbore-ddms -Dsonar.host.url=${SONAR_URL} \ -Dsonar.login=${SONAR_USERNAME} -Dsonar.password=${SONAR_PASSWORD} -Dsonar.branch.name=${BRANCH_NAME} - - safety check -r requirements.txt -r requirements_dev.txt -o safety_check.txt; EXITCODE=$? - + - echo "Logging into Docker Hub..." - docker login -u ${DOCKER_USERNAME} -p ${DOCKER_PASSWORD} @@ -90,7 +87,9 @@ phases: - echo "Pushing docker image..." - docker push ${ECR_IMAGE} - docker push ${ECR_IMAGE_BRANCH_LATEST} - + + - pip install -r ./requirements.txt + - pip install -r ./requirements_dev.txt - python -m pytest --junit-xml=unit_tests_report.xml --cov=app --cov-report=html --cov-report=xml ./tests/unit - echo "Generate build-info.json" -- GitLab From ffb3e03fdf92daf4f70c89c96c2429c77767922f Mon Sep 17 00:00:00 2001 From: Yunhua Koglin Date: Wed, 1 Sep 2021 09:24:11 -0500 Subject: [PATCH 25/36] update --- provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml b/provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml index 7006ad32..7185491c 100644 --- a/provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml +++ b/provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml @@ -87,7 +87,7 @@ phases: - echo "Pushing docker image..." - docker push ${ECR_IMAGE} - docker push ${ECR_IMAGE_BRANCH_LATEST} - + - pip install -r ./requirements.txt - pip install -r ./requirements_dev.txt - python -m pytest --junit-xml=unit_tests_report.xml --cov=app --cov-report=html --cov-report=xml ./tests/unit -- GitLab From d70266831d407373a10282fa81d590e0c6b51bbf Mon Sep 17 00:00:00 2001 From: Yunhua Koglin Date: Wed, 1 Sep 2021 09:28:57 -0500 Subject: [PATCH 26/36] update --- provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml b/provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml index 7185491c..b0e83e20 100644 --- a/provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml +++ b/provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml @@ -89,7 +89,7 @@ phases: - docker push ${ECR_IMAGE_BRANCH_LATEST} - pip install -r ./requirements.txt - - pip install -r ./requirements_dev.txt + - pip install -r ./requirements_dev.txt - python -m pytest --junit-xml=unit_tests_report.xml --cov=app --cov-report=html --cov-report=xml ./tests/unit - echo "Generate build-info.json" -- GitLab From d27774804c5a563ab11cc3c401f728670b32635a Mon Sep 17 00:00:00 2001 From: Yunhua Koglin Date: Wed, 1 Sep 2021 11:10:33 -0500 Subject: [PATCH 27/36] update --- provider/os-wellbore-ddms-aws/build-aws/Dockerfile | 4 ++-- provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/provider/os-wellbore-ddms-aws/build-aws/Dockerfile b/provider/os-wellbore-ddms-aws/build-aws/Dockerfile index df95cfae..0873b10b 100644 --- a/provider/os-wellbore-ddms-aws/build-aws/Dockerfile +++ b/provider/os-wellbore-ddms-aws/build-aws/Dockerfile @@ -15,9 +15,9 @@ FROM python:3.9-slim-buster COPY requirements.txt ./ -COPY osdu-core-lib-python-aws-1.0.2.tar.gz ./ +COPY osdu-core-lib-python-aws-2.0.2.tar.gz ./ RUN pip install -r requirements.txt -RUN pip install ./osdu-core-lib-python-aws-1.0.2.tar.gz +RUN pip install ./osdu-core-lib-python-aws-2.0.2.tar.gz COPY ./app /app COPY provider/os-wellbore-ddms-aws/build-aws/entrypoint.sh . diff --git a/provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml b/provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml index b0e83e20..ec7de53f 100644 --- a/provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml +++ b/provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml @@ -65,9 +65,9 @@ phases: --region us-east-1 \ --format pypi \ --package osdu-core-lib-python-aws \ - --package-version 1.0.2 \ - --asset osdu-core-lib-python-aws-1.0.2.tar.gz \ - ./osdu-core-lib-python-aws-1.0.2.tar.gz + --package-version 2.0.2 \ + --asset osdu-core-lib-python-aws-2.0.2.tar.gz \ + ./osdu-core-lib-python-aws-2.0.2.tar.gz - aws codeartifact login --tool pip --domain osdu-dev --domain-owner $AWS_ACCOUNT_ID --repository pypi-store -- GitLab From dddf3b2e9434366720efc9f44e0d68f21f145479 Mon Sep 17 00:00:00 2001 From: Yunhua Koglin Date: Wed, 1 Sep 2021 12:32:54 -0500 Subject: [PATCH 28/36] put ibm back --- .../dask/dask_bulk_storage.py | 1 - app/bulk_persistence/tenant_provider.py | 12 +- app/conf.py | 14 +- app/injector/ibm_injector.py | 33 ++ app/injector/main_injector.py | 8 +- requirements.in | 5 +- requirements.txt | 424 ++++++++++++++++-- 7 files changed, 442 insertions(+), 55 deletions(-) create mode 100644 app/injector/ibm_injector.py diff --git a/app/bulk_persistence/dask/dask_bulk_storage.py b/app/bulk_persistence/dask/dask_bulk_storage.py index 9ec88a18..8109bead 100644 --- a/app/bulk_persistence/dask/dask_bulk_storage.py +++ b/app/bulk_persistence/dask/dask_bulk_storage.py @@ -201,7 +201,6 @@ class DaskBulkStorage: we should be able to change or support other format easily ? schema={} instead of 'infer' fixes wrong inference for columns of type string starting with nan values """ - get_logger().info(f"_save_with_dask, path : {path}") return self._submit_with_trace(dd.to_parquet, ddf, path, schema={}, engine='pyarrow', diff --git a/app/bulk_persistence/tenant_provider.py b/app/bulk_persistence/tenant_provider.py index 0320c8a3..560a48c0 100644 --- a/app/bulk_persistence/tenant_provider.py +++ b/app/bulk_persistence/tenant_provider.py @@ -32,12 +32,12 @@ async def resolve_tenant(data_partition_id: str) -> Tenant: bucket_name='wdms-osdu' ) - # if Config.cloud_provider.value == 'ibm': - # return Tenant( - # data_partition_id=data_partition_id, - # project_id=Config.default_data_tenant_project_id.value, - # bucket_name='logstore-osdu-ibm' - # ) + if Config.cloud_provider.value == 'ibm': + return Tenant( + data_partition_id=data_partition_id, + project_id=Config.default_data_tenant_project_id.value, + bucket_name='logstore-osdu-ibm' + ) if Config.cloud_provider.value == 'aws': return Tenant( diff --git a/app/conf.py b/app/conf.py index 3c34bb1f..a43219bb 100644 --- a/app/conf.py +++ b/app/conf.py @@ -318,13 +318,13 @@ def cloud_provider_additional_environment(config: ConfigurationContainer): override=True, validator=validator_path_must_exist) - # if provider == 'ibm': - # config.add_from_env(attribute_name='default_data_tenant_project_id', - # env_var_key='OS_WELLBORE_DDMS_DATA_PROJECT_ID', - # description='IBM data tenant ID', - # default='logstore-ibm', - # is_mandatory=True, - # override=True) + if provider == 'ibm': + config.add_from_env(attribute_name='default_data_tenant_project_id', + env_var_key='OS_WELLBORE_DDMS_DATA_PROJECT_ID', + description='IBM data tenant ID', + default='logstore-ibm', + is_mandatory=True, + override=True) if provider == 'aws': config.add_from_env(attribute_name='aws_region', env_var_key='AWS_REGION', diff --git a/app/injector/ibm_injector.py b/app/injector/ibm_injector.py new file mode 100644 index 00000000..72a6ab92 --- /dev/null +++ b/app/injector/ibm_injector.py @@ -0,0 +1,33 @@ +from osdu.core.api.storage.blob_storage_base import BlobStorageBase +from osdu_ibm.storage.blob_storage_ibm import IBMObjectStorage + +from app.utils import get_http_client_session +from app.utils import Context +from .app_injector import AppInjector, AppInjectorModule +from app.bulk_persistence import resolve_tenant +from app.bulk_persistence.dask.dask_bulk_storage import DaskBulkStorage +from osdu_ibm.storage.dask_storage_parameters import get_dask_storage_parameters as ibm_parameters + + +class IBMInjector(AppInjectorModule): + def configure(self, app_injector: AppInjector): + app_injector.register(BlobStorageBase, IBMInjector.build_ibm_blob_storage) + app_injector.register(DaskBulkStorage, IBMInjector.build_ibm_dask_blob_storage) + + @staticmethod + async def build_ibm_blob_storage(*args, **kwargs) -> BlobStorageBase: + ctx: Context = Context.current() + # TODO to be reviewed + tenant = await resolve_tenant(ctx.partition_id) + return IBMObjectStorage( + session=get_http_client_session(), + service_account_file=tenant.credentials + ) + + @staticmethod + async def build_ibm_dask_blob_storage() -> DaskBulkStorage: + # raise NotImplementedError() + ctx: Context = Context.current() + tenant = await resolve_tenant(ctx.partition_id) + params = await ibm_parameters(tenant) + return await DaskBulkStorage.create(params) diff --git a/app/injector/main_injector.py b/app/injector/main_injector.py index c2d25936..85170d0d 100644 --- a/app/injector/main_injector.py +++ b/app/injector/main_injector.py @@ -23,7 +23,7 @@ from .app_injector import AppInjector, AppInjectorModule, WithLifeTime from app.injector.az_injector import AzureInjector from app.injector.aws_injector import AwsInjector from app.injector.gcp_injector import GCPInjector -# from app.injector.ibm_injector import IBMInjector +from app.injector.ibm_injector import IBMInjector from app.clients import StorageRecordServiceClient from app.clients.storage_service_blob_storage import StorageRecordServiceBlobStorage @@ -64,9 +64,9 @@ class MainInjector(AppInjectorModule): logger.info('using gcp injector') GCPInjector().configure(app_injector) - # if Config.cloud_provider.value == 'ibm': - # logger.info('using ibm injector') - # IBMInjector().configure(app_injector) + if Config.cloud_provider.value == 'ibm': + logger.info('using ibm injector') + IBMInjector().configure(app_injector) if Config.cloud_provider.value == 'aws': logger.info('using aws injector') diff --git a/requirements.in b/requirements.in index 18a6d25c..6b7fcbc6 100644 --- a/requirements.in +++ b/requirements.in @@ -32,9 +32,8 @@ osdu-log-recognition-lib>=0.0.9 osdu-data-ecosystem-storage~=1.2.0 osdu-data-ecosystem-search>=0.3.2, <0.4 +osdu-core-lib-python-ibm~=1.2.0 osdu-core-lib-python-gcp~=1.1.0 osdu-core-lib-python-azure~=1.2.1 -osdu-core-lib-python-aws>=0.0.2, <0.1 +osdu-core-lib-python-aws>=0.0.1, <0.1 osdu-core-lib-python~=1.1.0 -boto3 -s3fs \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 08c50505..23f06719 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,38 +1,394 @@ -fastapi>=0.65.1 -aiohttp==3.7.4.post0 -cryptography>=2.7 -pyarrow>=0.15.0 -pandas==1.2.4 -uvicorn -Click -backoff -structlog -python-rapidjson -python-multipart -jsonpath-ng # TODO: remove when bulk_path no longer supported -jsonbender==0.9.3 -python-ulid -opencensus -opencensus-ext-stackdriver -opencensus-ext-azure -opencensus-ext-ocagent -opencensus-ext-logging +# +# This file is autogenerated by pip-compile with python 3.9 +# To update, run: +# +# pip-compile requirements.in +# +--index-url https://aws:eyJ2ZXIiOjEsImlzdSI6MTYzMDUxNTYyNiwiZW5jIjoiQTEyOEdDTSIsInRhZyI6ImZ5UFlYYnZhRkhOM1dNdmVpNkc3dlEiLCJleHAiOjE2MzA1NTg4MjYsImFsZyI6IkExMjhHQ01LVyIsIml2IjoiZUVfazRKTW5aRlp3TDg1diJ9.Y1iu6isjfagWbL9b4ENdcA.jQxd_AbMvL9kQfjm.2Tu8K3TDWU5eqpzV11gx3J129Du1MqP5M_WLKwboVyB1YWDoFOr2C4ScG9lCRgcAQ8scS4cenTq3UgUf47-q7qgHSwlufAPN6Nx333LkFzO_M8VeFot6Lkoa9IP2o9QDY74TRWPh5hDnRJtWD1VIfFsVf7AitjCjF3xspuCwaX1KR0_LhnI6KecomjL77U77I_yKmPXwELBLOKO-hkN1xHM8g3r09zDjce1Npb_4lioy4MKEHuLAMUgTGRr3XF1W1FA6M5rEezV09A_J-NE_zZHBrshFykg8ryb-Of05obpRb3Hd2xcuawIFbNogLuqoy6JZ2pg514YCL6kZIqvACFKHF9dUIHHZuM_VkWLYTmflaG_Bb1mpgqvf1Ls1J1vdASgzqMaY7lGqqdTl98PkOgRrqwmFoJa-IfCYolgM1GyU-YPwSWOk_Vefmc5EZ81Q8v94PJz0BuUxCibccaLN1Jf53etog5k-on7typcj5eUi2xpAFcmllle7x396PT0-UasKeN6bls37nI86neYtkLAMN6fvwFztOYNBivvE4iiYHMoqV8yqXPBE6cvelMK7qd7O-4EHJQDWrvEsIrsL9g072bh1F9ax9ji7AZ8UK8LxS08blHBQTcCeEWTuigTdzZ71ob73P1P0cEvZpqlJHec0dw34d8Kz4gP5vSgEJn3LRbXgXOweJaJBFSZ5R8NidqUDjz6NGz66r4bwKfrx9Gwr_GdTo1rWamHhO0bwaaCbenaIpnCe6fF4gZFBrxsDbmZzxMUrB1Z_e6OFyVRlk6O0jJ5qKeOsqh9RLb0wyVcp1g7dVFGVkqOe6UawYthfpq5Snzxmj0JkuuRi1u5xce5KlizGhxzoQZRwWHy0CuucCIrvcicBvKqKzD0C-BR0Sf_QbMIYPN_Jxocf334HeljN-aSM_KsCoQ2KYSKa-YxOfqGy1_PtrBMw05gvnbTUdMu3PWa2e7MaKnyhUHjDNfzaFitCOjxCDQ62q_zTIlA6TgUNDTdGDGKXhsPBJSFANIa4SP0urePXEUWM-3acisZjuSYSSbMxtrnVaQUD5dm8OfPfFA4n_WBA_RWnxETykHk0qEIkiuMt9_9hJQChwo0PFIRGDTgv48ClEYt2VFT1QTEsOYbc-lcMJ3Mx-YNCxyPm0Yb5NdPJcTBV2WjJnNijGTjwk0Ko4mjFidUo51NPICqDtRLZqf-BipDyAQtKQie-D9OeM_Rx_nKEuE6MVD1m6K1Kb3CaHbm-dV2RM7Qv-5YKlYR_rW5ei9PVF3DiNOXLaPguTM8W962tIjM71zM4Oc_PSb7ze8ol89Boknya-3LqV7ihdIJ2IWY7FKHOWfW8UqPfM0Vox6esu1LnW3Hu4NjUFu76.g75g14el7A4k-hChhbHG1Q@osdu-dev-888733619319.d.codeartifact.us-east-1.amazonaws.com/pypi/osdu-python/simple/ +--extra-index-url https://community.opengroup.org/api/v4/projects/465/packages/pypi/simple/ -# for chunking feature -dask[distributed]==2021.4.1 +adal==1.2.7 + # via azure-datalake-store +adlfs==0.7.7 + # via osdu-core-lib-python-azure +aiohttp==3.7.4.post0 + # via + # -r requirements.in + # adlfs + # gcsfs + # osdu-core-lib-python + # osdu-core-lib-python-gcp +aioredis==1.3.1 + # via osdu-core-lib-python +anyio==3.2.1 + # via httpcore +asgiref==3.4.1 + # via uvicorn +async-timeout==3.0.1 + # via + # aiohttp + # aioredis +attrs==21.2.0 + # via + # aiohttp + # osdu-core-lib-python +azure-common==1.1.27 + # via + # azure-keyvault-certificates + # azure-keyvault-keys + # azure-keyvault-secrets +azure-core==1.16.0 + # via + # adlfs + # azure-identity + # azure-keyvault-certificates + # azure-keyvault-keys + # azure-keyvault-secrets + # azure-storage-blob +azure-datalake-store==0.0.52 + # via adlfs +azure-identity==1.6.0 + # via + # adlfs + # osdu-core-lib-python-azure +azure-keyvault==4.1.0 + # via osdu-core-lib-python-azure +azure-keyvault-certificates==4.3.0 + # via azure-keyvault +azure-keyvault-keys==4.4.0 + # via azure-keyvault +azure-keyvault-secrets==4.3.0 + # via azure-keyvault +azure-storage-blob==12.8.1 + # via + # adlfs + # osdu-core-lib-python-azure +backoff==1.11.1 + # via -r requirements.in +boto3==1.18.1 + # via osdu-core-lib-python-ibm +botocore==1.21.1 + # via + # boto3 + # osdu-core-lib-python-ibm + # s3transfer +cachetools==4.2.2 + # via + # google-auth + # osdu-core-lib-python-azure +certifi==2021.5.30 + # via + # httpx + # msrest + # requests +cffi==1.14.6 + # via + # azure-datalake-store + # cryptography +chardet==4.0.0 + # via aiohttp +charset-normalizer==2.0.3 + # via requests +click==8.0.1 + # via + # -r requirements.in + # distributed + # uvicorn +cloudpickle==1.6.0 + # via + # dask + # distributed +cryptography==3.4.7 + # via + # -r requirements.in + # adal + # azure-identity + # azure-keyvault-keys + # azure-storage-blob + # msal + # osdu-core-lib-python + # osdu-core-lib-python-gcp + # pyjwt +dask[distributed]==2021.7.2 + # via + # -r requirements.in + # distributed +decorator==5.0.9 + # via + # gcsfs + # jsonpath-ng +distributed==2021.07.2 + # via dask +fastapi==0.68.1 + # via + # -r requirements.in + # osdu-data-ecosystem-search + # osdu-data-ecosystem-storage fsspec==2021.7.0 + # via + # -r requirements.in + # adlfs + # dask + # gcsfs + # osdu-core-lib-python-azure +gcsfs==2021.7.0 + # via osdu-core-lib-python-gcp +google-api-core[grpc]==1.31.0 + # via + # google-cloud-core + # google-cloud-monitoring + # google-cloud-trace + # opencensus +google-auth==1.33.0 + # via + # gcsfs + # google-api-core + # google-auth-oauthlib + # google-cloud-core +google-auth-oauthlib==0.4.4 + # via gcsfs +google-cloud-core==1.7.1 + # via google-cloud-trace +google-cloud-monitoring==0.36.0 + # via opencensus-ext-stackdriver +google-cloud-trace==0.24.0 + # via opencensus-ext-stackdriver +googleapis-common-protos==1.53.0 + # via google-api-core +grpcio==1.38.1 + # via + # google-api-core + # opencensus-ext-ocagent + # opencensus-proto +h11==0.12.0 + # via + # httpcore + # uvicorn +heapdict==1.0.1 + # via zict +hiredis==2.0.0 + # via aioredis +httpcore==0.13.6 + # via httpx +httpx==0.18.2 + # via + # osdu-data-ecosystem-search + # osdu-data-ecosystem-storage +idna==3.2 + # via + # anyio + # requests + # rfc3986 + # yarl +isodate==0.6.0 + # via msrest +jmespath==0.10.0 + # via + # boto3 + # botocore +jsonbender==0.9.3 + # via -r requirements.in +jsonpath-ng==1.5.3 + # via -r requirements.in +locket==0.2.1 + # via partd +msal==1.12.0 + # via + # azure-identity + # msal-extensions +msal-extensions==0.3.0 + # via azure-identity +msgpack==1.0.2 + # via distributed +msrest==0.6.21 + # via + # azure-keyvault-certificates + # azure-keyvault-keys + # azure-keyvault-secrets + # azure-storage-blob +multidict==5.1.0 + # via + # aiohttp + # yarl natsort==7.1.1 ---extra-index-url \ -https://community.opengroup.org/api/v4/projects/465/packages/pypi/simple/ - -osdu-log-recognition-lib>=0.0.9 - -osdu-data-ecosystem-storage~=1.2.0 -osdu-data-ecosystem-search>=0.3.2, <0.4 + # via -r requirements.in +numpy==1.21.1 + # via + # pandas + # pyarrow +oauthlib==3.1.1 + # via requests-oauthlib +opencensus==0.7.13 + # via + # -r requirements.in + # opencensus-ext-azure + # opencensus-ext-logging + # opencensus-ext-ocagent + # opencensus-ext-stackdriver +opencensus-context==0.1.2 + # via opencensus +opencensus-ext-azure==1.0.8 + # via -r requirements.in +opencensus-ext-logging==0.1.0 + # via -r requirements.in +opencensus-ext-ocagent==0.7.1 + # via -r requirements.in +opencensus-ext-stackdriver==0.7.4 + # via -r requirements.in +opencensus-proto==0.1.0 + # via opencensus-ext-ocagent +osdu-core-lib-python==1.1.0 + # via -r requirements.in +osdu-core-lib-python-aws==0.0.2 + # via -r requirements.in +osdu-core-lib-python-azure==1.2.1 + # via -r requirements.in +osdu-core-lib-python-gcp==1.1.0 + # via -r requirements.in +osdu-core-lib-python-ibm==1.2.0 + # via -r requirements.in +osdu-data-ecosystem-search==0.3.3 + # via -r requirements.in +osdu-data-ecosystem-storage==1.2.0 + # via -r requirements.in +osdu-log-recognition-lib==0.0.9 + # via -r requirements.in +packaging==21.0 + # via + # dask + # google-api-core +pandas==1.2.4 + # via -r requirements.in +partd==1.2.0 + # via dask +ply==3.11 + # via jsonpath-ng +portalocker==1.7.1 + # via msal-extensions +protobuf==3.17.3 + # via + # google-api-core + # googleapis-common-protos +psutil==5.8.0 + # via + # distributed + # opencensus-ext-azure +pyarrow==4.0.1 + # via -r requirements.in +pyasn1==0.4.8 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.2.8 + # via google-auth +pycparser==2.20 + # via cffi +pydantic==1.8.2 + # via + # fastapi + # osdu-data-ecosystem-search + # osdu-data-ecosystem-storage +pyjwt[crypto]==2.1.0 + # via + # adal + # msal + # osdu-core-lib-python + # osdu-core-lib-python-gcp +pyparsing==2.4.7 + # via packaging +python-dateutil==2.8.2 + # via + # adal + # botocore + # pandas +python-multipart==0.0.5 + # via -r requirements.in +python-rapidjson==1.4 + # via -r requirements.in +python-ulid==1.0.3 + # via -r requirements.in +pytz==2021.1 + # via + # google-api-core + # pandas +pyyaml==5.4.1 + # via + # dask + # distributed +requests==2.26.0 + # via + # adal + # azure-core + # azure-datalake-store + # gcsfs + # google-api-core + # msal + # msrest + # opencensus-ext-azure + # requests-oauthlib +requests-oauthlib==1.3.0 + # via + # google-auth-oauthlib + # msrest +rfc3986[idna2008]==1.5.0 + # via httpx +rsa==4.7.2 + # via google-auth +s3transfer==0.5.0 + # via boto3 +six==1.16.0 + # via + # azure-core + # azure-identity + # azure-keyvault-keys + # google-api-core + # google-auth + # google-cloud-core + # grpcio + # isodate + # jsonpath-ng + # protobuf + # python-dateutil + # python-multipart +sniffio==1.2.0 + # via + # anyio + # httpcore + # httpx +sortedcontainers==2.4.0 + # via distributed +starlette==0.14.2 + # via fastapi +structlog==21.1.0 + # via -r requirements.in +tblib==1.7.0 + # via distributed +toolz==0.11.1 + # via + # dask + # distributed + # partd +tornado==6.1 + # via distributed +typing-extensions==3.10.0.0 + # via + # aiohttp + # osdu-data-ecosystem-search + # osdu-data-ecosystem-storage + # pydantic +urllib3==1.26.6 + # via + # botocore + # requests +uvicorn==0.14.0 + # via -r requirements.in +xmltodict==0.11.0 + # via osdu-log-recognition-lib +yarl==1.6.3 + # via aiohttp +zict==2.0.0 + # via distributed -osdu-core-lib-python-ibm>=0.0.1, <0.1 -osdu-core-lib-python-gcp~=1.1.0 -osdu-core-lib-python-azure~=1.1.1 -osdu-core-lib-python-aws>=0.0.1, <0.1 -osdu-core-lib-python~=1.1.0 -s3fs==2021.8.0 +s3fs +# The following packages are considered to be unsafe in a requirements file: +# setuptools -- GitLab From 76eb0b993c5cebe6facbc1f7f9aaccf82c8bd424 Mon Sep 17 00:00:00 2001 From: Yunhua Koglin Date: Wed, 1 Sep 2021 12:36:42 -0500 Subject: [PATCH 29/36] update requirements --- requirements.txt | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 23f06719..f6d9c964 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,9 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.7 # To update, run: # # pip-compile requirements.in # ---index-url https://aws:eyJ2ZXIiOjEsImlzdSI6MTYzMDUxNTYyNiwiZW5jIjoiQTEyOEdDTSIsInRhZyI6ImZ5UFlYYnZhRkhOM1dNdmVpNkc3dlEiLCJleHAiOjE2MzA1NTg4MjYsImFsZyI6IkExMjhHQ01LVyIsIml2IjoiZUVfazRKTW5aRlp3TDg1diJ9.Y1iu6isjfagWbL9b4ENdcA.jQxd_AbMvL9kQfjm.2Tu8K3TDWU5eqpzV11gx3J129Du1MqP5M_WLKwboVyB1YWDoFOr2C4ScG9lCRgcAQ8scS4cenTq3UgUf47-q7qgHSwlufAPN6Nx333LkFzO_M8VeFot6Lkoa9IP2o9QDY74TRWPh5hDnRJtWD1VIfFsVf7AitjCjF3xspuCwaX1KR0_LhnI6KecomjL77U77I_yKmPXwELBLOKO-hkN1xHM8g3r09zDjce1Npb_4lioy4MKEHuLAMUgTGRr3XF1W1FA6M5rEezV09A_J-NE_zZHBrshFykg8ryb-Of05obpRb3Hd2xcuawIFbNogLuqoy6JZ2pg514YCL6kZIqvACFKHF9dUIHHZuM_VkWLYTmflaG_Bb1mpgqvf1Ls1J1vdASgzqMaY7lGqqdTl98PkOgRrqwmFoJa-IfCYolgM1GyU-YPwSWOk_Vefmc5EZ81Q8v94PJz0BuUxCibccaLN1Jf53etog5k-on7typcj5eUi2xpAFcmllle7x396PT0-UasKeN6bls37nI86neYtkLAMN6fvwFztOYNBivvE4iiYHMoqV8yqXPBE6cvelMK7qd7O-4EHJQDWrvEsIrsL9g072bh1F9ax9ji7AZ8UK8LxS08blHBQTcCeEWTuigTdzZ71ob73P1P0cEvZpqlJHec0dw34d8Kz4gP5vSgEJn3LRbXgXOweJaJBFSZ5R8NidqUDjz6NGz66r4bwKfrx9Gwr_GdTo1rWamHhO0bwaaCbenaIpnCe6fF4gZFBrxsDbmZzxMUrB1Z_e6OFyVRlk6O0jJ5qKeOsqh9RLb0wyVcp1g7dVFGVkqOe6UawYthfpq5Snzxmj0JkuuRi1u5xce5KlizGhxzoQZRwWHy0CuucCIrvcicBvKqKzD0C-BR0Sf_QbMIYPN_Jxocf334HeljN-aSM_KsCoQ2KYSKa-YxOfqGy1_PtrBMw05gvnbTUdMu3PWa2e7MaKnyhUHjDNfzaFitCOjxCDQ62q_zTIlA6TgUNDTdGDGKXhsPBJSFANIa4SP0urePXEUWM-3acisZjuSYSSbMxtrnVaQUD5dm8OfPfFA4n_WBA_RWnxETykHk0qEIkiuMt9_9hJQChwo0PFIRGDTgv48ClEYt2VFT1QTEsOYbc-lcMJ3Mx-YNCxyPm0Yb5NdPJcTBV2WjJnNijGTjwk0Ko4mjFidUo51NPICqDtRLZqf-BipDyAQtKQie-D9OeM_Rx_nKEuE6MVD1m6K1Kb3CaHbm-dV2RM7Qv-5YKlYR_rW5ei9PVF3DiNOXLaPguTM8W962tIjM71zM4Oc_PSb7ze8ol89Boknya-3LqV7ihdIJ2IWY7FKHOWfW8UqPfM0Vox6esu1LnW3Hu4NjUFu76.g75g14el7A4k-hChhbHG1Q@osdu-dev-888733619319.d.codeartifact.us-east-1.amazonaws.com/pypi/osdu-python/simple/ --extra-index-url https://community.opengroup.org/api/v4/projects/465/packages/pypi/simple/ adal==1.2.7 @@ -98,6 +97,8 @@ cloudpickle==1.6.0 # via # dask # distributed +colorama==0.4.4 + # via click cryptography==3.4.7 # via # -r requirements.in @@ -311,6 +312,8 @@ pytz==2021.1 # via # google-api-core # pandas + #pywin32==301 + # via portalocker pyyaml==5.4.1 # via # dask @@ -388,7 +391,7 @@ yarl==1.6.3 # via aiohttp zict==2.0.0 # via distributed - s3fs + # The following packages are considered to be unsafe in a requirements file: # setuptools -- GitLab From 7ce6379af0e0e56d891f7bb8098bf6dd74f87d4c Mon Sep 17 00:00:00 2001 From: Yunhua Koglin Date: Wed, 1 Sep 2021 14:01:09 -0500 Subject: [PATCH 30/36] update version --- provider/os-wellbore-ddms-aws/build-aws/Dockerfile | 4 ++-- provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/provider/os-wellbore-ddms-aws/build-aws/Dockerfile b/provider/os-wellbore-ddms-aws/build-aws/Dockerfile index 0873b10b..9e922cbb 100644 --- a/provider/os-wellbore-ddms-aws/build-aws/Dockerfile +++ b/provider/os-wellbore-ddms-aws/build-aws/Dockerfile @@ -15,9 +15,9 @@ FROM python:3.9-slim-buster COPY requirements.txt ./ -COPY osdu-core-lib-python-aws-2.0.2.tar.gz ./ +COPY osdu-core-lib-python-aws-0.0.3.tar.gz ./ RUN pip install -r requirements.txt -RUN pip install ./osdu-core-lib-python-aws-2.0.2.tar.gz +RUN pip install ./osdu-core-lib-python-aws-0.0.3.tar.gz COPY ./app /app COPY provider/os-wellbore-ddms-aws/build-aws/entrypoint.sh . diff --git a/provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml b/provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml index ec7de53f..185429be 100644 --- a/provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml +++ b/provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml @@ -65,9 +65,9 @@ phases: --region us-east-1 \ --format pypi \ --package osdu-core-lib-python-aws \ - --package-version 2.0.2 \ - --asset osdu-core-lib-python-aws-2.0.2.tar.gz \ - ./osdu-core-lib-python-aws-2.0.2.tar.gz + --package-version 0.0.3 \ + --asset osdu-core-lib-python-aws-0.0.3.tar.gz \ + ./osdu-core-lib-python-aws-0.0.3.tar.gz - aws codeartifact login --tool pip --domain osdu-dev --domain-owner $AWS_ACCOUNT_ID --repository pypi-store -- GitLab From 9125be6e3852911937beda98fec3cfe5d830ffcc Mon Sep 17 00:00:00 2001 From: Yunhua Koglin Date: Thu, 2 Sep 2021 09:19:12 -0500 Subject: [PATCH 31/36] using chunk --- provider/os-wellbore-ddms-aws/build-aws/Dockerfile | 5 ++--- .../os-wellbore-ddms-aws/build-aws/buildspec.yaml | 13 ------------- requirements.in | 2 +- requirements.txt | 2 +- 4 files changed, 4 insertions(+), 18 deletions(-) diff --git a/provider/os-wellbore-ddms-aws/build-aws/Dockerfile b/provider/os-wellbore-ddms-aws/build-aws/Dockerfile index 9e922cbb..36724538 100644 --- a/provider/os-wellbore-ddms-aws/build-aws/Dockerfile +++ b/provider/os-wellbore-ddms-aws/build-aws/Dockerfile @@ -12,12 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM python:3.9-slim-buster +FROM python:3.7-slim-buster COPY requirements.txt ./ -COPY osdu-core-lib-python-aws-0.0.3.tar.gz ./ + RUN pip install -r requirements.txt -RUN pip install ./osdu-core-lib-python-aws-0.0.3.tar.gz COPY ./app /app COPY provider/os-wellbore-ddms-aws/build-aws/entrypoint.sh . diff --git a/provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml b/provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml index 185429be..00f2ee15 100644 --- a/provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml +++ b/provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml @@ -56,19 +56,6 @@ phases: - chmod +x ./tests/aws-test/build-aws/prepare-dist.sh - ./tests/aws-test/build-aws/prepare-dist.sh - - aws codeartifact login --tool pip --domain osdu-dev --domain-owner $AWS_ACCOUNT_ID --repository osdu-python --region us-east-1 - - | - aws codeartifact get-package-version-asset \ - --domain osdu-dev \ - --domain-owner $AWS_ACCOUNT_ID \ - --repository osdu-python \ - --region us-east-1 \ - --format pypi \ - --package osdu-core-lib-python-aws \ - --package-version 0.0.3 \ - --asset osdu-core-lib-python-aws-0.0.3.tar.gz \ - ./osdu-core-lib-python-aws-0.0.3.tar.gz - - aws codeartifact login --tool pip --domain osdu-dev --domain-owner $AWS_ACCOUNT_ID --repository pypi-store - pip config set global.extra-index-url https://community.opengroup.org/api/v4/projects/465/packages/pypi/simple/ diff --git a/requirements.in b/requirements.in index 6b7fcbc6..02c70cd1 100644 --- a/requirements.in +++ b/requirements.in @@ -35,5 +35,5 @@ osdu-data-ecosystem-search>=0.3.2, <0.4 osdu-core-lib-python-ibm~=1.2.0 osdu-core-lib-python-gcp~=1.1.0 osdu-core-lib-python-azure~=1.2.1 -osdu-core-lib-python-aws>=0.0.1, <0.1 +osdu-core-lib-python-aws>=0.0.3, <0.1 osdu-core-lib-python~=1.1.0 diff --git a/requirements.txt b/requirements.txt index f6d9c964..b72d64a9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -240,7 +240,7 @@ opencensus-proto==0.1.0 # via opencensus-ext-ocagent osdu-core-lib-python==1.1.0 # via -r requirements.in -osdu-core-lib-python-aws==0.0.2 +osdu-core-lib-python-aws==0.0.3 # via -r requirements.in osdu-core-lib-python-azure==1.2.1 # via -r requirements.in -- GitLab From 85c1592af3334cd52b15c80dde572e7f39638e97 Mon Sep 17 00:00:00 2001 From: Yunhua Koglin Date: Thu, 2 Sep 2021 09:20:48 -0500 Subject: [PATCH 32/36] restore --- .../os-wellbore-ddms-aws/build-aws/buildspec.yaml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml b/provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml index 00f2ee15..c80cc448 100644 --- a/provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml +++ b/provider/os-wellbore-ddms-aws/build-aws/buildspec.yaml @@ -57,14 +57,18 @@ phases: - ./tests/aws-test/build-aws/prepare-dist.sh - aws codeartifact login --tool pip --domain osdu-dev --domain-owner $AWS_ACCOUNT_ID --repository pypi-store - - pip config set global.extra-index-url https://community.opengroup.org/api/v4/projects/465/packages/pypi/simple/ + + - pip install -r ./requirements.txt + - pip install -r ./requirements_dev.txt + # using jar instead of executable since exe requires 32 bit image and doesn't work on codebuild 4.0 - | java -jar sonar-scanner-4.6.2.2472/lib/sonar-scanner-cli-4.6.2.2472.jar -Dsonar.projectKey=wellbore-ddms -Dsonar.host.url=${SONAR_URL} \ -Dsonar.login=${SONAR_USERNAME} -Dsonar.password=${SONAR_PASSWORD} -Dsonar.branch.name=${BRANCH_NAME} + - safety check -r requirements.txt -r requirements_dev.txt -o safety_check.txt; EXITCODE=$? - + - echo "Logging into Docker Hub..." - docker login -u ${DOCKER_USERNAME} -p ${DOCKER_PASSWORD} @@ -74,9 +78,7 @@ phases: - echo "Pushing docker image..." - docker push ${ECR_IMAGE} - docker push ${ECR_IMAGE_BRANCH_LATEST} - - - pip install -r ./requirements.txt - - pip install -r ./requirements_dev.txt + - python -m pytest --junit-xml=unit_tests_report.xml --cov=app --cov-report=html --cov-report=xml ./tests/unit - echo "Generate build-info.json" -- GitLab From 8ff3fc60cdf0b5c7488a36aed8f74bac1ff40a20 Mon Sep 17 00:00:00 2001 From: Yunhua Koglin Date: Thu, 2 Sep 2021 16:08:00 -0500 Subject: [PATCH 33/36] update packages --- requirements.in | 4 ++-- requirements.txt | 24 +++++++++++++++--------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/requirements.in b/requirements.in index 02c70cd1..4b4fb375 100644 --- a/requirements.in +++ b/requirements.in @@ -23,7 +23,7 @@ opencensus-ext-logging dask[distributed]==2021.7.2 fsspec python-ulid - +s3fs==2021.8.0 --extra-index-url \ https://community.opengroup.org/api/v4/projects/465/packages/pypi/simple/ @@ -36,4 +36,4 @@ osdu-core-lib-python-ibm~=1.2.0 osdu-core-lib-python-gcp~=1.1.0 osdu-core-lib-python-azure~=1.2.1 osdu-core-lib-python-aws>=0.0.3, <0.1 -osdu-core-lib-python~=1.1.0 +osdu-core-lib-python~=1.1.0 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index b72d64a9..7099de40 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.7 +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # # pip-compile requirements.in @@ -10,13 +10,18 @@ adal==1.2.7 # via azure-datalake-store adlfs==0.7.7 # via osdu-core-lib-python-azure +aiobotocore==1.4.1 + # via s3fs aiohttp==3.7.4.post0 # via # -r requirements.in # adlfs + # aiobotocore # gcsfs # osdu-core-lib-python # osdu-core-lib-python-gcp +aioitertools==0.8.0 + # via aiobotocore aioredis==1.3.1 # via osdu-core-lib-python anyio==3.2.1 @@ -64,10 +69,11 @@ azure-storage-blob==12.8.1 # osdu-core-lib-python-azure backoff==1.11.1 # via -r requirements.in -boto3==1.18.1 +boto3==1.17.106 # via osdu-core-lib-python-ibm -botocore==1.21.1 +botocore==1.20.106 # via + # aiobotocore # boto3 # osdu-core-lib-python-ibm # s3transfer @@ -97,8 +103,6 @@ cloudpickle==1.6.0 # via # dask # distributed -colorama==0.4.4 - # via click cryptography==3.4.7 # via # -r requirements.in @@ -132,6 +136,7 @@ fsspec==2021.7.0 # dask # gcsfs # osdu-core-lib-python-azure + # s3fs gcsfs==2021.7.0 # via osdu-core-lib-python-gcp google-api-core[grpc]==1.31.0 @@ -312,8 +317,6 @@ pytz==2021.1 # via # google-api-core # pandas - #pywin32==301 - # via portalocker pyyaml==5.4.1 # via # dask @@ -337,7 +340,9 @@ rfc3986[idna2008]==1.5.0 # via httpx rsa==4.7.2 # via google-auth -s3transfer==0.5.0 +s3fs==2021.8.0 + # via -r requirements.in +s3transfer==0.4.2 # via boto3 six==1.16.0 # via @@ -385,13 +390,14 @@ urllib3==1.26.6 # requests uvicorn==0.14.0 # via -r requirements.in +wrapt==1.12.1 + # via aiobotocore xmltodict==0.11.0 # via osdu-log-recognition-lib yarl==1.6.3 # via aiohttp zict==2.0.0 # via distributed -s3fs # The following packages are considered to be unsafe in a requirements file: # setuptools -- GitLab From 3a0d40eda8376e2dec63b6e14283c2e7568257ab Mon Sep 17 00:00:00 2001 From: Yunhua Koglin Date: Wed, 15 Sep 2021 13:09:06 -0500 Subject: [PATCH 34/36] basic test --- tests/aws-test/build-aws/run-integration-tests.sh | 2 +- tests/aws-test/build-aws/run-tests.sh | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/aws-test/build-aws/run-integration-tests.sh b/tests/aws-test/build-aws/run-integration-tests.sh index 0fcba959..20e286db 100644 --- a/tests/aws-test/build-aws/run-integration-tests.sh +++ b/tests/aws-test/build-aws/run-integration-tests.sh @@ -23,4 +23,4 @@ cd tests/integration python3 gen_postman_env.py --token $svctoken --base_url $svc_url --cloud_provider "aws" --acl_domain $acl_domain --legal_tag $legal_tag --data_partition $tenant -pytest ./functional --environment="./generated/postman_environment.json" --filter-tag=!search +pytest ./functional --environment="./generated/postman_environment.json" --filter-tag=basic diff --git a/tests/aws-test/build-aws/run-tests.sh b/tests/aws-test/build-aws/run-tests.sh index cc370c56..aa661d0c 100644 --- a/tests/aws-test/build-aws/run-tests.sh +++ b/tests/aws-test/build-aws/run-tests.sh @@ -44,12 +44,13 @@ rm -rf test-reports/ mkdir test-reports cd integration + acl_domain='example.com' legal_tag='opendes-sdmstestlegaltag' python3 gen_postman_env.py --token $token --base_url $svc_url --cloud_provider "aws" --acl_domain $acl_domain --legal_tag $legal_tag --data_partition $tenant -pytest ./functional --environment="./generated/postman_environment.json" --filter-tag=!search +pytest ./functional --environment="./generated/postman_environment.json" --filter-tag=basic TEST_EXIT_CODE=$? deactivate -- GitLab From 2f55fa75f09dd420e31cf6b960bc9176173ffd75 Mon Sep 17 00:00:00 2001 From: Yunhua Koglin Date: Wed, 15 Sep 2021 13:30:35 -0500 Subject: [PATCH 35/36] update notice --- NOTICE | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/NOTICE b/NOTICE index 7112b62a..9774a03e 100644 --- a/NOTICE +++ b/NOTICE @@ -7,6 +7,7 @@ Apache-2.0 ======================================================================== The following software have components provided under the terms of this license: +- aiobotocore (from https://github.com/aio-libs/aiobotocore) - aiohttp (from https://github.com/aio-libs/aiohttp/) - async-timeout (from https://github.com/aio-libs/async_timeout/) - boto3 (from https://github.com/boto/boto3) @@ -37,6 +38,7 @@ The following software have components provided under the terms of this license: - packaging (from https://github.com/pypa/packaging) - pandas (from http://pandas.pydata.org) - pep517 (from https://github.com/takluyver/pep517) +- proto-plus (from https://github.com/googleapis/proto-plus-python.git) - pyarrow (from https://arrow.apache.org/) - pytest-asyncio (from https://github.com/pytest-dev/pytest-asyncio) - pytest-dependency (from https://github.com/RKrahl/pytest-dependency) @@ -69,6 +71,7 @@ The following software have components provided under the terms of this license: - pyasn1-modules (from http://sourceforge.net/projects/pyasn1/) - pycparser (from https://github.com/eliben/pycparser) - tblib (from https://github.com/ionelmc/python-tblib) +- wrapt (from https://github.com/GrahamDumpleton/wrapt) ======================================================================== BSD-3-Clause @@ -113,10 +116,12 @@ The following software have components provided under the terms of this license: - python-dateutil (from https://dateutil.readthedocs.org) - python-rapidjson (from https://github.com/python-rapidjson/python-rapidjson) - requests-oauthlib (from https://github.com/requests/requests-oauthlib) +- s3fs (from http://github.com/dask/s3fs/) - starlette (from https://github.com/encode/starlette) - tblib (from https://github.com/ionelmc/python-tblib) - toolz (from http://github.com/pytoolz/toolz/) - uvicorn (from https://github.com/tomchristie/uvicorn) +- wrapt (from https://github.com/GrahamDumpleton/wrapt) - zict (from http://github.com/dask/zict/) ======================================================================== @@ -132,6 +137,7 @@ The following software have components provided under the terms of this license: - numpy (from http://www.numpy.org) - pandas (from http://pandas.pydata.org) - partd (from http://github.com/dask/partd/) +- s3fs (from http://github.com/dask/s3fs/) - toolz (from http://github.com/pytoolz/toolz/) ======================================================================== @@ -220,6 +226,7 @@ The following software have components provided under the terms of this license: - PyYAML (from http://pyyaml.org/wiki/PyYAML) - adal (from https://github.com/AzureAD/azure-activedirectory-library-for-python) - aiohttp (from https://github.com/aio-libs/aiohttp/) +- aioitertools (from https://github.com/jreese/aioitertools) - aioredis (from https://github.com/aio-libs/aioredis) - anyio (from https://pypi.org/project/anyio/3.3.0/) - asgiref (from http://github.com/django/asgiref/) @@ -246,6 +253,7 @@ The following software have components provided under the terms of this license: - iniconfig (from http://github.com/RonnyPfannschmidt/iniconfig) - jmespath (from https://github.com/jmespath/jmespath.py) - jsonschema (from http://github.com/Julian/jsonschema) +- mockito (from https://github.com/kaste/mockito-python) - msal (from https://github.com/AzureAD/microsoft-authentication-library-for-python) - msal-extensions (from https://pypi.org/project/msal-extensions/0.1.3/) - msrest (from https://github.com/Azure/msrest-for-python) @@ -272,7 +280,7 @@ The following software have components provided under the terms of this license: - sniffio (from https://github.com/python-trio/sniffio) - structlog (from http://www.structlog.org/) - toml (from https://github.com/uiri/toml) -- tomli (from https://pypi.org/project/tomli/1.1.0/) +- tomli (from https://pypi.org/project/tomli/1.2.1/) - urllib3 (from https://urllib3.readthedocs.io/) - xmltodict (from https://github.com/martinblech/xmltodict) - zipp (from https://github.com/jaraco/zipp) -- GitLab From a88f4d92bec3dda4a078212d2c5893642e6f72ae Mon Sep 17 00:00:00 2001 From: Yunhua Koglin Date: Wed, 15 Sep 2021 21:00:47 +0000 Subject: [PATCH 36/36] add DaskBulkStorage --- app/injector/aws_injector.py | 10 ++++++++-- requirements.in | 6 +++--- requirements.txt | 25 ++++++++++++++++--------- tests/aws-test/build-aws/run-tests.sh | 1 - 4 files changed, 27 insertions(+), 15 deletions(-) diff --git a/app/injector/aws_injector.py b/app/injector/aws_injector.py index 27876189..425c8cb6 100644 --- a/app/injector/aws_injector.py +++ b/app/injector/aws_injector.py @@ -14,7 +14,9 @@ from osdu.core.api.storage.blob_storage_base import BlobStorageBase from osdu_aws.storage.storage_aws import AwsStorage - +from osdu_aws.storage.dask_storage_parameters import get_dask_storage_parameters as aws_parameters +from app.utils import Context +from app.bulk_persistence import resolve_tenant from app.bulk_persistence.dask.dask_bulk_storage import DaskBulkStorage from .app_injector import AppInjector, AppInjectorModule from app.conf import Config @@ -34,4 +36,8 @@ class AwsInjector(AppInjectorModule): @staticmethod async def build_aws_dask_blob_storage() -> DaskBulkStorage: - raise NotImplementedError() + ctx: Context = Context.current() + tenant = await resolve_tenant(ctx.partition_id) + service_account_file=f'{Config.aws_region.value}$${Config.aws_env.value}' + params = await aws_parameters(tenant, service_account_file) + return await DaskBulkStorage.create(params) diff --git a/requirements.in b/requirements.in index 6b7fcbc6..4b4fb375 100644 --- a/requirements.in +++ b/requirements.in @@ -23,7 +23,7 @@ opencensus-ext-logging dask[distributed]==2021.7.2 fsspec python-ulid - +s3fs==2021.8.0 --extra-index-url \ https://community.opengroup.org/api/v4/projects/465/packages/pypi/simple/ @@ -35,5 +35,5 @@ osdu-data-ecosystem-search>=0.3.2, <0.4 osdu-core-lib-python-ibm~=1.2.0 osdu-core-lib-python-gcp~=1.1.0 osdu-core-lib-python-azure~=1.2.1 -osdu-core-lib-python-aws>=0.0.1, <0.1 -osdu-core-lib-python~=1.1.0 +osdu-core-lib-python-aws>=0.0.3, <0.1 +osdu-core-lib-python~=1.1.0 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 3f7f9b2e..7099de40 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.7 +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # # pip-compile requirements.in @@ -10,13 +10,18 @@ adal==1.2.7 # via azure-datalake-store adlfs==0.7.7 # via osdu-core-lib-python-azure +aiobotocore==1.4.1 + # via s3fs aiohttp==3.7.4.post0 # via # -r requirements.in # adlfs + # aiobotocore # gcsfs # osdu-core-lib-python # osdu-core-lib-python-gcp +aioitertools==0.8.0 + # via aiobotocore aioredis==1.3.1 # via osdu-core-lib-python anyio==3.2.1 @@ -64,10 +69,11 @@ azure-storage-blob==12.8.1 # osdu-core-lib-python-azure backoff==1.11.1 # via -r requirements.in -boto3==1.18.1 +boto3==1.17.106 # via osdu-core-lib-python-ibm -botocore==1.21.1 +botocore==1.20.106 # via + # aiobotocore # boto3 # osdu-core-lib-python-ibm # s3transfer @@ -97,8 +103,6 @@ cloudpickle==1.6.0 # via # dask # distributed -colorama==0.4.4 - # via click cryptography==3.4.7 # via # -r requirements.in @@ -132,6 +136,7 @@ fsspec==2021.7.0 # dask # gcsfs # osdu-core-lib-python-azure + # s3fs gcsfs==2021.7.0 # via osdu-core-lib-python-gcp google-api-core[grpc]==1.31.0 @@ -240,7 +245,7 @@ opencensus-proto==0.1.0 # via opencensus-ext-ocagent osdu-core-lib-python==1.1.0 # via -r requirements.in -osdu-core-lib-python-aws==0.0.2 +osdu-core-lib-python-aws==0.0.3 # via -r requirements.in osdu-core-lib-python-azure==1.2.1 # via -r requirements.in @@ -312,8 +317,6 @@ pytz==2021.1 # via # google-api-core # pandas - #pywin32==301 - # via portalocker pyyaml==5.4.1 # via # dask @@ -337,7 +340,9 @@ rfc3986[idna2008]==1.5.0 # via httpx rsa==4.7.2 # via google-auth -s3transfer==0.5.0 +s3fs==2021.8.0 + # via -r requirements.in +s3transfer==0.4.2 # via boto3 six==1.16.0 # via @@ -385,6 +390,8 @@ urllib3==1.26.6 # requests uvicorn==0.14.0 # via -r requirements.in +wrapt==1.12.1 + # via aiobotocore xmltodict==0.11.0 # via osdu-log-recognition-lib yarl==1.6.3 diff --git a/tests/aws-test/build-aws/run-tests.sh b/tests/aws-test/build-aws/run-tests.sh index aa661d0c..c911d5a7 100644 --- a/tests/aws-test/build-aws/run-tests.sh +++ b/tests/aws-test/build-aws/run-tests.sh @@ -44,7 +44,6 @@ rm -rf test-reports/ mkdir test-reports cd integration - acl_domain='example.com' legal_tag='opendes-sdmstestlegaltag' -- GitLab