Commit 773b5efc authored by Yunhua Koglin's avatar Yunhua Koglin
Browse files

Merge branch 'aws-chunk' into 'master'

Chunking support for AWS

See merge request !229
parents f410bdc4 d406b0b4
Pipeline #65981 passed with stages
in 20 minutes and 23 seconds
......@@ -253,6 +253,7 @@ The following software have components provided under the terms of this license:
- iniconfig (from http://github.com/RonnyPfannschmidt/iniconfig)
- jmespath (from https://github.com/jmespath/jmespath.py)
- jsonschema (from http://github.com/Julian/jsonschema)
- mockito (from https://github.com/kaste/mockito-python)
- msal (from https://github.com/AzureAD/microsoft-authentication-library-for-python)
- msal-extensions (from https://pypi.org/project/msal-extensions/0.1.3/)
- msrest (from https://github.com/Azure/msrest-for-python)
......
......@@ -14,7 +14,9 @@
from osdu.core.api.storage.blob_storage_base import BlobStorageBase
from osdu_aws.storage.storage_aws import AwsStorage
from osdu_aws.storage.dask_storage_parameters import get_dask_storage_parameters as aws_parameters
from app.utils import Context
from app.bulk_persistence import resolve_tenant
from app.bulk_persistence.dask.dask_bulk_storage import DaskBulkStorage
from .app_injector import AppInjector, AppInjectorModule
from app.conf import Config
......@@ -34,4 +36,8 @@ class AwsInjector(AppInjectorModule):
@staticmethod
async def build_aws_dask_blob_storage() -> DaskBulkStorage:
raise NotImplementedError()
ctx: Context = Context.current()
tenant = await resolve_tenant(ctx.partition_id)
service_account_file=f'{Config.aws_region.value}$${Config.aws_env.value}'
params = await aws_parameters(tenant, service_account_file)
return await DaskBulkStorage.create(params)
......@@ -23,7 +23,7 @@ opencensus-ext-logging
dask[distributed]==2021.7.2
fsspec
python-ulid
s3fs==2021.8.0
--extra-index-url \
https://community.opengroup.org/api/v4/projects/465/packages/pypi/simple/
......@@ -35,5 +35,5 @@ osdu-data-ecosystem-search>=0.3.2, <0.4
osdu-core-lib-python-ibm~=1.2.0
osdu-core-lib-python-gcp~=1.1.0
osdu-core-lib-python-azure~=1.2.1
osdu-core-lib-python-aws>=0.0.1, <0.1
osdu-core-lib-python~=1.1.0
osdu-core-lib-python-aws>=0.0.3, <0.1
osdu-core-lib-python~=1.1.0
\ No newline at end of file
#
# This file is autogenerated by pip-compile with python 3.7
# This file is autogenerated by pip-compile with python 3.9
# To update, run:
#
# pip-compile requirements.in
......@@ -10,13 +10,18 @@ adal==1.2.7
# via azure-datalake-store
adlfs==0.7.7
# via osdu-core-lib-python-azure
aiobotocore==1.4.1
# via s3fs
aiohttp==3.7.4.post0
# via
# -r requirements.in
# adlfs
# aiobotocore
# gcsfs
# osdu-core-lib-python
# osdu-core-lib-python-gcp
aioitertools==0.8.0
# via aiobotocore
aioredis==1.3.1
# via osdu-core-lib-python
anyio==3.2.1
......@@ -64,10 +69,11 @@ azure-storage-blob==12.8.1
# osdu-core-lib-python-azure
backoff==1.11.1
# via -r requirements.in
boto3==1.18.1
boto3==1.17.106
# via osdu-core-lib-python-ibm
botocore==1.21.1
botocore==1.20.106
# via
# aiobotocore
# boto3
# osdu-core-lib-python-ibm
# s3transfer
......@@ -97,8 +103,6 @@ cloudpickle==1.6.0
# via
# dask
# distributed
colorama==0.4.4
# via click
cryptography==3.4.7
# via
# -r requirements.in
......@@ -132,6 +136,7 @@ fsspec==2021.7.0
# dask
# gcsfs
# osdu-core-lib-python-azure
# s3fs
gcsfs==2021.7.0
# via osdu-core-lib-python-gcp
google-api-core[grpc]==1.31.0
......@@ -240,7 +245,7 @@ opencensus-proto==0.1.0
# via opencensus-ext-ocagent
osdu-core-lib-python==1.1.0
# via -r requirements.in
osdu-core-lib-python-aws==0.0.2
osdu-core-lib-python-aws==0.0.3
# via -r requirements.in
osdu-core-lib-python-azure==1.2.1
# via -r requirements.in
......@@ -312,8 +317,6 @@ pytz==2021.1
# via
# google-api-core
# pandas
#pywin32==301
# via portalocker
pyyaml==5.4.1
# via
# dask
......@@ -337,7 +340,9 @@ rfc3986[idna2008]==1.5.0
# via httpx
rsa==4.7.2
# via google-auth
s3transfer==0.5.0
s3fs==2021.8.0
# via -r requirements.in
s3transfer==0.4.2
# via boto3
six==1.16.0
# via
......@@ -385,6 +390,8 @@ urllib3==1.26.6
# requests
uvicorn==0.14.0
# via -r requirements.in
wrapt==1.12.1
# via aiobotocore
xmltodict==0.11.0
# via osdu-log-recognition-lib
yarl==1.6.3
......
......@@ -44,7 +44,6 @@ rm -rf test-reports/
mkdir test-reports
cd integration
acl_domain='example.com'
legal_tag='opendes-sdmstestlegaltag'
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment