Commit dddf3b2e authored by Yunhua Koglin's avatar Yunhua Koglin
Browse files

put ibm back

parent d2777480
......@@ -201,7 +201,6 @@ class DaskBulkStorage:
we should be able to change or support other format easily ?
schema={} instead of 'infer' fixes wrong inference for columns of type string starting with nan values
"""
get_logger().info(f"_save_with_dask, path : {path}")
return self._submit_with_trace(dd.to_parquet, ddf, path,
schema={},
engine='pyarrow',
......
......@@ -32,12 +32,12 @@ async def resolve_tenant(data_partition_id: str) -> Tenant:
bucket_name='wdms-osdu'
)
# if Config.cloud_provider.value == 'ibm':
# return Tenant(
# data_partition_id=data_partition_id,
# project_id=Config.default_data_tenant_project_id.value,
# bucket_name='logstore-osdu-ibm'
# )
if Config.cloud_provider.value == 'ibm':
return Tenant(
data_partition_id=data_partition_id,
project_id=Config.default_data_tenant_project_id.value,
bucket_name='logstore-osdu-ibm'
)
if Config.cloud_provider.value == 'aws':
return Tenant(
......
......@@ -318,13 +318,13 @@ def cloud_provider_additional_environment(config: ConfigurationContainer):
override=True,
validator=validator_path_must_exist)
# if provider == 'ibm':
# config.add_from_env(attribute_name='default_data_tenant_project_id',
# env_var_key='OS_WELLBORE_DDMS_DATA_PROJECT_ID',
# description='IBM data tenant ID',
# default='logstore-ibm',
# is_mandatory=True,
# override=True)
if provider == 'ibm':
config.add_from_env(attribute_name='default_data_tenant_project_id',
env_var_key='OS_WELLBORE_DDMS_DATA_PROJECT_ID',
description='IBM data tenant ID',
default='logstore-ibm',
is_mandatory=True,
override=True)
if provider == 'aws':
config.add_from_env(attribute_name='aws_region',
env_var_key='AWS_REGION',
......
from osdu.core.api.storage.blob_storage_base import BlobStorageBase
from osdu_ibm.storage.blob_storage_ibm import IBMObjectStorage
from app.utils import get_http_client_session
from app.utils import Context
from .app_injector import AppInjector, AppInjectorModule
from app.bulk_persistence import resolve_tenant
from app.bulk_persistence.dask.dask_bulk_storage import DaskBulkStorage
from osdu_ibm.storage.dask_storage_parameters import get_dask_storage_parameters as ibm_parameters
class IBMInjector(AppInjectorModule):
def configure(self, app_injector: AppInjector):
app_injector.register(BlobStorageBase, IBMInjector.build_ibm_blob_storage)
app_injector.register(DaskBulkStorage, IBMInjector.build_ibm_dask_blob_storage)
@staticmethod
async def build_ibm_blob_storage(*args, **kwargs) -> BlobStorageBase:
ctx: Context = Context.current()
# TODO to be reviewed
tenant = await resolve_tenant(ctx.partition_id)
return IBMObjectStorage(
session=get_http_client_session(),
service_account_file=tenant.credentials
)
@staticmethod
async def build_ibm_dask_blob_storage() -> DaskBulkStorage:
# raise NotImplementedError()
ctx: Context = Context.current()
tenant = await resolve_tenant(ctx.partition_id)
params = await ibm_parameters(tenant)
return await DaskBulkStorage.create(params)
......@@ -23,7 +23,7 @@ from .app_injector import AppInjector, AppInjectorModule, WithLifeTime
from app.injector.az_injector import AzureInjector
from app.injector.aws_injector import AwsInjector
from app.injector.gcp_injector import GCPInjector
# from app.injector.ibm_injector import IBMInjector
from app.injector.ibm_injector import IBMInjector
from app.clients import StorageRecordServiceClient
from app.clients.storage_service_blob_storage import StorageRecordServiceBlobStorage
......@@ -64,9 +64,9 @@ class MainInjector(AppInjectorModule):
logger.info('using gcp injector')
GCPInjector().configure(app_injector)
# if Config.cloud_provider.value == 'ibm':
# logger.info('using ibm injector')
# IBMInjector().configure(app_injector)
if Config.cloud_provider.value == 'ibm':
logger.info('using ibm injector')
IBMInjector().configure(app_injector)
if Config.cloud_provider.value == 'aws':
logger.info('using aws injector')
......
......@@ -32,9 +32,8 @@ osdu-log-recognition-lib>=0.0.9
osdu-data-ecosystem-storage~=1.2.0
osdu-data-ecosystem-search>=0.3.2, <0.4
osdu-core-lib-python-ibm~=1.2.0
osdu-core-lib-python-gcp~=1.1.0
osdu-core-lib-python-azure~=1.2.1
osdu-core-lib-python-aws>=0.0.2, <0.1
osdu-core-lib-python-aws>=0.0.1, <0.1
osdu-core-lib-python~=1.1.0
boto3
s3fs
\ No newline at end of file
fastapi>=0.65.1
aiohttp==3.7.4.post0
cryptography>=2.7
pyarrow>=0.15.0
pandas==1.2.4
uvicorn
Click
backoff
structlog
python-rapidjson
python-multipart
jsonpath-ng # TODO: remove when bulk_path no longer supported
jsonbender==0.9.3
python-ulid
opencensus
opencensus-ext-stackdriver
opencensus-ext-azure
opencensus-ext-ocagent
opencensus-ext-logging
#
# This file is autogenerated by pip-compile with python 3.9
# To update, run:
#
# pip-compile requirements.in
#
--index-url https://aws:eyJ2ZXIiOjEsImlzdSI6MTYzMDUxNTYyNiwiZW5jIjoiQTEyOEdDTSIsInRhZyI6ImZ5UFlYYnZhRkhOM1dNdmVpNkc3dlEiLCJleHAiOjE2MzA1NTg4MjYsImFsZyI6IkExMjhHQ01LVyIsIml2IjoiZUVfazRKTW5aRlp3TDg1diJ9.Y1iu6isjfagWbL9b4ENdcA.jQxd_AbMvL9kQfjm.2Tu8K3TDWU5eqpzV11gx3J129Du1MqP5M_WLKwboVyB1YWDoFOr2C4ScG9lCRgcAQ8scS4cenTq3UgUf47-q7qgHSwlufAPN6Nx333LkFzO_M8VeFot6Lkoa9IP2o9QDY74TRWPh5hDnRJtWD1VIfFsVf7AitjCjF3xspuCwaX1KR0_LhnI6KecomjL77U77I_yKmPXwELBLOKO-hkN1xHM8g3r09zDjce1Npb_4lioy4MKEHuLAMUgTGRr3XF1W1FA6M5rEezV09A_J-NE_zZHBrshFykg8ryb-Of05obpRb3Hd2xcuawIFbNogLuqoy6JZ2pg514YCL6kZIqvACFKHF9dUIHHZuM_VkWLYTmflaG_Bb1mpgqvf1Ls1J1vdASgzqMaY7lGqqdTl98PkOgRrqwmFoJa-IfCYolgM1GyU-YPwSWOk_Vefmc5EZ81Q8v94PJz0BuUxCibccaLN1Jf53etog5k-on7typcj5eUi2xpAFcmllle7x396PT0-UasKeN6bls37nI86neYtkLAMN6fvwFztOYNBivvE4iiYHMoqV8yqXPBE6cvelMK7qd7O-4EHJQDWrvEsIrsL9g072bh1F9ax9ji7AZ8UK8LxS08blHBQTcCeEWTuigTdzZ71ob73P1P0cEvZpqlJHec0dw34d8Kz4gP5vSgEJn3LRbXgXOweJaJBFSZ5R8NidqUDjz6NGz66r4bwKfrx9Gwr_GdTo1rWamHhO0bwaaCbenaIpnCe6fF4gZFBrxsDbmZzxMUrB1Z_e6OFyVRlk6O0jJ5qKeOsqh9RLb0wyVcp1g7dVFGVkqOe6UawYthfpq5Snzxmj0JkuuRi1u5xce5KlizGhxzoQZRwWHy0CuucCIrvcicBvKqKzD0C-BR0Sf_QbMIYPN_Jxocf334HeljN-aSM_KsCoQ2KYSKa-YxOfqGy1_PtrBMw05gvnbTUdMu3PWa2e7MaKnyhUHjDNfzaFitCOjxCDQ62q_zTIlA6TgUNDTdGDGKXhsPBJSFANIa4SP0urePXEUWM-3acisZjuSYSSbMxtrnVaQUD5dm8OfPfFA4n_WBA_RWnxETykHk0qEIkiuMt9_9hJQChwo0PFIRGDTgv48ClEYt2VFT1QTEsOYbc-lcMJ3Mx-YNCxyPm0Yb5NdPJcTBV2WjJnNijGTjwk0Ko4mjFidUo51NPICqDtRLZqf-BipDyAQtKQie-D9OeM_Rx_nKEuE6MVD1m6K1Kb3CaHbm-dV2RM7Qv-5YKlYR_rW5ei9PVF3DiNOXLaPguTM8W962tIjM71zM4Oc_PSb7ze8ol89Boknya-3LqV7ihdIJ2IWY7FKHOWfW8UqPfM0Vox6esu1LnW3Hu4NjUFu76.g75g14el7A4k-hChhbHG1Q@osdu-dev-888733619319.d.codeartifact.us-east-1.amazonaws.com/pypi/osdu-python/simple/
--extra-index-url https://community.opengroup.org/api/v4/projects/465/packages/pypi/simple/
# for chunking feature
dask[distributed]==2021.4.1
adal==1.2.7
# via azure-datalake-store
adlfs==0.7.7
# via osdu-core-lib-python-azure
aiohttp==3.7.4.post0
# via
# -r requirements.in
# adlfs
# gcsfs
# osdu-core-lib-python
# osdu-core-lib-python-gcp
aioredis==1.3.1
# via osdu-core-lib-python
anyio==3.2.1
# via httpcore
asgiref==3.4.1
# via uvicorn
async-timeout==3.0.1
# via
# aiohttp
# aioredis
attrs==21.2.0
# via
# aiohttp
# osdu-core-lib-python
azure-common==1.1.27
# via
# azure-keyvault-certificates
# azure-keyvault-keys
# azure-keyvault-secrets
azure-core==1.16.0
# via
# adlfs
# azure-identity
# azure-keyvault-certificates
# azure-keyvault-keys
# azure-keyvault-secrets
# azure-storage-blob
azure-datalake-store==0.0.52
# via adlfs
azure-identity==1.6.0
# via
# adlfs
# osdu-core-lib-python-azure
azure-keyvault==4.1.0
# via osdu-core-lib-python-azure
azure-keyvault-certificates==4.3.0
# via azure-keyvault
azure-keyvault-keys==4.4.0
# via azure-keyvault
azure-keyvault-secrets==4.3.0
# via azure-keyvault
azure-storage-blob==12.8.1
# via
# adlfs
# osdu-core-lib-python-azure
backoff==1.11.1
# via -r requirements.in
boto3==1.18.1
# via osdu-core-lib-python-ibm
botocore==1.21.1
# via
# boto3
# osdu-core-lib-python-ibm
# s3transfer
cachetools==4.2.2
# via
# google-auth
# osdu-core-lib-python-azure
certifi==2021.5.30
# via
# httpx
# msrest
# requests
cffi==1.14.6
# via
# azure-datalake-store
# cryptography
chardet==4.0.0
# via aiohttp
charset-normalizer==2.0.3
# via requests
click==8.0.1
# via
# -r requirements.in
# distributed
# uvicorn
cloudpickle==1.6.0
# via
# dask
# distributed
cryptography==3.4.7
# via
# -r requirements.in
# adal
# azure-identity
# azure-keyvault-keys
# azure-storage-blob
# msal
# osdu-core-lib-python
# osdu-core-lib-python-gcp
# pyjwt
dask[distributed]==2021.7.2
# via
# -r requirements.in
# distributed
decorator==5.0.9
# via
# gcsfs
# jsonpath-ng
distributed==2021.07.2
# via dask
fastapi==0.68.1
# via
# -r requirements.in
# osdu-data-ecosystem-search
# osdu-data-ecosystem-storage
fsspec==2021.7.0
# via
# -r requirements.in
# adlfs
# dask
# gcsfs
# osdu-core-lib-python-azure
gcsfs==2021.7.0
# via osdu-core-lib-python-gcp
google-api-core[grpc]==1.31.0
# via
# google-cloud-core
# google-cloud-monitoring
# google-cloud-trace
# opencensus
google-auth==1.33.0
# via
# gcsfs
# google-api-core
# google-auth-oauthlib
# google-cloud-core
google-auth-oauthlib==0.4.4
# via gcsfs
google-cloud-core==1.7.1
# via google-cloud-trace
google-cloud-monitoring==0.36.0
# via opencensus-ext-stackdriver
google-cloud-trace==0.24.0
# via opencensus-ext-stackdriver
googleapis-common-protos==1.53.0
# via google-api-core
grpcio==1.38.1
# via
# google-api-core
# opencensus-ext-ocagent
# opencensus-proto
h11==0.12.0
# via
# httpcore
# uvicorn
heapdict==1.0.1
# via zict
hiredis==2.0.0
# via aioredis
httpcore==0.13.6
# via httpx
httpx==0.18.2
# via
# osdu-data-ecosystem-search
# osdu-data-ecosystem-storage
idna==3.2
# via
# anyio
# requests
# rfc3986
# yarl
isodate==0.6.0
# via msrest
jmespath==0.10.0
# via
# boto3
# botocore
jsonbender==0.9.3
# via -r requirements.in
jsonpath-ng==1.5.3
# via -r requirements.in
locket==0.2.1
# via partd
msal==1.12.0
# via
# azure-identity
# msal-extensions
msal-extensions==0.3.0
# via azure-identity
msgpack==1.0.2
# via distributed
msrest==0.6.21
# via
# azure-keyvault-certificates
# azure-keyvault-keys
# azure-keyvault-secrets
# azure-storage-blob
multidict==5.1.0
# via
# aiohttp
# yarl
natsort==7.1.1
--extra-index-url \
https://community.opengroup.org/api/v4/projects/465/packages/pypi/simple/
osdu-log-recognition-lib>=0.0.9
osdu-data-ecosystem-storage~=1.2.0
osdu-data-ecosystem-search>=0.3.2, <0.4
# via -r requirements.in
numpy==1.21.1
# via
# pandas
# pyarrow
oauthlib==3.1.1
# via requests-oauthlib
opencensus==0.7.13
# via
# -r requirements.in
# opencensus-ext-azure
# opencensus-ext-logging
# opencensus-ext-ocagent
# opencensus-ext-stackdriver
opencensus-context==0.1.2
# via opencensus
opencensus-ext-azure==1.0.8
# via -r requirements.in
opencensus-ext-logging==0.1.0
# via -r requirements.in
opencensus-ext-ocagent==0.7.1
# via -r requirements.in
opencensus-ext-stackdriver==0.7.4
# via -r requirements.in
opencensus-proto==0.1.0
# via opencensus-ext-ocagent
osdu-core-lib-python==1.1.0
# via -r requirements.in
osdu-core-lib-python-aws==0.0.2
# via -r requirements.in
osdu-core-lib-python-azure==1.2.1
# via -r requirements.in
osdu-core-lib-python-gcp==1.1.0
# via -r requirements.in
osdu-core-lib-python-ibm==1.2.0
# via -r requirements.in
osdu-data-ecosystem-search==0.3.3
# via -r requirements.in
osdu-data-ecosystem-storage==1.2.0
# via -r requirements.in
osdu-log-recognition-lib==0.0.9
# via -r requirements.in
packaging==21.0
# via
# dask
# google-api-core
pandas==1.2.4
# via -r requirements.in
partd==1.2.0
# via dask
ply==3.11
# via jsonpath-ng
portalocker==1.7.1
# via msal-extensions
protobuf==3.17.3
# via
# google-api-core
# googleapis-common-protos
psutil==5.8.0
# via
# distributed
# opencensus-ext-azure
pyarrow==4.0.1
# via -r requirements.in
pyasn1==0.4.8
# via
# pyasn1-modules
# rsa
pyasn1-modules==0.2.8
# via google-auth
pycparser==2.20
# via cffi
pydantic==1.8.2
# via
# fastapi
# osdu-data-ecosystem-search
# osdu-data-ecosystem-storage
pyjwt[crypto]==2.1.0
# via
# adal
# msal
# osdu-core-lib-python
# osdu-core-lib-python-gcp
pyparsing==2.4.7
# via packaging
python-dateutil==2.8.2
# via
# adal
# botocore
# pandas
python-multipart==0.0.5
# via -r requirements.in
python-rapidjson==1.4
# via -r requirements.in
python-ulid==1.0.3
# via -r requirements.in
pytz==2021.1
# via
# google-api-core
# pandas
pyyaml==5.4.1
# via
# dask
# distributed
requests==2.26.0
# via
# adal
# azure-core
# azure-datalake-store
# gcsfs
# google-api-core
# msal
# msrest
# opencensus-ext-azure
# requests-oauthlib
requests-oauthlib==1.3.0
# via
# google-auth-oauthlib
# msrest
rfc3986[idna2008]==1.5.0
# via httpx
rsa==4.7.2
# via google-auth
s3transfer==0.5.0
# via boto3
six==1.16.0
# via
# azure-core
# azure-identity
# azure-keyvault-keys
# google-api-core
# google-auth
# google-cloud-core
# grpcio
# isodate
# jsonpath-ng
# protobuf
# python-dateutil
# python-multipart
sniffio==1.2.0
# via
# anyio
# httpcore
# httpx
sortedcontainers==2.4.0
# via distributed
starlette==0.14.2
# via fastapi
structlog==21.1.0
# via -r requirements.in
tblib==1.7.0
# via distributed
toolz==0.11.1
# via
# dask
# distributed
# partd
tornado==6.1
# via distributed
typing-extensions==3.10.0.0
# via
# aiohttp
# osdu-data-ecosystem-search
# osdu-data-ecosystem-storage
# pydantic
urllib3==1.26.6
# via
# botocore
# requests
uvicorn==0.14.0
# via -r requirements.in
xmltodict==0.11.0
# via osdu-log-recognition-lib
yarl==1.6.3
# via aiohttp
zict==2.0.0
# via distributed
osdu-core-lib-python-ibm>=0.0.1, <0.1
osdu-core-lib-python-gcp~=1.1.0
osdu-core-lib-python-azure~=1.1.1
osdu-core-lib-python-aws>=0.0.1, <0.1
osdu-core-lib-python~=1.1.0
s3fs==2021.8.0
s3fs
# The following packages are considered to be unsafe in a requirements file:
# setuptools
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment