Commit c7ed0d13 authored by Paromita Mitra's avatar Paromita Mitra
Browse files

added more storage options for s3 compatible services

parent 208ca2ff
Pipeline #64851 passed with stage
in 1 minute and 13 seconds
from typing import Optional
import os
from typing import Optional, List, Any
from osdu.core.api.storage.dask_storage_parameters import DaskStorageParameters
from osdu.core.api.storage.tenant import Tenant
from .blob_storage_factory_ibm import IBMBlobStorageFactory
async def get_dask_storage_parameters(tenant: Tenant, directory: Optional[str] = None) -> DaskStorageParameters:
storage = IBMBlobStorageFactory()
access_key = storage._access_key
secret_key = storage._secret_key
endpoint_url = storage._endpointURL
print("minio endpoint url ", endpoint_url)
base_directory = f'{tenant.bucket_name}/{directory}' if directory else tenant.bucket_name
storage_options = {"key": access_key, "secret": secret_key}
return DaskStorageParameters(protocol='https',
base_directory=base_directory,
storage_options=storage_options)
class DaskStorageParametersFactoryIBM:
_endpointURL = None
_access_key = None
_secret_key = None
_region = None
_bucket_name_prefix = None
_signature_version = 's3v4'
def __init__(self, session=None, service_account_file: Optional[str] = None):
self._endpointURL = os.getenv("COS_URL", "NA")
self._access_key = os.getenv("COS_ACCESS_KEY", "NA")
self._secret_key = os.getenv("COS_SECRET_KEY", "NA")
self._region = os.getenv("COS_REGION", "us-east-1")
self._bucket_name_prefix = os.getenv("BUCKET_PREFIX", "dev")
async def get_dask_storage_parameters(self, tenant: Tenant,
directory: Optional[str] = None) -> DaskStorageParameters:
access_key = self._access_key
print("minio access key", access_key)
secret_key = self._secret_key
print("minio secret key", secret_key)
endpointurl = self._endpointURL
signature_version = self._signature_version
print("minio endpoint url ", endpointurl)
base_directory = f'{tenant.bucket_name}/{directory}' if directory else tenant.bucket_name
print("minio base_directory", base_directory)
storage_options = {"anon": False, "use_ssl": True, "key": access_key, "secret": secret_key, "client_kwargs": {"endpoint_url": endpointurl}, "config_kwargs": {"s3": {'signature_version': signature_version}}}
return DaskStorageParameters(protocol='s3',
base_directory=base_directory,
storage_options=storage_options)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment