Commit 07cbeddf authored by Yunhua Koglin's avatar Yunhua Koglin
Browse files

clean up

parent 1be63d23
......@@ -12,5 +12,5 @@
# See the License for the specific language governing permissions and
# limitations under the License.
__version__ = '2.0.2'
__version__ = '0.0.2'
......@@ -14,9 +14,6 @@
from typing import Optional
# import s3fs
# import fsspec
from osdu.core.api.storage.dask_storage_parameters import DaskStorageParameters
from osdu.core.api.storage.tenant import Tenant
from .storage_aws import AwsStorage
......@@ -24,13 +21,9 @@ from .storage_aws import AwsStorage
async def get_dask_storage_parameters(tenant: Tenant, directory: Optional[str] = None) -> DaskStorageParameters:
storage = AwsStorage(session=None,service_account_file=directory)
await storage.getCredentials(tenant)
# storage._access_key = credentials['AccessKeyId']
# storage._secret_key = credentials['SecretAccessKey']
# storage._session_token = credentials['SessionToken']
base_directory = storage._bucket_name+"/"+tenant.bucket_name
storage_options={'anon': False, 'use_ssl': True, 'key':storage._access_key, 'secret':storage._secret_key, 'token':storage._session_token}
print("here is the test\n\n\n")
return DaskStorageParameters(protocol='s3',
base_directory=base_directory,
storage_options=storage_options)
\ No newline at end of file
......@@ -166,6 +166,7 @@ class AwsStorage(BlobStorageBase):
timeout: int = 30, **kwargs) -> Blob:
await self.getCredentials(tenant)
bucket = tenant.bucket_name
# bucket here is in fact, a folder in the seismic ddms bucket. Same for other operations (download, delete...)
object_name = f"{bucket}/{object_name}"
s3_client = boto3.client(
's3',
......@@ -195,9 +196,6 @@ class AwsStorage(BlobStorageBase):
Bucket=self._bucket_name,
Key=object_name
)
print("download metadata\n")
print(metadata)
print("end of download_metadata\n\n")
return Blob(identifier=object_name,
bucket=tenant.bucket_name,
......@@ -217,10 +215,6 @@ class AwsStorage(BlobStorageBase):
await self.getCredentials(tenant)
bucket = tenant.bucket_name
object_name = f"{bucket}/{object_name}"
# encoded_object_name = quote(object_name, safe='')
print("delete_object_name_in_storage: \n\n")
print(object_name)
s3_client = boto3.client(
's3',
......@@ -249,9 +243,6 @@ class AwsStorage(BlobStorageBase):
aws_secret_access_key=self._secret_key,
aws_session_token=self._session_token
)
# outfile = io.BytesIO()
# s3_client.download_fileobj(bucket, object_name, outfile)
# return outfile.getvalue()
obj = s3_client.get_object(Bucket=self._bucket_name, Key=object_name)
return obj['Body'].read()
......@@ -274,10 +265,6 @@ class AwsStorage(BlobStorageBase):
Bucket=self._bucket_name,
Key=object_name
)
print("download metadata\n")
print(metadata)
print("end of download_metadata\n\n")
return Blob(identifier=object_name,
bucket=bucket,
name=metadata.get('name', object_name),
......@@ -304,11 +291,7 @@ class AwsStorage(BlobStorageBase):
if max_result is not None:
kwargs['MaxKeys'] = max_result
kwargs['Prefix'] = prefix_name
print("list_objects_prefix_is \n\n")
print(prefix_name)
print("end_list_objects\n\n")
s3_client = boto3.client(
's3',
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment