Commit 76e2bce2 authored by Yunhua Koglin's avatar Yunhua Koglin
Browse files

update

parent fb61582d
......@@ -12,5 +12,5 @@
# See the License for the specific language governing permissions and
# limitations under the License.
__version__ = '2.0.3'
__version__ = '2.0.4'
......@@ -20,7 +20,8 @@ from osdu.core.api.storage.blob_storage_base import BlobStorageBase
from osdu.core.api.storage.blob import Blob
from osdu.core.api.storage.tenant import Tenant
import boto3
from datetime import datetime, timedelta, timezone
from botocore.exceptions import ClientError, WaiterError
try:
import ujson as json
except ImportError:
......@@ -46,6 +47,7 @@ class AwsStorage(BlobStorageBase):
self._secret_key = ""
self._session_token = ""
self._bucket_name =""
self._timestamp =datetime.now(timezone.utc)
def getPolicy(self, bucket: str, keypath:str):
""" policy for access keypath folder in bucket """
......@@ -122,7 +124,8 @@ class AwsStorage(BlobStorageBase):
async def getCredentials(self, tenant: Tenant):
if len(self._session_token) > 2 and datetime.now(timezone.utc) < self._timestamp:
return
""" get credentials to access s3 bucket for tenant's folder """
# all tenants in one bucket(self._bucket_name). Each tenant.bucket_name is in fact, a folder in self._bucket_name
folder = tenant.bucket_name
......@@ -152,6 +155,8 @@ class AwsStorage(BlobStorageBase):
self._access_key = credentials['AccessKeyId']
self._secret_key = credentials['SecretAccessKey']
self._session_token = credentials['SessionToken']
timeexpire = credentials['Expiration']
self._timestamp = timeexpire - timedelta(seconds = 30) # expire 30 sec before it should
async def upload(self, tenant: Tenant, object_name: str, file_data: Any, *,
overwrite: bool = True,
......@@ -184,25 +189,29 @@ class AwsStorage(BlobStorageBase):
extra['etag'] = if_match or if_not_match
if len(extra)==0:
response = s3_client.upload_fileobj(stream_data, self._bucket_name,object_name)
await s3_client.upload_fileobj(stream_data, self._bucket_name,object_name)
else:
response = s3_client.upload_fileobj(stream_data, self._bucket_name, object_name,ExtraArgs=extra)
print("\nresponse:\n")
print(response)
print("end of response\n\n")
return Blob(identifier=object_name,
bucket=tenant.bucket_name,
name=object_name,
metadata=metadata,
acl=auth,
content_type=content_type,
time_created=str(''),
time_updated=str(''),
size=-1,
etag=extra.get('etag', None),
provider_specific=response)
await s3_client.upload_fileobj(stream_data, self._bucket_name, object_name,ExtraArgs=extra)
metadata = s3_client.head_object(
Bucket=self._bucket_name,
Key=object_name
)
print("download metadata\n")
print(metadata)
print("end of download_metadata\n\n")
# return response
return Blob(identifier=object_name,
bucket=tenant.bucket_name,
name=metadata.get('name', object_name),
metadata=metadata,
acl=metadata.get('acl', auth),
content_type=metadata.get('ContentType', content_type),
time_created=metadata.get('timeCreated', None),
time_updated=metadata.get('LastModified', None),
size=metadata.get('ContentLength', -1),
etag=metadata.get('Etag', extra['etag'])
)
async def delete(self, tenant: Tenant, object_name: str,
*, auth: Optional = None, timeout: int = 10, **kwargs):
......@@ -219,7 +228,7 @@ class AwsStorage(BlobStorageBase):
)
response = s3_client.delete_object(
Bucket=self._bucket_name,
Key=encoded_object_name,
Key=encoded_object_name
)
return response
......@@ -260,13 +269,13 @@ class AwsStorage(BlobStorageBase):
metadata = s3_client.head_object(
Bucket=self._bucket_name,
Key=object_name,
Key=object_name
)
print("download metadata\n")
print(metadata)
print("end of download_metadata\n\n")
return Blob(identifier=metadata.get('ETag', None),
return Blob(identifier=object_name,
bucket=bucket,
name=metadata.get('name', object_name),
metadata=metadata,
......@@ -274,7 +283,8 @@ class AwsStorage(BlobStorageBase):
content_type=metadata.get('ContentType', None),
time_created=metadata.get('timeCreated', None),
time_updated=metadata.get('LastModified', None),
size=metadata.get('ContentLength', -1)
size=metadata.get('ContentLength', -1),
etag=metadata.get('Etag', None)
)
async def list_objects(self, tenant: Tenant, *args,
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment