Commit 8630cfbf authored by Yunhua Koglin's avatar Yunhua Koglin
Browse files

update

parent c7ff4725
......@@ -33,6 +33,10 @@ def set_default(obj):
return list(obj)
raise TypeError
MAX_CONTENT_LENGTH_SIMPLE_UPLOAD = 5 * 1024 * 1024 # 5 MB
class UploadType(enum.Enum):
SIMPLE = 1
RESUMABLE = 2
class AwsStorage(BlobStorageBase):
def __init__(self, session=None, service_account_file: Optional[str] = None):
......@@ -168,6 +172,7 @@ class AwsStorage(BlobStorageBase):
aws_secret_access_key=self._secret_key,
aws_session_token=self._session_token
)
stream_data = self._preprocess_data(file_data)
extra: dict = {}
if(content_type != None):
extra['ContentType'] = content_type
......@@ -177,9 +182,9 @@ class AwsStorage(BlobStorageBase):
extra['ACL'] = auth
if len(extra)==0:
response = s3_client.upload_fileobj(file_data, self._bucket_name,object_name)
response = s3_client.upload_fileobj(stream_data, self._bucket_name,object_name)
else:
response = s3_client.upload_fileobj(file_data, self._bucket_name, object_name,ExtraArgs=extra)
response = s3_client.upload_fileobj(stream_data, self._bucket_name, object_name,ExtraArgs=extra)
return response
async def delete(self, tenant: Tenant, object_name: str,
......@@ -217,9 +222,7 @@ class AwsStorage(BlobStorageBase):
)
outfile = io.BytesIO()
s3_client.download_fileobj(bucket, object_name, outfile)
outfile.seek(0)
return outfile
# return outfile.getvalue()
return outfile.getvalue()
# return obj['Body'].read()
async def download_metadata(self, tenant: Tenant, object_name: str,
......@@ -282,3 +285,22 @@ class AwsStorage(BlobStorageBase):
objects_list.append(key['Key'])
return objects_list
@staticmethod
def _preprocess_data(data: Any) -> io.IOBase:
if data is None:
return io.StringIO('')
if isinstance(data, bytes):
return io.BytesIO(data)
if isinstance(data, str):
return io.BytesIO(data.encode('utf-8'))
# if isinstance(data, io.TextIOWrapper):
# return io.BytesIO(data.encoding('utf-8'))
if isinstance(data, io.IOBase):
return data
raise TypeError(f'unsupported upload type: "{type(data)}"')
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment