Commit 0a23016f authored by Yannick's avatar Yannick
Browse files

merge update into upload

parent 7b13585d
......@@ -147,13 +147,36 @@ class AzureAioBlobStorage(BlobStorageBase):
provider_specific=properties
)
async def _upload(self, tenant: Tenant,
object_name: str,
file_data: Any,
overwrite: bool,
content_type: str = None,
metadata: dict = None,
**kwargs) -> Blob:
async def upload(self, tenant: Tenant, object_name: str, file_data: Any, *,
overwrite: bool = True,
if_match=None,
if_not_match=None,
auth: Optional = None, content_type: str = None, metadata: dict = None,
timeout: int = 30, **kwargs) -> Blob:
"""
upload blob data, fail if already exist
:param tenant: tenant info
:param object_name: maps to file name
:param file_data: Any
:param overwrite: if False, will fail if object already exist. If True, will replace if exist.(Default=True)
:param if_match: (ETag value) update will fail if the blob to overwrite doesn't match the ETag provided.
Cannot be used with `if_not_match`. It expects ETag value. ETag can be get using `download_metadata`
or in response of an upload.
:param if_not_match: (ETag value) update will fail if the blob to overwrite matches the ETag provided.
Cannot be used with `if_match`. It expects ETag value. ETag can be get using `download_metadata` or
in response of an upload.
:param auth: Optional = None,
:param content_type: str = None,
:param metadata: dict = None,
:param timeout: UNUSED
:return: blob
"""
assert not (if_match and if_not_match), "if_match and if_not_match cannot be set simultaneous"
conditions = {}
if if_match or if_not_match:
conditions['etag'] = if_match or if_not_match
conditions['match_condition'] = MatchConditions.IfNotModified if if_match else MatchConditions.IfModified
container = tenant.bucket_name
blob_service_client = await self._get_blob_service_client(tenant)
async with blob_service_client:
......@@ -164,7 +187,7 @@ class AzureAioBlobStorage(BlobStorageBase):
overwrite=overwrite,
metadata=metadata,
content_settings=content_settings,
**kwargs)
**conditions)
return Blob(identifier=object_name,
bucket=container,
name=upload_response.get('name', object_name),
......@@ -175,53 +198,4 @@ class AzureAioBlobStorage(BlobStorageBase):
time_updated=str(upload_response.get('last_modified', '')),
size=upload_response.get('size', -1),
etag=upload_response.get('etag', None),
provider_specific=upload_response
)
async def update(self, tenant: Tenant, object_name: str, file_data: Any, *,
if_match=None,
if_not_match=None,
auth: Optional = None,
content_type: str = None,
metadata: dict = None,
timeout: int = 30,
**kwargs) -> Blob:
"""
update blob data, create if not exist, overwrite if exist. Use `if_match` and `if_not_match` for conditional
update. `if_match` and `if_not_match` cannot be set simultaneously and are expected to work with the ETag
that can be get using `download_metadata`.
:param tenant: tenant info
:param object_name: maps to file name
:param file_data: Any, *,
:param if_match: (ETag value) update will fail if the blob to overwrite doesn't match the ETag provided.
:param if_not_match: (ETag value) update will fail if the blob to overwrite matches the ETag provided.
:param auth: UNUSED,
:param content_type: str = None,
:param metadata: dict = None,
:param timeout: UNUSED
:return: blob
"""
assert not (if_match and if_not_match), "if_match and if_not_match cannot be set simultaneous"
conditions = {}
if if_match or if_not_match:
conditions['etag'] = if_match or if_not_match
conditions['match_condition'] = MatchConditions.IfNotModified if if_match else MatchConditions.IfModified
return await self._upload(tenant, object_name, file_data, True, content_type, metadata, **conditions)
async def upload(self, tenant: Tenant, object_name: str, file_data: Any,
*args, auth: Optional = None, content_type: str = None, metadata: dict = None,
timeout: int = 30, **kwargs) -> Blob:
"""
upload blob data, fail if already exist
:param tenant: tenant info
:param object_name: maps to file name
:param file_data: Any, *,
:param auth: Optional = None,
:param content_type: str = None,
:param metadata: dict = None,
:param timeout: UNUSED
:return: blob
"""
return await self._upload(tenant, object_name, file_data, False, content_type, metadata)
provider_specific=upload_response)
......@@ -54,30 +54,33 @@ async def test_download_metadata(az_client: AzureAioBlobStorage, test_tenant):
@pytest.mark.asyncio
async def test_update_with_condition(az_client: AzureAioBlobStorage, test_tenant):
async def test_overwrite_with_condition(az_client: AzureAioBlobStorage, test_tenant):
blob_name = 'testing_data/' + str(uuid.uuid4())
await az_client.upload(test_tenant, blob_name, b'1111')
with pytest.raises(AzureError): # StorageErrorException is internal
await az_client.upload(test_tenant, blob_name, b'1111', overwrite=False)
# update no condition
await az_client.update(test_tenant, blob_name, b'1112')
await az_client.upload(test_tenant, blob_name, b'1112')
assert await az_client.download(test_tenant, blob_name) == b'1112'
# successful update if_match
etag_1112 = (await az_client.download_metadata(test_tenant, blob_name)).etag
await az_client.update(test_tenant, blob_name, b'1113', if_match=etag_1112)
await az_client.upload(test_tenant, blob_name, b'1113', if_match=etag_1112)
assert await az_client.download(test_tenant, blob_name) == b'1113'
# should fail update if_match not satisfied
with pytest.raises(AzureError): # StorageErrorException is internal
await az_client.update(test_tenant, blob_name, b'1114', if_match=etag_1112)
await az_client.upload(test_tenant, blob_name, b'1114', if_match=etag_1112)
# success update if_not_match
await az_client.update(test_tenant, blob_name, b'1115', if_not_match=etag_1112)
await az_client.upload(test_tenant, blob_name, b'1115', if_not_match=etag_1112)
# should fail update if_not_match not satisfied
etag_1115 = (await az_client.download_metadata(test_tenant, blob_name)).etag
with pytest.raises(AzureError): # StorageErrorException is internal
await az_client.update(test_tenant, blob_name, b'1116', if_not_match=etag_1115)
await az_client.upload(test_tenant, blob_name, b'1116', if_not_match=etag_1115)
@pytest.mark.asyncio
......@@ -91,7 +94,7 @@ async def test_concurrent_update_only_one_should_succeed(az_client: AzureAioBlob
for _ in range(nb_repetition):
etag = (await az_client.download_metadata(test_tenant, blob_name)).etag
calls = [az_client.update(test_tenant, blob_name, str(c), if_match=etag) for c in range(concurrency)]
calls = [az_client.upload(test_tenant, blob_name, str(c), if_match=etag) for c in range(concurrency)]
# perform several concurrent call
result = await asyncio.gather(*calls, return_exceptions=True)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment