Commit 61294bac authored by Yannick's avatar Yannick Committed by Luc Yriarte
Browse files

update accordingly to core changes

parent c0c199ea
Pipeline #32288 failed with stage
in 25 seconds
......@@ -129,6 +129,22 @@ class GCloudAioStorage(BlobStorageBase):
bucket = tenant.bucket_name
return await self._download(project, bucket, object_name, auth=auth, timeout=timeout, params={'alt': 'media'})
def metadict_to_blob(self, metadata: dict) -> Blob:
# using 'generation' instead of ETag since 'If-Match' and 'If-None-Match' doesn't seems to work as documented
# here https://cloud.google.com/storage/docs/json_api/v1/parameters#ifmatch
return Blob(identifier=metadata.get('id', None),
bucket=metadata.get('bucket', None),
name=metadata.get('name', None),
metadata=metadata,
acl=metadata.get('acl', None),
content_type=metadata.get('contentType', None),
time_created=metadata.get('timeCreated', None),
time_updated=metadata.get('updated', None),
size=metadata.get('size', -1),
etag=str(metadata.get('generation', '')) or None,
provider_specific=metadata)
async def download_metadata(self, tenant: Tenant, object_name: str,
*args, auth: Optional = None, timeout: int = 10, **kwargs) -> Blob:
"""
......@@ -144,32 +160,35 @@ class GCloudAioStorage(BlobStorageBase):
bucket = tenant.bucket_name
data = await self._download(project, bucket, object_name, auth=auth, timeout=timeout)
metadata: dict = json.loads(data.decode())
return Blob(identifier=metadata.get('id', None),
bucket=metadata.get('bucket', bucket),
name=metadata.get('name', object_name),
metadata=metadata,
acl=metadata.get('acl', None),
content_type=metadata.get('contentType', None),
time_created=metadata.get('timeCreated', None),
time_updated=metadata.get('updated', None),
size=metadata.get('size', -1)
)
return self.metadict_to_blob(metadata)
# TODO: if `metadata` is set, use multipart upload:
# https://cloud.google.com/storage/docs/json_api/v1/how-tos/upload
async def upload(self, tenant: Tenant, object_name: str, file_data: Any,
*args, auth: Optional = None, content_type: str = None, metadata: dict = None,
timeout: int = 30, **kwargs):
async def upload(self, tenant: Tenant, object_name: str, file_data: Any, *,
overwrite: bool = True,
if_match=None,
if_not_match=None,
auth: Optional = None,
content_type: str = None,
metadata: dict = None,
timeout: int = 30, **kwargs) -> Blob:
"""
upload blob data
:param tenant: tenant info
:param object_name: maps to file name
:param file_data: Any, *,
:param file_data: Any,
:param overwrite: if False, will fail if object already exist. If True, will replace if exist. (Default = True)
:param if_match: (ETag value) update will fail if the blob to overwrite doesn't match the ETag provided.
Cannot be used with `if_not_match`. It expects ETag value. ETag can be get using `download_metadata` or
in response of an upload or update.
:param if_not_match: (ETag value) update will fail if the blob to overwrite matches the ETag provided.
Cannot be used with `if_match`. It expects ETag value. ETag can be get using `download_metadata` or
in response of an upload or update.
:param auth: Optional = None,
:param content_type: str = None,
:param metadata: dict = None,
:param timeout: int = 30, **kwargs
:param return: blob id
:param timeout: int = 30
:return: blob
"""
project = tenant.project_id
bucket = tenant.bucket_name
......@@ -182,7 +201,18 @@ class GCloudAioStorage(BlobStorageBase):
content_type = content_type or mimetypes.guess_type(object_name)[0]
parameters = {}
headers = {}
headers = kwargs.get('headers', {})
if not overwrite: # Upon google documentation
parameters['ifGenerationMatch'] = '0'
assert not(if_match and if_not_match), "if_match and if_not_match cannot be set simultaneous"
# Use 'generation' instead of Etag and 'If-Match' and 'If-None-Match' header, this is transparent for the caller
if if_match:
parameters['ifGenerationMatch'] = str(if_match)
if if_not_match:
parameters['ifGenerationNotMatch'] = str(if_not_match)
headers.update({
'Content-Length': str(content_length),
'Content-Type': content_type or '',
......@@ -194,12 +224,14 @@ class GCloudAioStorage(BlobStorageBase):
if upload_type == UploadType.SIMPLE:
# if metadata:
# log.warning('metadata will be ignored for upload_type=Simple')
return await self._upload_simple(project, bucket, url, object_name, stream, parameters, headers,
upload_response = await self._upload_simple(project, bucket, url, object_name, stream, parameters, headers,
auth=auth, timeout=timeout)
return self.metadict_to_blob(upload_response)
if upload_type == UploadType.RESUMABLE:
return await self._upload_resumable(project, bucket, url, object_name, stream, parameters, headers,
upload_response = await self._upload_resumable(project, bucket, url, object_name, stream, parameters, headers,
auth=auth, metadata=metadata, timeout=timeout)
return self.metadict_to_blob(upload_response)
raise TypeError(f'upload type {upload_type} not supported')
......
# osdu core lib main python
# osdu-core-lib-python>=0.4.0, <0.5
# for google provider
aiohttp
cryptography
......
# osdu core lib main python
--extra-index-url \
https://community.opengroup.org/api/v4/projects/465/packages/pypi/simple/
osdu-core-lib-python>=0.4.0, <0.5
osdu-core-lib-python==1.0.0.dev270686
......@@ -7,9 +7,11 @@ import aiohttp
import uuid
from osdu.core.api.storage.tenant import Tenant
class _TESTING_CFG:
credentials = TESTING_GCP_DATA_PROJECT_CREDENTIALS
TEST_DATA = {
'initial_files': [
# object_name , content
......@@ -31,7 +33,9 @@ async def storage_client(request):
@pytest.fixture
async def test_tenant():
return Tenant(project_id=TESTING_GCP_DATA_PROJECT_ID, bucket_name='testing-osdu-core', data_partition_id='testing-partition-name')
return Tenant(project_id=TESTING_GCP_DATA_PROJECT_ID, bucket_name='testing-osdu-core',
data_partition_id='testing-partition-name')
@pytest.fixture
async def temp_directory() -> str:
......@@ -41,6 +45,7 @@ async def temp_directory() -> str:
# teardown - recursively delete the tmp directory
shutil.rmtree(tmpdir, ignore_errors=True)
@pytest.mark.asyncio
async def test_list_objects(storage_client, test_tenant):
result = await storage_client.list_objects(test_tenant)
......@@ -88,7 +93,8 @@ async def test_upload_file_bin_input(storage_client, temp_directory, test_tenant
content_bin = b'expected content 123456789'
with open(file_c, 'rb') as file_bin_input:
await storage_client.upload(test_tenant, 'file_bin_input', file_bin_input)
assert await storage_client.download(test_tenant, 'file_bin_input') == content_bin
assert await storage_client.download(test_tenant, 'file_bin_input') == content_bin
@pytest.mark.asyncio
async def test_upload_file_txt_input(storage_client, temp_directory, test_tenant):
......@@ -98,20 +104,23 @@ async def test_upload_file_txt_input(storage_client, temp_directory, test_tenant
content_bin = b'expected content 123456789'
with open(file_c, 'r') as file_txt_input:
await storage_client.upload(test_tenant, 'file_txt_input', file_txt_input)
assert await storage_client.download(test_tenant, 'file_txt_input') == content_bin
assert await storage_client.download(test_tenant, 'file_txt_input') == content_bin
@pytest.mark.asyncio
async def test_upload_str_input(storage_client, test_tenant):
content_bin = b'expected content 123456789'
content_str = content_bin.decode('utf-8')
await storage_client.upload(test_tenant, 'str_input', content_str)
assert await storage_client.download(test_tenant, 'str_input') == content_bin
assert await storage_client.download(test_tenant, 'str_input') == content_bin
@pytest.mark.asyncio
async def test_upload_bin_input(storage_client, test_tenant):
content_bin = b'expected content 123456789'
await storage_client.upload(test_tenant, 'bin_input', content_bin)
assert await storage_client.download(test_tenant, 'bin_input') == content_bin
assert await storage_client.download(test_tenant, 'bin_input') == content_bin
@pytest.mark.asyncio
async def test_upload_empty_input(storage_client, test_tenant):
......@@ -119,14 +128,38 @@ async def test_upload_empty_input(storage_client, test_tenant):
actual_data = await storage_client.download(test_tenant, 'empty_input')
assert len(actual_data) == 0
@pytest.mark.asyncio
async def test_upload_int_input(storage_client, test_tenant):
with pytest.raises(TypeError):
await storage_client.upload(test_tenant, 'int_input', 123456)
@pytest.mark.asyncio
async def test_overwrite_with_condition(storage_client, test_tenant):
blob_name = 'testing_data/' + str(uuid.uuid4())
await storage_client.upload(test_tenant, blob_name, b'1111')
with pytest.raises(Exception): # StorageErrorException is internal
await storage_client.upload(test_tenant, blob_name, b'1111', overwrite=False)
# update no condition
await storage_client.upload(test_tenant, blob_name, b'1112')
assert await storage_client.download(test_tenant, blob_name) == b'1112'
# successful update if_match
etag_1112 = (await storage_client.download_metadata(test_tenant, blob_name)).etag
await storage_client.upload(test_tenant, blob_name, b'1113', if_match=etag_1112)
assert await storage_client.download(test_tenant, blob_name) == b'1113'
# should fail update if_match not satisfied
with pytest.raises(Exception): # StorageErrorException is internal
await storage_client.upload(test_tenant, blob_name, b'1114', if_match=etag_1112)
# success update if_not_match
await storage_client.upload(test_tenant, blob_name, b'1115', if_not_match=etag_1112)
# should fail update if_not_match not satisfied
etag_1115 = (await storage_client.download_metadata(test_tenant, blob_name)).etag
with pytest.raises(Exception): # StorageErrorException is internal
await storage_client.upload(test_tenant, blob_name, b'1116', if_not_match=etag_1115)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment