update accordingly to core changes

osdu_gcp/storage/blob_storage_gcp.py

@@ -129,6 +129,22 @@ class GCloudAioStorage(BlobStorageBase):
         bucket = tenant.bucket_name
         return await self._download(project, bucket, object_name, auth=auth, timeout=timeout, params={'alt': 'media'})
 
+
+    def metadict_to_blob(self, metadata: dict) -> Blob:
+        # using 'generation' instead of ETag since 'If-Match' and 'If-None-Match' doesn't seems to work as documented
+        # here https://cloud.google.com/storage/docs/json_api/v1/parameters#ifmatch
+        return Blob(identifier=metadata.get('id', None),
+                    bucket=metadata.get('bucket', None),
+                    name=metadata.get('name', None),
+                    metadata=metadata,
+                    acl=metadata.get('acl', None),
+                    content_type=metadata.get('contentType', None),
+                    time_created=metadata.get('timeCreated', None),
+                    time_updated=metadata.get('updated', None),
+                    size=metadata.get('size', -1),
+                    etag=str(metadata.get('generation', '')) or None,
+                    provider_specific=metadata)
+
     async def download_metadata(self, tenant: Tenant, object_name: str,
                                 *args, auth: Optional = None, timeout: int = 10, **kwargs) -> Blob:
         """
@@ -144,32 +160,35 @@ class GCloudAioStorage(BlobStorageBase):
         bucket = tenant.bucket_name
         data = await self._download(project, bucket, object_name, auth=auth, timeout=timeout)
         metadata: dict = json.loads(data.decode())
-        return Blob(identifier=metadata.get('id', None),
-                    bucket=metadata.get('bucket', bucket),
-                    name=metadata.get('name', object_name),
-                    metadata=metadata,
-                    acl=metadata.get('acl', None),
-                    content_type=metadata.get('contentType', None),
-                    time_created=metadata.get('timeCreated', None),
-                    time_updated=metadata.get('updated', None),
-                    size=metadata.get('size', -1)
-                    )
+        return self.metadict_to_blob(metadata)
 
     # TODO: if `metadata` is set, use multipart upload:
     # https://cloud.google.com/storage/docs/json_api/v1/how-tos/upload
-    async def upload(self, tenant: Tenant, object_name: str, file_data: Any,
-                     *args, auth: Optional = None, content_type: str = None, metadata: dict = None,
-                     timeout: int = 30, **kwargs):
+    async def upload(self, tenant: Tenant, object_name: str, file_data: Any, *,
+                     overwrite: bool = True,
+                     if_match=None,
+                     if_not_match=None,
+                     auth: Optional = None,
+                     content_type: str = None,
+                     metadata: dict = None,
+                     timeout: int = 30, **kwargs) -> Blob:
         """
         upload blob data
         :param tenant: tenant info
         :param object_name: maps to file name
-        :param file_data: Any, *,
+        :param file_data: Any,
+        :param overwrite: if False, will fail if object already exist. If True, will replace if exist. (Default = True)
+        :param if_match: (ETag value) update will fail if the blob to overwrite doesn't match the ETag provided.
+              Cannot be used with `if_not_match`. It expects ETag value. ETag can be get using `download_metadata` or
+              in response of an upload or update.
+        :param if_not_match: (ETag value)  update will fail if the blob to overwrite matches the ETag provided.
+              Cannot be used with `if_match`. It expects ETag value. ETag can be get using `download_metadata` or
+              in response of an upload or update.
         :param auth: Optional = None,
         :param content_type: str = None,
         :param metadata: dict = None,
-        :param timeout: int = 30, **kwargs
-        :param return: blob id
+        :param timeout: int = 30
+        :return: blob
         """
         project = tenant.project_id
         bucket = tenant.bucket_name
@@ -182,7 +201,18 @@ class GCloudAioStorage(BlobStorageBase):
         content_type = content_type or mimetypes.guess_type(object_name)[0]
 
         parameters = {}
-        headers = {}
+        headers = kwargs.get('headers', {})
+
+        if not overwrite:  # Upon google documentation
+            parameters['ifGenerationMatch'] = '0'
+
+        assert not(if_match and if_not_match), "if_match and if_not_match cannot be set simultaneous"
+        # Use 'generation' instead of Etag and 'If-Match' and 'If-None-Match' header, this is transparent for the caller
+        if if_match:
+            parameters['ifGenerationMatch'] = str(if_match)
+        if if_not_match:
+            parameters['ifGenerationNotMatch'] = str(if_not_match)
+
         headers.update({
             'Content-Length': str(content_length),
             'Content-Type': content_type or '',
@@ -194,12 +224,14 @@ class GCloudAioStorage(BlobStorageBase):
         if upload_type == UploadType.SIMPLE:
             # if metadata:
             #     log.warning('metadata will be ignored for upload_type=Simple')
-            return await self._upload_simple(project, bucket, url, object_name, stream, parameters, headers,
+            upload_response = await self._upload_simple(project, bucket, url, object_name, stream, parameters, headers,
                                              auth=auth, timeout=timeout)
+            return self.metadict_to_blob(upload_response)
 
         if upload_type == UploadType.RESUMABLE:
-            return await self._upload_resumable(project, bucket, url, object_name, stream, parameters, headers,
+            upload_response = await self._upload_resumable(project, bucket, url, object_name, stream, parameters, headers,
                                                 auth=auth, metadata=metadata, timeout=timeout)
+            return self.metadict_to_blob(upload_response)
 
         raise TypeError(f'upload type {upload_type} not supported')
 

requirements.txt

-# osdu core lib main python
-# osdu-core-lib-python>=0.4.0, <0.5
 # for google provider
 aiohttp
 cryptography

requirements_opengroup.txt

 # osdu core lib main python
 --extra-index-url \
 https://community.opengroup.org/api/v4/projects/465/packages/pypi/simple/
-osdu-core-lib-python>=0.4.0, <0.5
+osdu-core-lib-python==1.0.0.dev270686

tests/storage/test_blob_storage_gcp.py

@@ -7,9 +7,11 @@ import aiohttp
 import uuid
 from osdu.core.api.storage.tenant import Tenant
 
+
 class _TESTING_CFG:
     credentials = TESTING_GCP_DATA_PROJECT_CREDENTIALS
 
+
 TEST_DATA = {
     'initial_files': [
         # object_name , content
@@ -31,7 +33,9 @@ async def storage_client(request):
 
 @pytest.fixture
 async def test_tenant():
-    return Tenant(project_id=TESTING_GCP_DATA_PROJECT_ID, bucket_name='testing-osdu-core', data_partition_id='testing-partition-name')
+    return Tenant(project_id=TESTING_GCP_DATA_PROJECT_ID, bucket_name='testing-osdu-core',
+                  data_partition_id='testing-partition-name')
+
 
 @pytest.fixture
 async def temp_directory() -> str:
@@ -41,6 +45,7 @@ async def temp_directory() -> str:
     # teardown - recursively delete the tmp directory
     shutil.rmtree(tmpdir, ignore_errors=True)
 
+
 @pytest.mark.asyncio
 async def test_list_objects(storage_client, test_tenant):
     result = await storage_client.list_objects(test_tenant)
@@ -88,7 +93,8 @@ async def test_upload_file_bin_input(storage_client, temp_directory, test_tenant
     content_bin = b'expected content 123456789'
     with open(file_c, 'rb') as file_bin_input:
         await storage_client.upload(test_tenant, 'file_bin_input', file_bin_input)
-    assert await storage_client.download(test_tenant,  'file_bin_input') == content_bin
+    assert await storage_client.download(test_tenant, 'file_bin_input') == content_bin
+
 
 @pytest.mark.asyncio
 async def test_upload_file_txt_input(storage_client, temp_directory, test_tenant):
@@ -98,20 +104,23 @@ async def test_upload_file_txt_input(storage_client, temp_directory, test_tenant
     content_bin = b'expected content 123456789'
     with open(file_c, 'r') as file_txt_input:
         await storage_client.upload(test_tenant, 'file_txt_input', file_txt_input)
-    assert await storage_client.download(test_tenant,  'file_txt_input') == content_bin
+    assert await storage_client.download(test_tenant, 'file_txt_input') == content_bin
+
 
 @pytest.mark.asyncio
 async def test_upload_str_input(storage_client, test_tenant):
     content_bin = b'expected content 123456789'
     content_str = content_bin.decode('utf-8')
     await storage_client.upload(test_tenant, 'str_input', content_str)
-    assert await storage_client.download(test_tenant,  'str_input') == content_bin
+    assert await storage_client.download(test_tenant, 'str_input') == content_bin
+
 
 @pytest.mark.asyncio
 async def test_upload_bin_input(storage_client, test_tenant):
     content_bin = b'expected content 123456789'
     await storage_client.upload(test_tenant, 'bin_input', content_bin)
-    assert await storage_client.download(test_tenant,  'bin_input') == content_bin
+    assert await storage_client.download(test_tenant, 'bin_input') == content_bin
+
 
 @pytest.mark.asyncio
 async def test_upload_empty_input(storage_client, test_tenant):
@@ -119,14 +128,38 @@ async def test_upload_empty_input(storage_client, test_tenant):
     actual_data = await storage_client.download(test_tenant, 'empty_input')
     assert len(actual_data) == 0
 
+
 @pytest.mark.asyncio
 async def test_upload_int_input(storage_client, test_tenant):
     with pytest.raises(TypeError):
         await storage_client.upload(test_tenant, 'int_input', 123456)
 
 
+@pytest.mark.asyncio
+async def test_overwrite_with_condition(storage_client, test_tenant):
+    blob_name = 'testing_data/' + str(uuid.uuid4())
+    await storage_client.upload(test_tenant, blob_name, b'1111')
+
+    with pytest.raises(Exception):  # StorageErrorException is internal
+        await storage_client.upload(test_tenant, blob_name, b'1111', overwrite=False)
 
+    # update no condition
+    await storage_client.upload(test_tenant, blob_name, b'1112')
+    assert await storage_client.download(test_tenant, blob_name) == b'1112'
 
+    # successful update if_match
+    etag_1112 = (await storage_client.download_metadata(test_tenant, blob_name)).etag
+    await storage_client.upload(test_tenant, blob_name, b'1113', if_match=etag_1112)
+    assert await storage_client.download(test_tenant, blob_name) == b'1113'
 
+    # should fail update if_match not satisfied
+    with pytest.raises(Exception):  # StorageErrorException is internal
+        await storage_client.upload(test_tenant, blob_name, b'1114', if_match=etag_1112)
 
+    # success update if_not_match
+    await storage_client.upload(test_tenant, blob_name, b'1115', if_not_match=etag_1112)
 
+    # should fail update if_not_match not satisfied
+    etag_1115 = (await storage_client.download_metadata(test_tenant, blob_name)).etag
+    with pytest.raises(Exception):  # StorageErrorException is internal
+        await storage_client.upload(test_tenant, blob_name, b'1116', if_not_match=etag_1115)