Commit e7533316 authored by Yannick's avatar Yannick
Browse files

map core exceptions

parent 0a23016f
Pipeline #35786 passed with stage
in 38 seconds
from osdu.core.api.storage.blob_storage_base import BlobStorageBase
from osdu.core.api.storage.blob import Blob
from typing import Optional, List, Any from typing import Optional, List, Any
from azure.storage.blob.aio import BlobServiceClient from azure.storage.blob.aio import BlobServiceClient
from azure.storage.blob import ContentSettings from azure.storage.blob import ContentSettings
from azure.identity.aio import DefaultAzureCredential from azure.identity.aio import DefaultAzureCredential
from osdu.core.api.storage.tenant import Tenant
from azure.core import MatchConditions from azure.core import MatchConditions
from azure.core import exceptions as AzureExceptions
from osdu.core.api.storage.blob_storage_base import BlobStorageBase
from osdu.core.api.storage.blob import Blob
from osdu.core.api.storage.tenant import Tenant
from osdu.core.api.storage.exceptions import (
with_blobstorage_exception,
AuthenticationException,
ResourceNotFoundException,
ResourceExistsException,
PreconditionFailedException)
from osdu_az.partition.partition_service import PartitionService from osdu_az.partition.partition_service import PartitionService
...@@ -23,6 +33,14 @@ class AzureAioBlobStorage(BlobStorageBase): ...@@ -23,6 +33,14 @@ class AzureAioBlobStorage(BlobStorageBase):
Credentials = None Credentials = None
ExceptionMapping = {
AzureExceptions.ClientAuthenticationError: AuthenticationException,
AzureExceptions.ResourceNotFoundError: ResourceNotFoundException,
AzureExceptions.ResourceExistsError: ResourceExistsException,
AzureExceptions.ResourceModifiedError: PreconditionFailedException,
AzureExceptions.ResourceNotModifiedError: PreconditionFailedException,
}
def _build_url(self, storage_account: str): def _build_url(self, storage_account: str):
return f'https://{storage_account}.blob.core.windows.net' return f'https://{storage_account}.blob.core.windows.net'
...@@ -50,6 +68,7 @@ class AzureAioBlobStorage(BlobStorageBase): ...@@ -50,6 +68,7 @@ class AzureAioBlobStorage(BlobStorageBase):
async def _get_storage_account_name(self, data_partition_id: str): async def _get_storage_account_name(self, data_partition_id: str):
return await PartitionService.get_storage_account_name(data_partition_id) return await PartitionService.get_storage_account_name(data_partition_id)
@with_blobstorage_exception(ExceptionMapping)
async def list_objects(self, tenant: Tenant, async def list_objects(self, tenant: Tenant,
*args, auth: Optional = None, prefix: str = '', page_token: Optional[str] = None, *args, auth: Optional = None, prefix: str = '', page_token: Optional[str] = None,
max_result: Optional[int] = None, timeout: int = 10, **kwargs) -> List[str]: max_result: Optional[int] = None, timeout: int = 10, **kwargs) -> List[str]:
...@@ -75,6 +94,7 @@ class AzureAioBlobStorage(BlobStorageBase): ...@@ -75,6 +94,7 @@ class AzureAioBlobStorage(BlobStorageBase):
break break
return result return result
@with_blobstorage_exception(ExceptionMapping)
async def delete(self, tenant: Tenant, object_name: str, async def delete(self, tenant: Tenant, object_name: str,
*args, auth: Optional = None, timeout: int = 10, **kwargs): *args, auth: Optional = None, timeout: int = 10, **kwargs):
""" """
...@@ -92,6 +112,7 @@ class AzureAioBlobStorage(BlobStorageBase): ...@@ -92,6 +112,7 @@ class AzureAioBlobStorage(BlobStorageBase):
container_client = blob_service_client.get_container_client(container) container_client = blob_service_client.get_container_client(container)
await container_client.delete_blob(object_name) await container_client.delete_blob(object_name)
@with_blobstorage_exception(ExceptionMapping)
async def download(self, tenant: Tenant, object_name: str, async def download(self, tenant: Tenant, object_name: str,
*args, auth: Optional = None, timeout: int = 10, **kwargs) -> bytes: *args, auth: Optional = None, timeout: int = 10, **kwargs) -> bytes:
""" """
...@@ -112,6 +133,7 @@ class AzureAioBlobStorage(BlobStorageBase): ...@@ -112,6 +133,7 @@ class AzureAioBlobStorage(BlobStorageBase):
return await data.readall() return await data.readall()
# not for now, parquet only # not for now, parquet only
@with_blobstorage_exception(ExceptionMapping)
async def download_metadata(self, tenant: Tenant, object_name: str, async def download_metadata(self, tenant: Tenant, object_name: str,
*args, auth: Optional = None, timeout: int = 10, **kwargs) -> Blob: *args, auth: Optional = None, timeout: int = 10, **kwargs) -> Blob:
""" """
...@@ -147,6 +169,7 @@ class AzureAioBlobStorage(BlobStorageBase): ...@@ -147,6 +169,7 @@ class AzureAioBlobStorage(BlobStorageBase):
provider_specific=properties provider_specific=properties
) )
@with_blobstorage_exception(ExceptionMapping)
async def upload(self, tenant: Tenant, object_name: str, file_data: Any, *, async def upload(self, tenant: Tenant, object_name: str, file_data: Any, *,
overwrite: bool = True, overwrite: bool = True,
if_match=None, if_match=None,
......
# osdu core lib main python # osdu core lib main python
--extra-index-url \ --extra-index-url \
https://community.opengroup.org/api/v4/projects/465/packages/pypi/simple/ https://community.opengroup.org/api/v4/projects/465/packages/pypi/simple/
osdu-core-lib-python==1.0.0.dev270686 osdu-core-lib-python==1.0.0.dev286249
from azure.core.exceptions import AzureError import asyncio
from osdu.core.api.storage.tenant import Tenant
from tests.conftest import Config
from osdu_az.storage.blob_storage_az import AzureAioBlobStorage
import pytest import pytest
from io import BytesIO from io import BytesIO
import uuid import uuid
from mock import patch from mock import patch
from azure.core import exceptions
import asyncio
from osdu.core.api.storage.tenant import Tenant
from osdu.core.api.storage.exceptions import *
from osdu_az.storage.blob_storage_az import AzureAioBlobStorage
from tests.conftest import Config
# Patch '_get_credentials', '_build_url' and '_get_storage_account_name' for tests # Patch '_get_credentials', '_build_url' and '_get_storage_account_name' for tests
@pytest.fixture @pytest.fixture
...@@ -58,7 +59,7 @@ async def test_overwrite_with_condition(az_client: AzureAioBlobStorage, test_ten ...@@ -58,7 +59,7 @@ async def test_overwrite_with_condition(az_client: AzureAioBlobStorage, test_ten
blob_name = 'testing_data/' + str(uuid.uuid4()) blob_name = 'testing_data/' + str(uuid.uuid4())
await az_client.upload(test_tenant, blob_name, b'1111') await az_client.upload(test_tenant, blob_name, b'1111')
with pytest.raises(AzureError): # StorageErrorException is internal with pytest.raises(ResourceExistsException):
await az_client.upload(test_tenant, blob_name, b'1111', overwrite=False) await az_client.upload(test_tenant, blob_name, b'1111', overwrite=False)
# update no condition # update no condition
...@@ -71,7 +72,7 @@ async def test_overwrite_with_condition(az_client: AzureAioBlobStorage, test_ten ...@@ -71,7 +72,7 @@ async def test_overwrite_with_condition(az_client: AzureAioBlobStorage, test_ten
assert await az_client.download(test_tenant, blob_name) == b'1113' assert await az_client.download(test_tenant, blob_name) == b'1113'
# should fail update if_match not satisfied # should fail update if_match not satisfied
with pytest.raises(AzureError): # StorageErrorException is internal with pytest.raises(PreconditionFailedException):
await az_client.upload(test_tenant, blob_name, b'1114', if_match=etag_1112) await az_client.upload(test_tenant, blob_name, b'1114', if_match=etag_1112)
# success update if_not_match # success update if_not_match
...@@ -79,7 +80,8 @@ async def test_overwrite_with_condition(az_client: AzureAioBlobStorage, test_ten ...@@ -79,7 +80,8 @@ async def test_overwrite_with_condition(az_client: AzureAioBlobStorage, test_ten
# should fail update if_not_match not satisfied # should fail update if_not_match not satisfied
etag_1115 = (await az_client.download_metadata(test_tenant, blob_name)).etag etag_1115 = (await az_client.download_metadata(test_tenant, blob_name)).etag
with pytest.raises(AzureError): # StorageErrorException is internal
with pytest.raises(PreconditionFailedException):
await az_client.upload(test_tenant, blob_name, b'1116', if_not_match=etag_1115) await az_client.upload(test_tenant, blob_name, b'1116', if_not_match=etag_1115)
...@@ -112,24 +114,20 @@ async def test_concurrent_update_only_one_should_succeed(az_client: AzureAioBlob ...@@ -112,24 +114,20 @@ async def test_concurrent_update_only_one_should_succeed(az_client: AzureAioBlob
async def test_download_not_existing_blob_should_throw(az_client: AzureAioBlobStorage, test_tenant): async def test_download_not_existing_blob_should_throw(az_client: AzureAioBlobStorage, test_tenant):
# here we just ensure it does not silently fail and throw something for now (to be updated when proper exceptions # here we just ensure it does not silently fail and throw something for now (to be updated when proper exceptions
# will be defined in the core lib) # will be defined in the core lib)
with pytest.raises(exceptions.ResourceNotFoundError) as ex_info: with pytest.raises(ResourceNotFoundException):
# given a not existing blob # given a not existing blob
blob_name = 'testing_data/' + str(uuid.uuid4()) blob_name = 'testing_data/' + str(uuid.uuid4())
# when try to download it should fail # when try to download it should fail
await az_client.download(test_tenant, blob_name) await az_client.download(test_tenant, blob_name)
# print(ex_info.value)
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_invalid_storage_container(az_client: AzureAioBlobStorage): async def test_invalid_storage_container(az_client: AzureAioBlobStorage):
with pytest.raises(exceptions.ResourceNotFoundError) as ex_info: with pytest.raises(ResourceNotFoundException):
tenant = Tenant(project_id=Config.storage_account, bucket_name='not_existing_container', data_partition_id='local') tenant = Tenant(project_id=Config.storage_account, bucket_name='not_existing_container', data_partition_id='local')
await az_client.upload(tenant, 'blob_name', 'input_data') await az_client.upload(tenant, 'blob_name', 'input_data')
# print(ex_info.value)
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_list_objects(az_client: AzureAioBlobStorage, test_tenant): async def test_list_objects(az_client: AzureAioBlobStorage, test_tenant):
...@@ -158,6 +156,6 @@ async def test_delete(az_client: AzureAioBlobStorage, test_tenant): ...@@ -158,6 +156,6 @@ async def test_delete(az_client: AzureAioBlobStorage, test_tenant):
await az_client.download(test_tenant, blob_name) await az_client.download(test_tenant, blob_name)
await az_client.delete(test_tenant, blob_name) await az_client.delete(test_tenant, blob_name)
with pytest.raises(exceptions.ResourceNotFoundError): with pytest.raises(ResourceNotFoundException):
await az_client.download(test_tenant, blob_name) await az_client.download(test_tenant, blob_name)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment