Commit e84d2896 authored by Yannick's avatar Yannick
Browse files

map core exceptions

parent 940e763b
Pipeline #33996 passed with stage
in 7 minutes and 2 seconds
from osdu.core.api.storage.blob_storage_base import BlobStorageBase
from osdu.core.api.storage.blob import Blob
from typing import Optional, List, Any
from azure.storage.blob.aio import BlobServiceClient
from azure.storage.blob import ContentSettings
from azure.identity.aio import DefaultAzureCredential
from osdu.core.api.storage.tenant import Tenant
from azure.core import MatchConditions
from azure.core import exceptions as AzureExceptions
from osdu.core.api.storage.blob_storage_base import BlobStorageBase
from osdu.core.api.storage.blob import Blob
from osdu.core.api.storage.tenant import Tenant
from osdu.core.api.storage.exceptions import (
with_blobstorage_exception,
AuthenticationException,
ResourceNotFoundException,
ResourceExistsException,
PreconditionFailedException)
from osdu_az.partition.partition_service import PartitionService
......@@ -23,6 +33,14 @@ class AzureAioBlobStorage(BlobStorageBase):
Credentials = None
ExceptionMapping = {
AzureExceptions.ClientAuthenticationError: AuthenticationException,
AzureExceptions.ResourceNotFoundError: ResourceNotFoundException,
AzureExceptions.ResourceExistsError: ResourceExistsException,
AzureExceptions.ResourceModifiedError: PreconditionFailedException,
AzureExceptions.ResourceNotModifiedError: PreconditionFailedException,
}
def _build_url(self, storage_account: str):
return f'https://{storage_account}.blob.core.windows.net'
......@@ -50,6 +68,7 @@ class AzureAioBlobStorage(BlobStorageBase):
async def _get_storage_account_name(self, data_partition_id: str):
return await PartitionService.get_storage_account_name(data_partition_id)
@with_blobstorage_exception(ExceptionMapping)
async def list_objects(self, tenant: Tenant,
*args, auth: Optional = None, prefix: str = '', page_token: Optional[str] = None,
max_result: Optional[int] = None, timeout: int = 10, **kwargs) -> List[str]:
......@@ -75,6 +94,7 @@ class AzureAioBlobStorage(BlobStorageBase):
break
return result
@with_blobstorage_exception(ExceptionMapping)
async def delete(self, tenant: Tenant, object_name: str,
*args, auth: Optional = None, timeout: int = 10, **kwargs):
"""
......@@ -92,6 +112,7 @@ class AzureAioBlobStorage(BlobStorageBase):
container_client = blob_service_client.get_container_client(container)
await container_client.delete_blob(object_name)
@with_blobstorage_exception(ExceptionMapping)
async def download(self, tenant: Tenant, object_name: str,
*args, auth: Optional = None, timeout: int = 10, **kwargs) -> bytes:
"""
......@@ -112,6 +133,7 @@ class AzureAioBlobStorage(BlobStorageBase):
return await data.readall()
# not for now, parquet only
@with_blobstorage_exception(ExceptionMapping)
async def download_metadata(self, tenant: Tenant, object_name: str,
*args, auth: Optional = None, timeout: int = 10, **kwargs) -> Blob:
"""
......@@ -147,6 +169,7 @@ class AzureAioBlobStorage(BlobStorageBase):
provider_specific=properties
)
@with_blobstorage_exception(ExceptionMapping)
async def upload(self, tenant: Tenant, object_name: str, file_data: Any, *,
overwrite: bool = True,
if_match=None,
......
# osdu core lib main python
--extra-index-url \
https://community.opengroup.org/api/v4/projects/465/packages/pypi/simple/
osdu-core-lib-python==1.0.0.dev270686
osdu-core-lib-python==1.0.0.dev286249
from azure.core.exceptions import AzureError
from osdu.core.api.storage.tenant import Tenant
from tests.conftest import Config
from osdu_az.storage.blob_storage_az import AzureAioBlobStorage
import asyncio
import pytest
from io import BytesIO
import uuid
from mock import patch
from azure.core import exceptions
import asyncio
from osdu.core.api.storage.tenant import Tenant
from osdu.core.api.storage.exceptions import *
from osdu_az.storage.blob_storage_az import AzureAioBlobStorage
from tests.conftest import Config
# Patch '_get_credentials', '_build_url' and '_get_storage_account_name' for tests
@pytest.fixture
......@@ -58,7 +59,7 @@ async def test_overwrite_with_condition(az_client: AzureAioBlobStorage, test_ten
blob_name = 'testing_data/' + str(uuid.uuid4())
await az_client.upload(test_tenant, blob_name, b'1111')
with pytest.raises(AzureError): # StorageErrorException is internal
with pytest.raises(ResourceExistsException):
await az_client.upload(test_tenant, blob_name, b'1111', overwrite=False)
# update no condition
......@@ -71,7 +72,7 @@ async def test_overwrite_with_condition(az_client: AzureAioBlobStorage, test_ten
assert await az_client.download(test_tenant, blob_name) == b'1113'
# should fail update if_match not satisfied
with pytest.raises(AzureError): # StorageErrorException is internal
with pytest.raises(PreconditionFailedException):
await az_client.upload(test_tenant, blob_name, b'1114', if_match=etag_1112)
# success update if_not_match
......@@ -79,7 +80,8 @@ async def test_overwrite_with_condition(az_client: AzureAioBlobStorage, test_ten
# should fail update if_not_match not satisfied
etag_1115 = (await az_client.download_metadata(test_tenant, blob_name)).etag
with pytest.raises(AzureError): # StorageErrorException is internal
with pytest.raises(PreconditionFailedException):
await az_client.upload(test_tenant, blob_name, b'1116', if_not_match=etag_1115)
......@@ -112,24 +114,20 @@ async def test_concurrent_update_only_one_should_succeed(az_client: AzureAioBlob
async def test_download_not_existing_blob_should_throw(az_client: AzureAioBlobStorage, test_tenant):
# here we just ensure it does not silently fail and throw something for now (to be updated when proper exceptions
# will be defined in the core lib)
with pytest.raises(exceptions.ResourceNotFoundError) as ex_info:
with pytest.raises(ResourceNotFoundException):
# given a not existing blob
blob_name = 'testing_data/' + str(uuid.uuid4())
# when try to download it should fail
await az_client.download(test_tenant, blob_name)
# print(ex_info.value)
@pytest.mark.asyncio
async def test_invalid_storage_container(az_client: AzureAioBlobStorage):
with pytest.raises(exceptions.ResourceNotFoundError) as ex_info:
with pytest.raises(ResourceNotFoundException):
tenant = Tenant(project_id=Config.storage_account, bucket_name='not_existing_container', data_partition_id='local')
await az_client.upload(tenant, 'blob_name', 'input_data')
# print(ex_info.value)
@pytest.mark.asyncio
async def test_list_objects(az_client: AzureAioBlobStorage, test_tenant):
......@@ -158,6 +156,6 @@ async def test_delete(az_client: AzureAioBlobStorage, test_tenant):
await az_client.download(test_tenant, blob_name)
await az_client.delete(test_tenant, blob_name)
with pytest.raises(exceptions.ResourceNotFoundError):
with pytest.raises(ResourceNotFoundException):
await az_client.download(test_tenant, blob_name)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment