Commit 385a40fb authored by Luc Yriarte's avatar Luc Yriarte
Browse files

Merge branch 'slb-code-push' into 'master'

Slb code push 3

See merge request !4
parents ec49fa02 3426ee40
Pipeline #26503 failed with stages
in 3 minutes and 30 seconds
......@@ -8,12 +8,8 @@ stages:
build:
stage: test
script:
- echo ---- ---- ---- SYSTEM DEPENDENCIES ---- ---- ----
- apt update
- apt install -y --no-install-recommends git
- echo ---- ---- ---- BUILD IMAGE ---- ---- ----
- pip3 install -r requirements.txt
- pip3 install -r requirements_opengroup.txt
- pip3 install -r requirements_dev.txt
- echo ---- ---- ---- UNIT TESTS ---- ---- ----
- pytest tests --junitxml=report.xml
......@@ -26,12 +22,8 @@ build:
deploylib:
stage: deploy
script:
- echo ---- ---- ---- SYSTEM DEPENDENCIES ---- ---- ----
- apt update
- apt install -y --no-install-recommends git
- echo ---- ---- ---- BUILD IMAGE ---- ---- ----
- pip3 install -r requirements.txt
- pip3 install -r requirements_opengroup.txt
- pip3 install twine
- python3 setup.py sdist bdist_wheel
- TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=${CI_REGISTRY_USER} python -m twine upload --repository-url ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi dist/*
......
......@@ -62,7 +62,12 @@ variables:
| TESTING_AZ_CREDENTIALS | `Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==` |
| TESTING_AZ_CONTAINER | `wdms-osdu` |
**Note: value for TESTING_AZ_CREDENTIALS is default value for Azurite, it's not a private key*
**Note1: value for TESTING_AZ_CREDENTIALS is default value for Azurite, it's not a private key*
**Note2: The version of Azurite requires to be compatible with the azure SDK python libs installed. For instance,
azure-storage-blob 12.6+ requires Azurite 3.10. In case of invalid version, calls will fail with an error about invalid
header `x-ms-version` value. For more information see Microsoft
[Versioning for the Azure Storage services](https://docs.microsoft.com/en-us/rest/api/storageservices/versioning-for-the-azure-storage-services).*
### How to run locally using DefaultAzureCredential and the Azure CLI
......
__version__ = '0.1.5'
__version__ = '0.2.0'
......@@ -9,17 +9,36 @@ from osdu_az.partition.partition_service import PartitionService
class AzureAioBlobStorage(BlobStorageBase):
"""
Azure implementation of BlobStorageBase.
Attributes:
----------
Credentials: Set as class attribute. For efficiency purposes, it keeps a single instance of the
credentials and can then be closed when no longer needed by directly calling the class method
`close_credentials`. Credentials are designed to be re-used. It embeds, among other things, cache mechanism
and auto refresh token. It uses `DefaultAzureCredential`.
"""
Credentials = None
def _build_url(self, storage_account: str):
return f'https://{storage_account}.blob.core.windows.net'
def _get_credentials(self):
return DefaultAzureCredential(exclude_shared_token_cache_credential=True,
exclude_visual_studio_code_credential=True,
exclude_environment_credential=True)
if AzureAioBlobStorage.Credentials is None:
AzureAioBlobStorage.Credentials = DefaultAzureCredential(
exclude_shared_token_cache_credential=True,
exclude_visual_studio_code_credential=True,
exclude_environment_credential=True)
return AzureAioBlobStorage.Credentials
async def _close_credentials(self, credential):
await credential.close()
@classmethod
async def close_credentials(cls):
""" This cause to gracefully dispose credentials if any. Next calls will then initialize a new one """
credentials_to_close, cls.Credentials = cls.Credentials, None # swap
if credentials_to_close is not None:
await credentials_to_close.close()
async def _get_storage_account_name(self, data_partition_id: str):
return await PartitionService.get_storage_account_name(data_partition_id)
......@@ -70,15 +89,13 @@ class AzureAioBlobStorage(BlobStorageBase):
cred = self._get_credentials()
account_url = self._build_url(storage_account)
try:
blob_service_client = BlobServiceClient(account_url=account_url, credential=cred)
async with blob_service_client:
container_client = blob_service_client.get_container_client(container)
blob_client = container_client.get_blob_client(object_name)
data = await blob_client.download_blob()
return await data.readall()
finally:
await self._close_credentials(cred)
blob_service_client = BlobServiceClient(account_url=account_url, credential=cred)
async with blob_service_client:
container_client = blob_service_client.get_container_client(container)
blob_client = container_client.get_blob_client(object_name)
data = await blob_client.download_blob()
return await data.readall()
# not for now, parquet only
async def download_metadata(self, tenant: Tenant, object_name: str,
......@@ -113,12 +130,9 @@ class AzureAioBlobStorage(BlobStorageBase):
cred = self._get_credentials()
account_url = self._build_url(storage_account)
try:
blob_service_client = BlobServiceClient(account_url=account_url, credential=cred)
async with blob_service_client:
container_client = blob_service_client.get_container_client(container)
blob_client = container_client.get_blob_client(object_name)
data = await blob_client.upload_blob(file_data, overwrite=True, metadata=metadata)
return data
finally:
await self._close_credentials(cred)
blob_service_client = BlobServiceClient(account_url=account_url, credential=cred)
async with blob_service_client:
container_client = blob_service_client.get_container_client(container)
blob_client = container_client.get_blob_client(object_name)
data = await blob_client.upload_blob(file_data, overwrite=True, metadata=metadata)
return data
# osdu core lib main python
# osdu-core-lib-python>=0.4.0, <0.5
azure.storage.blob>=12.4
azure.identity
--extra-index-url \
https://community.opengroup.org/api/v4/projects/223/packages/pypi/simple/
osdu-core-lib-python>=0.4.0, <0.5
azure-storage-blob==12.6.0 # fix version, 12.7.0 & 12.7.1 cause hangs in tests
azure-identity
azure-keyvault
cachetools
pytest>=3
pytest-asyncio
pytest-cov
mock
azure.core
\ No newline at end of file
azure.core
mock # needed for AsyncMock for python 3.7
git+https://community.opengroup.org/osdu/platform/domain-data-mgmt-services/wellbore/lib/wellbore-core/wellbore-core-lib.git@slb-code-push#egg=osdu-core-lib-python
......@@ -11,8 +11,8 @@ class Config:
environ.setdefault('SERVICE_HOST_PARTITION', 'invalid-partition-service-name')
def pytest_configure(config):
blob_service_client = BlobServiceClient(account_url=Config.storage_account, credential= Config.credentials)
blob_service_client = BlobServiceClient(account_url=Config.storage_account, credential=Config.credentials)
try:
blob_service_client.create_container(Config.container)
except Exception as ex:
print(ex)
\ No newline at end of file
print(ex)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment