Commit 46346fae authored by Luc Yriarte's avatar Luc Yriarte
Browse files

Merge branch 'slb-code-push' into 'master'

Slb code push and pipeline

See merge request !2
parents 73c2fe27 03e091e3
Pipeline #17355 passed with stages
in 59 seconds
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# ide
.idea/
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
.hypothesis/
.pytest_cache/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# Jupyter Notebook
.ipynb_checkpoints
# pyenv
.python-version
# celery beat schedule file
celerybeat-schedule
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
.venv*
env/
venv/
venv*/
ENV/
env.bak/
venv.bak/
.virtualenv/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
default:
image: python:3.7-slim-buster
stages:
- test
- deploy
build:
stage: test
script:
- echo ---- ---- ---- SYSTEM DEPENDENCIES ---- ---- ----
- apt update
- apt install -y --no-install-recommends git
- echo ---- ---- ---- BUILD IMAGE ---- ---- ----
- pip3 install -r requirements.txt
- pip3 install -r requirements_opengroup.txt
- pip3 install -r requirements_dev.txt
- echo ---- ---- ---- UNIT TESTS ---- ---- ----
- pytest tests --junitxml=report.xml
artifacts:
when: always
reports:
junit: report.xml
# This job only runs on master, and it creates the lib and push it to the feed
deploylib:
stage: deploy
script:
- echo ---- ---- ---- SYSTEM DEPENDENCIES ---- ---- ----
- apt update
- apt install -y --no-install-recommends git
- echo ---- ---- ---- BUILD IMAGE ---- ---- ----
- pip3 install -r requirements.txt
- pip3 install -r requirements_opengroup.txt
- pip3 install twine
- python3 setup.py sdist bdist_wheel
- TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=${CI_REGISTRY_USER} python -m twine upload --repository-url ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi dist/*
rules:
- if: $CI_COMMIT_BRANCH == 'master'
## Introduction
Python package for osdu Azure cloud implementations.
## Requirements and dependencies
Requires Python 3.7+ and depends on osdu-core-lib-python
## Getting Started
### from package
install from package:
`pip install osdu-core-lib-python-azure`
### from sources
install dependencies:
`pip install -r requirements.txt`
## Tests
### install dependencies
unit tests uses [pytest](https://docs.pytest.org/en/stable/) framework. Other dependencies are
[pytest-asyncio](https://pypi.org/project/pytest-asyncio/) for testing asyncio code and
[pytest-cov](https://pypi.org/project/pytest-cov/) to produce coverage report.
### run unit tests
install testing dependencies:
`pip install -r requirements_dev.txt`
run all tests:
`python -m pytest ./tests`
run all tests and produce reports:
`python -m pytest --junit-xml=unit_tests_report.xml --cov=osdu_az --cov-report=html --cov-report=xml ./tests`
## License
Apache version 2.0 (see LICENSE.TXT)
## Contribute
Note: version number must be updated manually for now. Increment by one the version patch part in file osdu_az/\_\_init\_\_.py
__version__ = '0.0.2'
from osdu.core.api.storage.blob_storage_base import BlobStorageBase
from osdu.core.api.storage.blob import Blob
from typing import Optional, List, Any
class AzureAioBlobStorage(BlobStorageBase):
async def list_objects(self, project: str, container: str,
*, auth: Optional = None, prefix: str = '', page_token: Optional[str] = None,
max_result: Optional[int] = None, timeout: int = 10, **kwargs) -> List[str]:
"""
list all object within a bucket
:param auth: auth obj to perform the operation
:param project: project or account
:param container: container or bucket name
:param prefix: Filter results to objects whose names begin with this prefix
:param page_token: A previously-returned page token representing part of the larger set of results to view.
:param max_result: Maximum number of items to return.
:param timeout: timeout
:param kwargs:
:return: list of blob names
"""
raise NotImplementedError('Azure blob storage implementation, "list_objects" not implemented')
async def delete(self, project: str, container: str, object_name: str,
*, auth: Optional = None, timeout: int = 10, **kwargs):
"""
delete an object
:param auth: auth obj to perform the operation
:param project: project or account
:param container: container or bucket name
:param object_name:
:param timeout:
:param kwargs:
:return:
"""
raise NotImplementedError('Azure blob storage implementation, "delete" not implemented')
async def download(self, project: str, container: str, object_name: str,
*, auth: Optional = None, timeout: int = 10, **kwargs) -> bytes:
"""
download blob data
:param auth: auth obj to perform the operation
:param project: project or account
:param container: container or bucket name
:param object_name:
:param timeout:
:param kwargs:
:return:
"""
raise NotImplementedError('Azure blob storage implementation, "download" not implemented')
async def download_metadata(self, project: str, container: str, object_name: str,
*, auth: Optional = None, timeout: int = 10, **kwargs) -> Blob:
"""
download blob data
:param auth: auth obj to perform the operation
:param project: project or account
:param container: container or bucket name
:param object_name:
:param timeout:
:param kwargs:
:return: blob
"""
raise NotImplementedError('Azure blob storage implementation, "download_metadata" not implemented')
async def upload(self, project: str, container: str, object_name: str, file_data: Any,
*, auth: Optional = None, content_type: str = None, metadata: dict = None,
timeout: int = 30, **kwargs):
raise NotImplementedError('Azure blob storage implementation, "upload" not implemented')
# osdu core lib main python
# osdu-core-lib-python>=0.3.0, <0.4
pytest>=3
pytest-asyncio
pytest-cov
git+https://community.opengroup.org/osdu/platform/domain-data-mgmt-services/wellbore/lib/wellbore-core/wellbore-core-lib.git@slb-code-push#egg=osdu-core-lib-python
import os
import re
from setuptools import setup, find_packages
requirements_file = os.path.abspath(
os.path.join(os.path.dirname(__file__), 'requirements.txt')
)
test_requirements_file = os.path.abspath(
os.path.join(os.path.dirname(__file__), 'requirements_dev.txt')
)
# version is unified in osdu_az/__init__.py
def read_version():
regexp = re.compile(r"^__version__\W*=\W*'([\d.abrc]+(dev\d+)?)'")
init_py = os.path.join(os.path.dirname(__file__), 'osdu_az', '__init__.py')
with open(init_py) as f:
for line in f:
match = regexp.match(line)
if match is not None:
return match.group(1)
raise RuntimeError('Cannot find version in {}'.format(init_py))
with open(requirements_file) as f:
requirements = f.read().splitlines()
with open(test_requirements_file) as f:
test_requirements = f.read().splitlines()
setup(
name='osdu-core-lib-python-azure',
version=read_version(),
packages=find_packages(),
author="OSDU Wellbore data domain services team",
author_email="WellboreDDMS@slb.com",
description="OSDU python for wellbore DMS - azure cloud implementations",
long_description=open('README.md').read(),
include_package_data=True,
install_requires=requirements,
tests_require=test_requirements,
license="Apache License, Version 2.0",
# We need py 3.7 for some of the tools used here
python_requires='>=3.7',
)
from osdu_az.storage.blob_storage_az import AzureAioBlobStorage
import pytest
@pytest.fixture
async def az_client() -> AzureAioBlobStorage:
yield AzureAioBlobStorage()
async def assert_not_implemented(coro):
with pytest.raises(NotImplementedError) as ex_info:
await coro
ex = ex_info.value
assert 'azure' in str(ex).lower()
@pytest.mark.asyncio
async def test_list_objects_should_throw_not_implemented(az_client: AzureAioBlobStorage):
await assert_not_implemented(
az_client.list_objects('project', 'container')
)
@pytest.mark.asyncio
async def test_delete_should_throw_not_implemented(az_client: AzureAioBlobStorage):
await assert_not_implemented(
az_client.delete('project', 'container', 'id')
)
@pytest.mark.asyncio
async def test_download_metadata_should_throw_not_implemented(az_client: AzureAioBlobStorage):
await assert_not_implemented(
az_client.download_metadata('project', 'container', 'id')
)
@pytest.mark.asyncio
async def test_download_should_throw_not_implemented(az_client: AzureAioBlobStorage):
await assert_not_implemented(
az_client.download('project', 'container', 'id')
)
@pytest.mark.asyncio
async def test_upload_should_throw_not_implemented(az_client: AzureAioBlobStorage):
await assert_not_implemented(
az_client.upload('project', 'container', 'id', '')
)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment