Commit 03e091e3 authored by fabian serin's avatar fabian serin Committed by Luc Yriarte
Browse files

Adding pipeline and adapting the requirements (to be changed when other lib...

Adding pipeline and adapting the requirements (to be changed when other lib will be avalaible as pip fedd)
parent 73c2fe27
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# ide
.idea/
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
.hypothesis/
.pytest_cache/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# Jupyter Notebook
.ipynb_checkpoints
# pyenv
.python-version
# celery beat schedule file
celerybeat-schedule
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
.venv*
env/
venv/
venv*/
ENV/
env.bak/
venv.bak/
.virtualenv/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
default:
image: python:3.7-slim-buster
stages:
- test
- deploy
build:
stage: test
script:
- echo ---- ---- ---- SYSTEM DEPENDENCIES ---- ---- ----
- apt update
- apt install -y --no-install-recommends git
- echo ---- ---- ---- BUILD IMAGE ---- ---- ----
- pip3 install -r requirements.txt
- pip3 install -r requirements_opengroup.txt
- pip3 install -r requirements_dev.txt
- echo ---- ---- ---- UNIT TESTS ---- ---- ----
- pytest tests --junitxml=report.xml
artifacts:
when: always
reports:
junit: report.xml
# This job only runs on master, and it creates the lib and push it to the feed
deploylib:
stage: deploy
script:
- echo ---- ---- ---- SYSTEM DEPENDENCIES ---- ---- ----
- apt update
- apt install -y --no-install-recommends git
- echo ---- ---- ---- BUILD IMAGE ---- ---- ----
- pip3 install -r requirements.txt
- pip3 install -r requirements_opengroup.txt
- pip3 install twine
- python3 setup.py sdist bdist_wheel
- TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=${CI_REGISTRY_USER} python -m twine upload --repository-url ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi dist/*
rules:
- if: $CI_COMMIT_BRANCH == 'master'
## Introduction
Python package for osdu Azure cloud implementations.
## Requirements and dependencies
Requires Python 3.7+ and depends on osdu-core-lib-python
## Getting Started
### from package
install from package:
`pip install osdu-core-lib-python-azure`
### from sources
install dependencies:
`pip install -r requirements.txt`
## Tests
### install dependencies
unit tests uses [pytest](https://docs.pytest.org/en/stable/) framework. Other dependencies are
[pytest-asyncio](https://pypi.org/project/pytest-asyncio/) for testing asyncio code and
[pytest-cov](https://pypi.org/project/pytest-cov/) to produce coverage report.
### run unit tests
install testing dependencies:
`pip install -r requirements_dev.txt`
run all tests:
`python -m pytest ./tests`
run all tests and produce reports:
`python -m pytest --junit-xml=unit_tests_report.xml --cov=osdu_az --cov-report=html --cov-report=xml ./tests`
## License
Apache version 2.0 (see LICENSE.TXT)
## Contribute
Note: version number must be updated manually for now. Increment by one the version patch part in file osdu_az/\_\_init\_\_.py
__version__ = '0.0.2'
from osdu.core.api.storage.blob_storage_base import BlobStorageBase
from osdu.core.api.storage.blob import Blob
from typing import Optional, List, Any
class AzureAioBlobStorage(BlobStorageBase):
async def list_objects(self, project: str, container: str,
*, auth: Optional = None, prefix: str = '', page_token: Optional[str] = None,
max_result: Optional[int] = None, timeout: int = 10, **kwargs) -> List[str]:
"""
list all object within a bucket
:param auth: auth obj to perform the operation
:param project: project or account
:param container: container or bucket name
:param prefix: Filter results to objects whose names begin with this prefix
:param page_token: A previously-returned page token representing part of the larger set of results to view.
:param max_result: Maximum number of items to return.
:param timeout: timeout
:param kwargs:
:return: list of blob names
"""
raise NotImplementedError('Azure blob storage implementation, "list_objects" not implemented')
async def delete(self, project: str, container: str, object_name: str,
*, auth: Optional = None, timeout: int = 10, **kwargs):
"""
delete an object
:param auth: auth obj to perform the operation
:param project: project or account
:param container: container or bucket name
:param object_name:
:param timeout:
:param kwargs:
:return:
"""
raise NotImplementedError('Azure blob storage implementation, "delete" not implemented')
async def download(self, project: str, container: str, object_name: str,
*, auth: Optional = None, timeout: int = 10, **kwargs) -> bytes:
"""
download blob data
:param auth: auth obj to perform the operation
:param project: project or account
:param container: container or bucket name
:param object_name:
:param timeout:
:param kwargs:
:return:
"""
raise NotImplementedError('Azure blob storage implementation, "download" not implemented')
async def download_metadata(self, project: str, container: str, object_name: str,
*, auth: Optional = None, timeout: int = 10, **kwargs) -> Blob:
"""
download blob data
:param auth: auth obj to perform the operation
:param project: project or account
:param container: container or bucket name
:param object_name:
:param timeout:
:param kwargs:
:return: blob
"""
raise NotImplementedError('Azure blob storage implementation, "download_metadata" not implemented')
async def upload(self, project: str, container: str, object_name: str, file_data: Any,
*, auth: Optional = None, content_type: str = None, metadata: dict = None,
timeout: int = 30, **kwargs):
raise NotImplementedError('Azure blob storage implementation, "upload" not implemented')
# osdu core lib main python
# osdu-core-lib-python>=0.3.0, <0.4
pytest>=3
pytest-asyncio
pytest-cov
git+https://community.opengroup.org/osdu/platform/domain-data-mgmt-services/wellbore/lib/wellbore-core/wellbore-core-lib.git@slb-code-push#egg=osdu-core-lib-python
import os
import re
from setuptools import setup, find_packages
requirements_file = os.path.abspath(
os.path.join(os.path.dirname(__file__), 'requirements.txt')
)
test_requirements_file = os.path.abspath(
os.path.join(os.path.dirname(__file__), 'requirements_dev.txt')
)
# version is unified in osdu_az/__init__.py
def read_version():
regexp = re.compile(r"^__version__\W*=\W*'([\d.abrc]+(dev\d+)?)'")
init_py = os.path.join(os.path.dirname(__file__), 'osdu_az', '__init__.py')
with open(init_py) as f:
for line in f:
match = regexp.match(line)
if match is not None:
return match.group(1)
raise RuntimeError('Cannot find version in {}'.format(init_py))
with open(requirements_file) as f:
requirements = f.read().splitlines()
with open(test_requirements_file) as f:
test_requirements = f.read().splitlines()
setup(
name='osdu-core-lib-python-azure',
version=read_version(),
packages=find_packages(),
author="OSDU Wellbore data domain services team",
author_email="WellboreDDMS@slb.com",
description="OSDU python for wellbore DMS - azure cloud implementations",
long_description=open('README.md').read(),
include_package_data=True,
install_requires=requirements,
tests_require=test_requirements,
license="Apache License, Version 2.0",
# We need py 3.7 for some of the tools used here
python_requires='>=3.7',
)
from osdu_az.storage.blob_storage_az import AzureAioBlobStorage
import pytest
@pytest.fixture
async def az_client() -> AzureAioBlobStorage:
yield AzureAioBlobStorage()
async def assert_not_implemented(coro):
with pytest.raises(NotImplementedError) as ex_info:
await coro
ex = ex_info.value
assert 'azure' in str(ex).lower()
@pytest.mark.asyncio
async def test_list_objects_should_throw_not_implemented(az_client: AzureAioBlobStorage):
await assert_not_implemented(
az_client.list_objects('project', 'container')
)
@pytest.mark.asyncio
async def test_delete_should_throw_not_implemented(az_client: AzureAioBlobStorage):
await assert_not_implemented(
az_client.delete('project', 'container', 'id')
)
@pytest.mark.asyncio
async def test_download_metadata_should_throw_not_implemented(az_client: AzureAioBlobStorage):
await assert_not_implemented(
az_client.download_metadata('project', 'container', 'id')
)
@pytest.mark.asyncio
async def test_download_should_throw_not_implemented(az_client: AzureAioBlobStorage):
await assert_not_implemented(
az_client.download('project', 'container', 'id')
)
@pytest.mark.asyncio
async def test_upload_should_throw_not_implemented(az_client: AzureAioBlobStorage):
await assert_not_implemented(
az_client.upload('project', 'container', 'id', '')
)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment