Commit af432a2a authored by Siarhei Khaletski (EPAM)'s avatar Siarhei Khaletski (EPAM) 🚩
Browse files

Merge branch 'initial-branch' into 'master'

Initial commit

See merge request !1
parents 749af620 f34bc7b3
Pipeline #59672 passed with stages
in 33 seconds
# https://editorconfig.org/
root = true
[*]
indent_style = space
indent_size = 4
insert_final_newline = true
trim_trailing_whitespace = true
end_of_line = lf
charset = utf-8
[*.py]
max_line_length = 100
**/.idea
**/.DS_Store
# Byte-compiled / optimized / DLL files
**/__pycache__/
**/*.py[cod]
**/*$py.class
*.log
# C extensions
**/*.so
# Distribution / packaging
**/.Python
**/build/
**/develop-eggs/
**/dist/
**/downloads/
**/eggs/
**/.eggs/
**/lib/
**/lib64/
**/parts/
**/sdist/
**/var/
**/wheels/
**/share/python-wheels/
**/*.egg-info/
**/.installed.cfg
**/*.egg
**/MANIFEST
**/venv
# ignore coverage.py
htmlcov/*
.coverage
default:
image: python:3.6-slim-buster
stages:
- linters
- deploy
pylint:
stage: linters
allow_failure: true
script:
- python -m pip install setuptools pylint pylint_quotes pylint-exit
- pylint --rcfile=.pylintrc osdu_airflow/*/*.py || EXIT_CODE=$?
- exit ${EXIT_CODE}
isort:
allow_failure: true
stage: linters
script:
- python -m pip install setuptools isort
- isort -c -v osdu_airflow/*/*.py || EXIT_CODE=$?
- exit ${EXIT_CODE}
deploy:
stage: deploy
script:
- pip install twine
- python setup.py sdist bdist_wheel
- TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python -m twine upload --repository-url ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi dist/*
when: manual
[MASTER]
# Add operators and dags to sys-path
init-hook='import sys; sys.path.append("src/plugins"); sys.path.append("src/dags")'
# Specify a configuration file.
#rcfile=.pylintrc
# Profiled execution.
profile=no
# Add <file or directory> to the black list. It should be a base name, not a
# path. You may set this option multiple times.
ignore=.git, .venv, .idea, CVS
# Pickle collected data for later comparisons.
persistent=yes
# When enabled, pylint would attempt to guess common misconfiguration and emit
# user-friendly hints instead of false-positive error messages.
suggestion-mode=yes
# Allow loading of arbitrary C extensions. Extensions are imported into the
# active Python interpreter and may run arbitrary code.
unsafe-load-any-extension=no
# List of plugins (as comma separated values of python modules names) to load,
# usually to register additional checkers.
load-plugins=pylint_quotes
[MESSAGES CONTROL]
# Enable the message, report, category or checker with the given id(s). You can
# either give multiple identifier separated by comma (,) or put this option
# multiple time.
enable=c-extension-no-member
# Disable the message, report, category or checker with the given id(s). You
# can either give multiple identifier separated by comma (,) or put this option
# multiple time.
disable=no-member, no-self-use, import-error
[REPORTS]
# Python expression which should return a note less than 10 (10 is the highest
# note). You have access to the variables errors warning, statement which
# respectively contain the number of errors / warnings messages and the total
# number of statements analyzed. This is used by the global evaluation report
# (R0004).
evaluation=13.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
# Set the output format. Available formats are text, parseable, colorized, msvs
# (visual studio) and html
output-format=text
# Include message's id in output
include-ids=yes
# Tells whether to display a full report or only the messages
reports=yes
# Put messages in a separate file for each module / package specified on the
# command line instead of printing them on stdout. Reports (if any) will be
# written in a file name "pylint_global.[txt|html]".
files-output=no
# Activate the evaluation score.
score=yes
[REFACTORING]
# Maximum number of nested blocks for function / method body
max-nested-blocks=5
[VARIABLES]
# Tells whether we should check for unused import in __init__ files.
init-import=yes
# A regular expression matching the name of dummy variables (i.e. expectedly
# not used).
dummy-variables-rgx=(_+[a-zA-Z0-9]*?$)|dummy
# List of additional names supposed to be defined in builtins. Remember that
# you should avoid to define new builtins when possible.
additional-builtins=
# List of strings which can identify a callback function by name. A callback
# name must start or end with one of those strings.
callbacks=cb_,_cb
# List of qualified module names which can have objects that can redefine
# builtins.
redefining-builtins-modules=six.moves,future.builtins
[CLASSES]
# List of method names used to declare (i.e. assign) instance attributes.
defining-attr-methods=__init__,__new__,setUp
# List of valid names for the first argument in a class method.
valid-classmethod-first-arg=cls
# List of valid names for the first argument in a metaclass class method.
valid-metaclass-classmethod-first-arg=cls
# List of member names, which should be excluded from the protected access
# warning.
exclude-protected=_asdict,_fields,_replace,_source,_make
[BASIC]
# Required attributes for module, separated by a comma
required-attributes=
# Regular expression which should only match correct module names
module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
# Regular expression which should only match correct module level names
const-rgx=
# Regular expression which should only match correct class names
class-rgx=[A-Z_][a-zA-Z0-9]+$
# Regular expression which should only match correct function names
function-rgx=[a-z_][a-z0-9_]{2,60}$
# Regular expression which should only match correct method names
method-rgx=[a-z_][a-z0-9_]{2,60}$
# Regular expression which should only match correct instance attribute names
attr-rgx=[a-z_][a-z0-9_]{2,30}$
# Regular expression which should only match correct argument names
argument-rgx=[a-z_][a-z0-9_]{2,30}$
# Regular expression which should only match correct variable names
variable-rgx=[a-z_][a-z0-9_]{2,30}$
# Regular expression which should only match correct list comprehension /
# generator expression variable names
inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$
# Good variable names which should always be accepted, separated by a comma
good-names=e,f,i,j,k,ex,Run,_
# Bad variable names which should always be refused, separated by a comma
bad-names=foo,bar,baz,toto,tutu,tata
# Regular expression which should only match functions or classes name which do
# not require a docstring
no-docstring-rgx=__.*__
[FORMAT]
# Maximum number of characters on a single line.
max-line-length=100
# Maximum number of lines in a module
max-module-lines=1000
# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1
# tab).
indent-string=' '
# Set the linting for string quotes
string-quote=double
triple-quote=double
docstring-quote=double
[LOGGING]
# Logging modules to check that the string format arguments are in logging
# function parameter format.
logging-modules=logging
# Format style used to check logging format string. `old` means using %
# formatting, `new` is for `{}` formatting,and `fstr` is for f-strings.
logging-format-style=new
[MISCELLANEOUS]
# List of note tags to take in consideration, separated by a comma.
notes=FIXME,XXX,TODO
[SIMILARITIES]
# Minimum lines number of a similarity.
min-similarity-lines=4
# Ignore comments when computing similarities.
ignore-comments=yes
# Ignore docstrings when computing similarities.
ignore-docstrings=yes
# Ignore imports when computing similarities.
ignore-imports=no
[DESIGN]
# Maximum number of arguments for function / method
max-args=7
# Argument names that match this expression will be ignored. Default to name
# with leading underscore
ignored-argument-names=_.*
# Maximum number of locals for function / method body
max-locals=15
# Maximum number of return / yield for function / method body
max-returns=6
# Maximum number of branch for function / method body
max-branches=12
# Maximum number of statements in function / method body
max-statements=50
# Maximum number of parents for a class (see R0901).
max-parents=7
# Maximum number of attributes for a class (see R0902).
max-attributes=7
# Minimum number of public methods for a class (see R0903).
min-public-methods=0
# Maximum number of public methods for a class (see R0904).
max-public-methods=20
# Maximum number of boolean expressions in a if statement
max-bool-expr=5
[IMPORTS]
# Deprecated modules which should not be used, separated by a comma
deprecated-modules=regsub,TERMIOS,Bastion,rexec
# Create a graph of every (i.e. internal and external) dependencies in the
# given file (report RP0402 must not be disabled)
import-graph=
# Create a graph of external dependencies in the given file (report RP0402 must
# not be disabled)
ext-import-graph=
# Create a graph of internal dependencies in the given file (report RP0402 must
# not be disabled)
int-import-graph=
# Force import order to recognize a module as part of the standard
# compatibility libraries.
known-standard-library=
# Force import order to recognize a module as part of a third party library.
known-third-party=enchant
# Analyse import fallback blocks. This can be used to support both Python 2 and
# 3 compatible code, which means that the block might have code that exists
# only in one or another interpreter, leading to false positives when analysed.
analyse-fallback-blocks=no
[EXCEPTIONS]
# Exceptions that will emit a warning when being caught. Defaults to
# "BaseException, Exception".
overgeneral-exceptions=BaseException, Exception
# osdu-airflow-lib
# OSDU Airflow Library
The project contains all Airflow dependencies need to OSDU.
\ No newline at end of file
OSDU Airflow Library is the package providing Airflow specific logic.
## Contents
* [Getting Started](#getting-started)
* * [Backward Compatibility](#backward-compatibility)
* * [Installation from source](#installation-from-source)
* * [Installation from Package Registry](#installation-from-package-registry)
* [Package Lifecycle](#package-lifecycle)
* [Licence](#licence)
# Getting Started
## Backward Compatibility
Airflow 1.10.15 is as a “bridge” release but in OSDU Airflow 1.10.10 version should be supported.
## Installation from source
1. Pull the latest changes from https://community.opengroup.org/osdu/platform/data-flow/ingestion/osdu-airflow-lib
2. Use Python 3.6. Also, it is highly recommended using an isolated virtual environment for development purposes
(Creation of virtual environments: https://docs.python.org/3.6/library/venv.html)
3. Make sure you have setuptools and wheel installed
```sh
pip install --upgrade setuptools wheel
```
4. Change directory to the root of the project
```sh
cd path/to/osdu-airflow-lib
```
5. Make sure osdu-airflow isn't already installed
```sh
pip uninstall osdu-airflow
````
6. Install OSDU Airflow
```sh
python setup.py install
```
Example import after installing:
```python
from osdu_airflow.backward_compatibility.default_args import update_default_args
```
## Installation from Package Registry
```sh
pip install 'osdu-airflow' --extra-index-url=https://community.opengroup.org/api/v4/projects/668/packages/pypi/simple
```
# Package Lifecycle
The project can be deleted once Airflow 1.10.10 version support will be deprecated and no any additional logic will be added.
## Licence
Copyright © Google LLC
Copyright © EPAM Systems
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
[http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0)
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
A package to interface with OSDU microservices
# Copyright 2021 Google LLC
# Copyright 2021 EPAM Systems
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Copyright 2021 Google LLC
# Copyright 2021 EPAM Systems
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Copyright 2021 Google LLC
# Copyright 2021 EPAM Systems
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
""" Mock airflow.utils"""
import logging
from typing import Callable
import airflow
def deprecated_function_decorator(func: Callable):
"""
This decorator is used to mock deprecated decorators if we use Airflow 2.0
:param func:
:return:
"""
def wrapper(*args, **kwargs):
return func(*args, **kwargs)
return wrapper
if not airflow.__version__.startswith("2"):
logging.warning(
"'apply_defaults' is going to be removed in Airflow 2. Do not use it in the future.")
from airflow.utils import apply_defaults
else:
logging.warning("'apply_defaults' is removed in Airflow 2. "
"It is used here due to backward compatibility.")
apply_defaults = deprecated_function_decorator
# Copyright 2021 Google LLC
# Copyright 2021 EPAM Systems
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
import airflow
def update_default_args(default_args: dict) -> dict:
"""
Update default args of tasks with necessary args depending on Airflow version
:param default_args:
:return:
"""
if not airflow.__version__.startswith("2"):
default_args.update(
{
"provide_context": True
}
)
logging.warning(f"'provide_context' argument is going to be removed in Airflow 2.")
else:
logging.info(f"Airflow {airflow.__version__} is used. No need to update 'default_args'.")
return default_args
# Copyright 2021 Google LLC
# Copyright 2021 EPAM Systems
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import setuptools
with open("README.md", "r") as fh:
long_description = fh.read()
def get_version_from_file():
with open("VERSION", "r") as fh:
return fh.read().strip()
def prepare_version():
version = os.getenv("CI_COMMIT_TAG", '')
if version.startswith('v'):
# release tag version, e.g. v0.9.0
version = version[1:]
else:
# we assume that it is commit version
# https://packaging.python.org/guides/distributing-packages-using-setuptools/#local-version-identifiers
commit = os.environ["CI_COMMIT_SHORT_SHA"]
version = f"{get_version_from_file()}.dev+{commit}"
return version
setuptools.setup(
name="osdu_airflow",
version=prepare_version(),
author="OSDU team",
description="A package with Airflow dependencies specific for OSDU",
packages=setuptools.find_packages(exclude=["*test*"]),
classifiers=[
"Programming Language :: Python :: 3",
"License :: OSI Approved :: Apache License",
"Operating System :: OS Independent",
],
python_requires='>=3.6',
)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment