.gitlab-ci.yml 2.7 KB
Newer Older
1
2
3
4
5
variables:
  BUILD_TAG: $CI_COMMIT_TAG
  BUILD_COMMIT_SHORT_SHA: $CI_COMMIT_SHORT_SHA
  BUILD_ID: $CI_PIPELINE_IID

Siarhei Khaletski (EPAM)'s avatar
Siarhei Khaletski (EPAM) committed
6
7
8
9
10
default:
  image: python:3.6-slim-buster

stages:
  - linters
11
  - unit_tests
12
  - publish
Siarhei Khaletski (EPAM)'s avatar
Siarhei Khaletski (EPAM) committed
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
  - deploy

pylint:
  stage: linters
  allow_failure: true
  script:
    - python -m pip install setuptools pylint pylint_quotes pylint-exit
    - pylint --rcfile=.pylintrc osdu_airflow/*/*.py || EXIT_CODE=$?
    - exit ${EXIT_CODE}

isort:
  allow_failure: true
  stage: linters
  script:
    - python -m pip install setuptools isort
    - isort -c -v osdu_airflow/*/*.py || EXIT_CODE=$?
    - exit ${EXIT_CODE}

31
unit_tests_airflow_v1:
32
  stage: unit_tests
33
34
35
  image: 
    name: apache/airflow:1.10.14-python3.6 
    entrypoint: [""]
36
  script:
37
38
39
40
41
42
43
44
45
46
47
48
    - pip install -r requirements-dev.txt
    - sh -c "./osdu_airflow/tests/./unit_tests.sh"  || EXIT_CODE=$?
    - exit ${EXIT_CODE}

unit_tests_airflow_v2:
  stage: unit_tests
  image: 
    name: apache/airflow:2.1.2-python3.8
    entrypoint: [""]
  script:
    - pip install -r requirements-dev.txt
    - sh -c "./osdu_airflow/tests/./unit_tests.sh"  || EXIT_CODE=$?
49
50
    - exit ${EXIT_CODE}

51
52
publish:
  stage: publish
Siarhei Khaletski (EPAM)'s avatar
Siarhei Khaletski (EPAM) committed
53
54
55
  script:
    - pip install twine
    - python setup.py sdist bdist_wheel
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
    - TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python -m twine upload --repository-url ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi dist/* > output.txt && cat output.txt
  after_script:
    - cat output.txt | grep 'Uploading ' | grep -Po '(?<=osdu_airflow-)[0-9.a-z_+\-]*(?=.tar.gz)' > published_version.txt
  artifacts:
    paths:
      - published_version.txt
    expire_in: 1 day
  when: manual

.gcp-deploy-composer-env:
  image: gcr.io/google.com/cloudsdktool/cloud-sdk
  stage: deploy
  dependencies:
    - publish
  script:
    - cat published_version.txt
    - export LIB_VERSION=$(cat published_version.txt)
    - gcloud composer environments update $OSDU_GCP_ENV_VAR --location=us-central1 --update-pypi-package=osdu-airflow~=${LIB_VERSION%+*}
Siarhei Khaletski (EPAM)'s avatar
Siarhei Khaletski (EPAM) committed
74
  when: manual
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100

.gcp-activate-sa-comm: &gcp-activate-sa-comm
  - gcloud auth activate-service-account --key-file $OSDU_GCP_DEPLOY_FILE

.gcp-activate-sa-prod: &gcp-activate-sa-prod
  - gcloud auth activate-service-account --key-file $OSDU_GCP_PROD_DEPLOY_FILE

gcp-deploy-composer-env-comm:
  extends: .gcp-deploy-composer-env
  variables:
    OSDU_GCP_ENV_VAR: osdu-airflow
  before_script:
    - *gcp-activate-sa-comm
    - gcloud config set project nice-etching-277309
  except:
    - /^release\/*/

gcp-deploy-composer-env-prod:
  extends: .gcp-deploy-composer-env
  variables:
    OSDU_GCP_ENV_VAR: airflow-v2
  before_script:
    - *gcp-activate-sa-prod
    - gcloud config set project osdu-service-prod
  only:
    - /^release\/*/