.gitlab-ci.yml 6.42 KB
Newer Older
1
#  Copyright 2020 Google LLC
2
#  Copyright 2020 EPAM Systems
3
4
5
6
7
8
9
10
11
12
13
14
15
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

16
17
18
19
variables:
  OSDU_GCP_ENABLE_BOOTSTRAP: "true"
  OSDU_GCP_SERVICE: "ingestion-dags"

Scott Howard's avatar
Scott Howard committed
20
  #AWS variables
21
22
23
24
25
26
27
28
29
30
31
32
33
  AWS_DEPLOY_TARGET: EKS
  # AWS skip stages
  SKIP_BOOTSTRAP: 'true'
  SKIP_PUBLISH: 'true'
  SKIP_END_TO_END: 'false'
  # AWS collection vars
  AWS_COLLECTION_PATH: https://community.opengroup.org/osdu/platform/testing/-/raw/master/Postman%20Collection/29_CICD_Setup_Ingestion/R3%20Full%20manifest-based%20ingestion.postman_collection.json
  AWS_COLLECTION_ENV_PATH: https://community.opengroup.org/osdu/platform/testing/-/raw/master/Postman%20Collection/00_CICD_Setup_Environment/aws.OSDU%20R3%20PROD%20v2.4.postman_environment.json
  AWS_COLLECTION_TEST_PATH: ''
  AWS_TEST_FILE_INPUT: ''
  AWS_TEST_FILE_KEY: ''
  AWS_POSTMAN_TESTS_TO_SKIP: "01a - Reference data ingestion|01b - Workflow - Get Work Status|01c - Search Records|02a - Master data ingestion|02b - Workflow - Get Work Status|02c - Search Records|03a - Master data ingestion|03b - Workflow - Get Work Status|03c - Search Records|05a - WP ingestion, pre-defined ids crazy format|05b - Workflow - Get Work Status|05c - Search Records WP|05d - Search Records WPC|05e - Search Records Dataset|06a - WP ingestion surrogate keys|06b - Workflow - Get Work Status|07a - WP ingestion, pre-defined ids short format|07b - Workflow - Get Work Status|07c - Search Records WP|07d - Search Records WPC|07e - Search Records Dataset|08a - WP ingestion File Collection|08b - Workflow - Get Work Status|08c - Search Records WP|08d - Search Records WPC|08e - Search Records Dataset|09a - WP ingestion Artefact|09b - Workflow - Get Work Status|10a - Multiple manifests|10b - Workflow - Get Work Status|10c - Search Records MasterData|09d - Search Records Reference Data|11a - 3 types|11b - Workflow - Get Work Status|11c - Search Records WP|11d - Search Records WPC|11e - Search Records Dataset|11f- Search Records MasterData|11g - Search Records Reference Data|12a - NS parent data doesnt exist|12b - Workflow - Get Work Status|13a - NS extra WPC in WP block|13b - Workflow - Get Work Status|14a - NS extra Dataset in WPC block|14b - Workflow - Get Work Status|15a - NS File Collection no FileSource|15b - Workflow - Get Work Status|16a - NS File no File Source|16b - Workflow - Get Work Status"

34
image: google/cloud-sdk:alpine
35

36
stages:
37
  - linters
38
39
  - unit_tests
  - test_dags
40
  - containerize
41
  - bootstrap_dag
42
  - publish_dag
43
  - deploy
44
  - bootstrap
45
  - end_to_end_postman_tests
46

47
include:
48
49
  - project: "osdu/platform/ci-cd-pipelines"
    file: "cloud-providers/azure_dag.yml"
50

51
52
  - project: "osdu/platform/ci-cd-pipelines"
    file: "build/python-linters.yml"
53

54
55
  - project: "osdu/platform/ci-cd-pipelines"
    file: "cloud-providers/osdu-gcp-dag.yaml"
56

57
58
  - project: "osdu/platform/ci-cd-pipelines"
    file: "cloud-providers/osdu-gcp-bootstrap.yml"
59

Walter D's avatar
Walter D committed
60
61
  - local: "/devops/ibm/ibm-stages.yml"

62
63
64
65
66
67
  - project: "osdu/platform/ci-cd-pipelines"
    file: "cloud-providers/aws-global-variables.yml"

  - project: "osdu/platform/ci-cd-pipelines"
    file: "cloud-providers/aws-dag.yml"

68
69
70
71
72
73
74
75
76
77
osdu-gcp-containerize-bootstrap-gitlab:
  variables:
    IMAGE_NAME: osdu-gcp-bootstrap-dags
    BUILD_PATH: devops/gcp/Dockerfile

osdu-gcp-containerize-bootstrap-gcr:
  variables:
    IMAGE_NAME: osdu-gcp-bootstrap-dags
    BUILD_PATH: devops/gcp/Dockerfile

78
pylint:
79
80
81
82
  variables:
    SRC_FILES: "src/*/*.py"
  before_script:
    - pip install -r src/osdu_dags/requirements.txt
83
84

isort:
85
86
  variables:
    SRC_FILES: "src/*/*.py"
87

88
test_dags:
89
  tags: ["osdu-small"]
90
  stage: test_dags
91
  needs: ["pylint", "isort"]
92
  image: eu.gcr.io/osdu-cicd-epam/airflow-python-dags/airflow-python-dags:latest
93
94
95
96
97
98
99
100
101
  script:
    - chmod +x tests/test_dags.sh
    - tests/./test_dags.sh  || EXIT_CODE=$?
    - exit ${EXIT_CODE}
  artifacts:
    paths:
      - logs/
    when: on_failure
    expire_in: 1 week
102

103
unit_tests:
104
  tags: ["osdu-small"]
105
  stage: unit_tests
106
  needs: ["pylint", "isort"]
107
  image: eu.gcr.io/osdu-cicd-epam/airflow-python-dags/airflow-python-dags:latest
108
109
  script:
    - chmod +x tests/unit_tests.sh
110
    # - tests/./unit_tests.sh  || EXIT_CODE=$? #TODO: unit tests moved to airflow-osdu-lib, need to remove `unit_tests` CI step later
111
    - exit ${EXIT_CODE}
112

113
114
115
116
117
118
osdu_gcp_publish_dag:
  stage: publish_dag
  variables:
    BOOTSTRAPPED_DAG_PATH: src/osdu_dags/osdu-ingest-r3.py
    DAG_FILE: osdu-ingest-r3.py

Yan Sushchynski (EPAM)'s avatar
Yan Sushchynski (EPAM) committed
119
120
121
122
123
124
osdu_anthos_publish_dag:
  stage: publish_dag
  variables:
    BOOTSTRAPPED_DAG_PATH: src/osdu_dags/osdu-ingest-r3.py
    DAG_FILE: osdu-ingest-r3.py

125
osdu_gcp_deploy_composer_community:
126
  stage: deploy
127
128
129
130
131
  needs: ["unit_tests", "test_dags"]
  variables:
    DAG_FOLDER: "src/osdu_dags"

osdu_gcp_deploy_composer_pre_ship:
132
  stage: deploy
133
134
135
136
137
  needs: ["unit_tests", "test_dags"]
  variables:
    DAG_FOLDER: "src/osdu_dags"

osdu_gcp_postman_tests_community:
138
  stage: end_to_end_postman_tests
139
140
141
  variables:
    TEST_SOURCE_FILES_DIR: "Postman Collection/29_CICD_Setup_Ingestion"
    COLLECTION_PATH: "Postman Collection/29_CICD_Setup_Ingestion/R3 Full manifest-based ingestion.postman_collection.json"
142

143
osdu_gcp_postman_tests_pre_ship:
144
  stage: end_to_end_postman_tests
145
146
147
148
149
150
  variables:
    TEST_SOURCE_FILES_DIR: "Postman Collection/29_CICD_Setup_Ingestion"
    COLLECTION_PATH: "Postman Collection/29_CICD_Setup_Ingestion/R3 Full manifest-based ingestion.postman_collection.json"

osdu_gcp_bootstrap_dag_community:
  stage: deploy
151
  variables:
152
153
154
155
    OSDU_GCP_SKIP_BOOTSTRAP: "true"

osdu_gcp_bootstrap_dag_pre_ship:
  stage: deploy
156
  variables:
157
    OSDU_GCP_SKIP_BOOTSTRAP: "true"
158

Yan Sushchynski (EPAM)'s avatar
Yan Sushchynski (EPAM) committed
159
160
161
162
163
osdu_anthos_bootstrap_dag:
  stage: deploy
  variables:
    OSDU_ANTHOS_SKIP_BOOTSTRAP: "true"

164
165
166
# We are skipping this stage using AZURE_SKIP_STANDALONE_TESTS variable as there is already a common stage for Executing Unit
# tests, but we need to override here because of stage name mismatch
azure_standalone_tests:
167
168
169
170
  stage: unit_tests
  needs: ["pylint"]
  variables:
    AZURE_SKIP_STANDALONE_TESTS: "true"
171
172

azure_build_dag:
173
  needs: ["unit_tests"]