.gitlab-ci.yml 7.81 KB
Newer Older
ethiraj krishnamanaidu's avatar
ethiraj krishnamanaidu committed
1
# Copyright 2021 Schlumberger
2
# Copyright 2021 EPAM
ethiraj krishnamanaidu's avatar
ethiraj krishnamanaidu committed
3
4
5
6
7
8
9
10
11
12
13
14
15
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

16
variables:
17
  PIP_REQUIREMENTS: "requirements.txt requirements_dev.txt"
fabian serin's avatar
fabian serin committed
18

Yunhua Koglin's avatar
Yunhua Koglin committed
19
20
21
22
  AZURE_SERVICE: wellbore-ddms
  AZURE_DOCKER_SUBDIR: build/Dockerfile
  AZURE_TEST_SUBDIR: tests/integration
  AZURE_TEST_TYPE: python
23

Mykola Zamkovyi (EPAM)'s avatar
Mykola Zamkovyi (EPAM) committed
24
  AWS_SERVICE: wellbore-ddms
Yunhua Koglin's avatar
Yunhua Koglin committed
25
  AWS_SERVICE_NAMESPACE: osdu-wellbore-ddms
Mykola Zamkovyi (EPAM)'s avatar
Mykola Zamkovyi (EPAM) committed
26
27
28
29
  AWS_ENVIRONMENT: dev
  AWS_BUILD_SUBDIR: provider/os-wellbore-ddms-aws/build-aws
  AWS_TEST_SUBDIR: tests/aws-test/build-aws
  AWS_INT_TEST_TYPE: python
Yunhua Koglin's avatar
Yunhua Koglin committed
30
31
  AWS_DEPLOY_TARGET: EKS
  AWS_EKS_DEPLOYMENT_NAME: os-wellbore-ddms
Mykola Zamkovyi (EPAM)'s avatar
Mykola Zamkovyi (EPAM) committed
32

Oleksandr Kosse (EPAM)'s avatar
Oleksandr Kosse (EPAM) committed
33
  OSDU_GCP_SERVICE: wellbore
34
35
  OSDU_GCP_VENDOR: gcp
  OSDU_GCP_HELM_PACKAGE_CHARTS: "devops/gcp/deploy devops/gcp/configmap"
36
  OSDU_GCP_HELM_CONFIG_SERVICE_VARS: "--set data.os_wellbore_ddms_data_project_id=$OSDU_GCP_PROJECT"
37
  OSDU_GCP_HELM_DEPLOYMENT_SERVICE_VARS: "--set data.image=$CI_REGISTRY_IMAGE/osdu-gcp:$CI_COMMIT_SHORT_SHA --set data.serviceAccountName=$OSDU_GCP_SERVICE-k8s"
38
39
40
  OSDU_GCP_HELM_CONFIG_SERVICE: wellbore-config
  OSDU_GCP_HELM_DEPLOYMENT_SERVICE: wellbore-deploy
  OSDU_GCP_INT_TEST_TYPE: python
41

42
43
44
45
include:
  - project: "osdu/platform/ci-cd-pipelines"
    file: "standard-setup.yml"

46
47
48
  - project: "osdu/platform/ci-cd-pipelines"
    file: "build/python.yml"

49
  - project: "osdu/platform/ci-cd-pipelines"
50
    file: "scanners/fossa-python.yml"
51

52
53
54
  - project: "osdu/platform/ci-cd-pipelines"
    file: "scanners/gitlab-ultimate.yml"

55
  - project: "osdu/platform/ci-cd-pipelines"
56
    file: "cloud-providers/osdu-gcp-gke.yml"
vladislav's avatar
vladislav committed
57

Oleksandr Kosse (EPAM)'s avatar
Oleksandr Kosse (EPAM) committed
58
  - project: "osdu/platform/ci-cd-pipelines"
Anuj Gupta's avatar
Anuj Gupta committed
59
    file: "cloud-providers/ibm-wellbore-git.yml"
60

Oleksandr Kosse (EPAM)'s avatar
Oleksandr Kosse (EPAM) committed
61
  - local: "/devops/azure/azure-wellbore.yml"
62

Oleksandr Kosse (EPAM)'s avatar
Oleksandr Kosse (EPAM) committed
63
  - project: "osdu/platform/ci-cd-pipelines"
Yunhua Koglin's avatar
Yunhua Koglin committed
64
    file: "cloud-providers/aws-global.yml"
Mykola Zamkovyi (EPAM)'s avatar
Mykola Zamkovyi (EPAM) committed
65

66
67
68
  - project: "osdu/platform/ci-cd-pipelines"
    file: "cloud-providers/aws-python.yml"

69
70
71
72
73
74
75
76
77
78
verify_existing_requirements:
  extends:
    - .python
    - .skipForTriggeringMergeRequests

  stage: build
  before_script:
    - pip3 install --upgrade pip-tools

  script:
79
80
    - pip-compile requirements.in
    - pip-compile requirements_dev.in
81
    - git diff
82
83
    # checking difference between existing requirements and the newly generated one
    # and using the number of lines as exit status code
84
    - exit $(git status -s | grep -e "^ M " | wc -l)
85
86
87
88

  artifacts:
    when: always
    paths:
89
      - "requirements*.txt"
90
91
92

    expire_in: 2 days

93
94
95
96
97
98
99
compile-and-unit-test:
  artifacts:
    when: always
    paths:
      - all-requirements.txt
      - spec/generated/openapi.json

100
containerize:
101
  extends: .skipForTriggeringMergeRequests
102
103
  stage: containerize
  image: docker:19.03
104
  tags: ["osdu-medium"]
105
  variables:
106
    IMAGE_TAG: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG:$CI_COMMIT_SHA
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
  script:
    - echo ---- ---- ---- SYSTEM DEPENDENCIES ---- ---- ----
    - apk update
    - apk add git
    - echo ---- ---- ---- BUILD IMAGE ---- ---- ----
    - commit_id=$(git log -n 1 --pretty="%h")
    - echo ---- ---- TAG NAME
    - tag_name="_gitlab_$commit_id"
    - echo $tag_name
    - echo ---- ---- DATE
    - current_utc_date=`date --utc`
    - echo $current_utc_date
    - echo ---- ---- COMMIT BRANCH
    - commit_branch=$commit_id
    - echo $commit_branch
    - echo ---- ---- BUILD IMAGE
    - docker build -t $IMAGE_TAG -t=$CI_REGISTRY_IMAGE:latest --rm . -f ./build/Dockerfile --build-arg PIP_WHEEL_DIR=python-packages --build-arg build_date="$current_utc_date" --build-arg build_number=$commit_id --build-arg commit_id=$commit_id --build-arg build_origin="Gitlab" --build-arg commit_branch=$commit_branch
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
    - echo ---- ---- PUSH IMAGE
    - docker push $IMAGE_TAG
    - docker push $CI_REGISTRY_IMAGE:latest

129
osdu-gcp-containerize-gitlab:
130
  stage: containerize
131
  image: docker:19.03
132
  cache: {}
133
  tags: ["osdu-medium"]
134
  variables:
135
136
    IMAGE_TAG: $CI_REGISTRY_IMAGE/osdu-gcp:$CI_COMMIT_SHORT_SHA
    IMAGE_TAG_LATEST: $CI_REGISTRY_IMAGE/osdu-gcp:latest
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
  script:
    - echo ---- ---- ---- SYSTEM DEPENDENCIES ---- ---- ----
    - apk update
    - apk add git
    - echo ---- ---- ---- BUILD IMAGE ---- ---- ----
    - commit_id=$(git log -n 1 --pretty="%h")
    - echo ---- ---- TAG NAME
    - tag_name="_gitlab_$commit_id"
    - echo $tag_name
    - echo ---- ---- DATE
    - current_utc_date=`date --utc`
    - echo $current_utc_date
    - echo ---- ---- COMMIT BRANCH
    - commit_branch=$commit_id
    - echo $commit_branch
    - echo ---- ---- BUILD IMAGE
153
    - docker build -t $IMAGE_TAG -t $IMAGE_TAG_LATEST --rm . -f ./build/Dockerfile --build-arg PIP_WHEEL_DIR=python-packages --build-arg build_date="$current_utc_date" --build-arg build_number=$commit_id --build-arg commit_id=$commit_id --build-arg build_origin="Gitlab" --build-arg commit_branch=$commit_branch
154
155
156
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
    - echo ---- ---- PUSH IMAGE
    - docker push $IMAGE_TAG
157
    - docker push $IMAGE_TAG_LATEST
158

159
160
161
162
osdu-gcp-containerize-for-release:
  stage: containerize
  image: docker:19.03
  cache: {}
163
  tags: ["osdu-medium"]
164
165
  only:
    variables:
166
      - $OSDU_GCP == 'true' && $CI_COMMIT_TAG != null
167
  variables:
168
    IMAGE_TAG_RELEASE: $CI_REGISTRY_IMAGE/osdu-gcp:$CI_COMMIT_TAG
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
  script:
    - echo ---- ---- ---- SYSTEM DEPENDENCIES ---- ---- ----
    - apk update
    - apk add git
    - echo ---- ---- ---- BUILD IMAGE ---- ---- ----
    - commit_id=$(git log -n 1 --pretty="%h")
    - echo ---- ---- TAG NAME
    - tag_name="_gitlab_$commit_id"
    - echo $tag_name
    - echo ---- ---- DATE
    - current_utc_date=`date --utc`
    - echo $current_utc_date
    - echo ---- ---- COMMIT BRANCH
    - commit_branch=$commit_id
    - echo $commit_branch
    - echo ---- ---- BUILD IMAGE
185
    - docker build -t $IMAGE_TAG_RELEASE --rm . -f ./build/Dockerfile --build-arg PIP_WHEEL_DIR=python-packages --build-arg build_date="$current_utc_date" --build-arg build_number=$commit_id --build-arg commit_id=$commit_id --build-arg build_origin="Gitlab" --build-arg commit_branch=$commit_branch
186
187
188
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
    - echo ---- ---- PUSH IMAGE
    - docker push $IMAGE_TAG_RELEASE
Yunhua Koglin's avatar
Yunhua Koglin committed
189

190
osdu-gcp-test-python:
191
  extends: .osdu-gcp-variables
192
  stage: integration
193
  image: gcr.io/google.com/cloudsdktool/cloud-sdk
194
  needs: ["osdu-gcp-deploy-deployment"]
195
196
197
198
199
200
  script:
    - apt-get install -y python3-venv
    - python3 -m venv env
    - source env/bin/activate
    - pip install --upgrade pip
    - pip install wheel pytest pytest-cov
Luc Yriarte's avatar
Luc Yriarte committed
201
202
203
204
    - >
      for REQ in $PIP_REQUIREMENTS ; do
        pip install -r $REQ
      done
205
    - cd tests/integration
206
    - echo $OSDU_GCP_INTEGRATION_TESTER | base64 -d > file.json
207
    - gcloud auth activate-service-account --key-file file.json
208
    - gcloud config set project $OSDU_GCP_PROJECT
209
210
    - >
      python gen_postman_env.py
211
      --token $(gcloud auth print-access-token)
212
      --base_url $OSDU_GCP_WELLBORE_BASE_URL
213
214
215
      --cloud_provider $OSDU_GCP_VENDOR
      --data_partition $OSDU_GCP_TENANT
      --acl_domain $DOMAIN
216
      --legal_tag $LEGAL_TAG
Luc Yriarte's avatar
Luc Yriarte committed
217
    - pytest ./functional --environment="./generated/postman_environment.json" --filter-tag=!search
218

219
220
221
222
# Disable maven job in gcp common gke pipeline
osdu-gcp-test:
  extends:
    - .osdu-gcp-variables
223

224
225
226
227
osdu-gcp-dev2-test:
  extends:
    - .osdu-gcp-dev2-variables

228
229
230
# Allow failure on private development deployments
ibm-deploy-devpri:
  allow_failure: true