osdu-gcp-gke.yml 10.4 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
# Copyright 2021 Google LLC
# Copyright 2017-2019, Schlumberger
# Copyright 2021 EPAM
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Global Protected Variables:
#
# OSDU_GCP
# OSDU_GCP_DEPLOY_FILE
# OSDU_GCP_SERVICE_ACCOUNT
# OSDU_GCP_INTEGRATION_TESTER
# OSDU_GCP_NO_DATA_ACCESS_TESTER

include:
  - local: "cloud-providers/osdu-gcp-global.yml"

.osdu-gcp-variables:
  variables:
    # services' urls
    OSDU_GCP_AUTHORIZE_API: http://entitlements/api/entitlements/v2/
32
    OSDU_GCP_PARTITION_API: http://partition/api/partition/v1/
Oleksandr Kosse (EPAM)'s avatar
Oleksandr Kosse (EPAM) committed
33
    OSDU_GCP_LEGALTAG_API: http://legal/api/legal/v1/
34
    OSDU_GCP_INDEXER_API: http://indexer/api/indexer/v2/
Oleksandr Kosse (EPAM)'s avatar
Oleksandr Kosse (EPAM) committed
35
    OSDU_GCP_CRS_API: https://crs-converter-attcrcktoa-uc.a.run.app/api/crs/converter/v2/
36
    OSDU_GCP_SEARCH_API: http://search/api/search/v2/
Oleksandr Kosse (EPAM)'s avatar
Oleksandr Kosse (EPAM) committed
37
38
    OSDU_GCP_STORAGE_URL: http://storage/api/storage/v2/
    OSDU_GCP_RECORDS_ROOT_URL: http://storage/api/storage/v2/
39
    OSDU_GCP_FILE_URL: https://community.osdu-gcp.go3-nrg.projects.epam.com/api/file
40
    OSDU_GCP_SCHEMA_URL: https://os-schema-attcrcktoa-uc.a.run.app/api/schema-service/v1/schema
41
    OSDU_GCP_INDEXER_QUEUE_URL: https://community.osdu-gcp.go3-nrg.projects.epam.com/api/indexer-queue/v1/_dps/task-handlers/enqueue
42
    OSDU_GCP_WELLBORE_BASE_URL: https://community.osdu-gcp.go3-nrg.projects.epam.com/api/os-wellbore-ddms
43
    OSDU_GCP_SERVICE_HOST_SEARCH: https://community.osdu-gcp.go3-nrg.projects.epam.com/api/search
44
    OSDU_GCP_POLICY_API: https://community.osdu-gcp.go3-nrg.projects.epam.com/api/policy/v1/
45
    OSDU_GCP_UNIT_HOSTNAME: community.osdu-gcp.go3-nrg.projects.epam.com
46
    OSDU_GCP_WELL_DELIVERY: https://community.osdu-gcp.go3-nrg.projects.epam.com/api/well-delivery/
47
48
49
50
51
52
    # common helm variables
    OSDU_GCP_HELM_DEPLOYMENT_DIR: "devops/gcp/deploy"
    OSDU_GCP_HELM_CONFIG_DIR: "devops/gcp/configmap"
    OSDU_GCP_HELM_NAMESPACE: default
    # common variables
    OSDU_GCP_PROJECT: nice-etching-277309
53
    OSDU_GCP_TENANT: osdu
54
55
    OSDU_GCP_CLUSTER: asm-primary
    OSDU_GCP_ZONE: us-central1-c
56
    OSDU_GCP_CLOUDRUN_REGION: us-central1
57
58
    OSDU_GCP_PORT: 8080
    REDIS_GROUP_HOST: 10.234.198.27
59
    REDIS_SEARCH_HOST: 10.99.138.107
60
    REDIS_STORAGE_HOST: 10.16.59.203
61
    REDIS_SYNC_HOST: 10.116.62.35
62
63
64
65
    OSDU_GCP_LOCAL_IMAGE_TAG_SHA: $CI_REGISTRY_IMAGE/osdu-gcp:$CI_COMMIT_SHORT_SHA
    # variables for partition
    OSDU_GCP_PARTITION_KEY_RING: csqp
    OSDU_GCP_PARTITION_KMS_KEY: managerService
66
    # variables for indexer-queue
67
68
    OSDU_GCP_INDEXER_QUEUE_CLOUDTASK_NAME: os-indexer-queue-osdu
    OSDU_GCP_INDEXER_HOST: https://community.osdu-gcp.go3-nrg.projects.epam.com
69
70
71
72
73
74
75
    # variables for file deployment and tests
    OSDU_GCP_DATASTORE_NAMESPACE: osdu-namespace
    FILE_BUCKET: nice-etching-277309-file
    ACL_OWNERS: data.default.owners
    ACL_VIEWERS: data.default.viewers
    TARGET_AUDIENCE: 689762842995-pv217jo3k8j803kk6gqf52qb5amos3a9.apps.googleusercontent.com
    DATA_PARTITION_ID: osdu
Oleksii Tsyganov (EPAM)'s avatar
Oleksii Tsyganov (EPAM) committed
76
    USER_ID: common-user
77
    TIME_ZONE: UTC+0
78
79
80
    SHARED_TENANT: osdu
    PRIVATE_TENANT1: osdu
    PRIVATE_TENANT2: osdu
81
82
    # variables for unit deployment
    OSDU_GCP_UNIT_CATALOG_BUCKET: nice-etching-277309-unit-catalog-bucket
83
84
85
86
87
88
89
90
91
92
93
94
    # common variables for tests
    DOMAIN: osdu-gcp.go3-nrg.projects.epam.com
    GOOGLE_AUDIENCE: 689762842995-pv217jo3k8j803kk6gqf52qb5amos3a9.apps.googleusercontent.com
    INTEGRATION_TEST_AUDIENCE: 689762842995-pv217jo3k8j803kk6gqf52qb5amos3a9.apps.googleusercontent.com
    LEGAL_TAG: osdu-demo-legaltag
    OTHER_RELEVANT_DATA_COUNTRIES: US
    DEFAULT_DATA_PARTITION_ID_TENANT1: osdu
    GCLOUD_PROJECT: nice-etching-277309
    OSDU_GCP_TESTING_DIR: testing
    OSDU_GCP_TEST_SUBDIR: testing/$OSDU_GCP_SERVICE-test-$OSDU_GCP_VENDOR
    ENVIRONMENT: dev
    PARTITION_BASE_URL: https://community.osdu-gcp.go3-nrg.projects.epam.com/
95
    GOOGLE_AUDIENCES: 689762842995-pv217jo3k8j803kk6gqf52qb5amos3a9.apps.googleusercontent.com
96
97
98
99
100
101
102
103
104
    # variables for legal tests
    MY_TENANT_PROJECT: osdu
    MY_TENANT: osdu
    SKIP_HTTP_TESTS: "true"
    HOST_URL: https://community.osdu-gcp.go3-nrg.projects.epam.com/api/legal/v1/
    ENABLE_FULL_BUCKET_NAME: "true"
    # entitlements integration tests
    TENANT_NAME: osdu
    OSDU_GCP_ENTITLEMENTS_URL: https://community.osdu-gcp.go3-nrg.projects.epam.com/api/entitlements/v2/
105
106
107
108
109
110
111
112
113
    # indexer int tests
    ENTITLEMENTS_HOST: https://community.osdu-gcp.go3-nrg.projects.epam.com/api/entitlements/v2/
    ELASTIC_PASSWORD: $OSDU_GCP_ELASTIC_PASSWORD
    ELASTIC_USER_NAME: $OSDU_GCP_ELASTIC_USER
    ELASTIC_HOST: $OSDU_GCP_ELASTIC_HOST
    ELASTIC_PORT: $OSDU_GCP_ELASTIC_PORT
    INDEXER_HOST: https://community.osdu-gcp.go3-nrg.projects.epam.com/api/indexer/v2/
    DATA_GROUP: osdu
    ENTITLEMENTS_DOMAIN: $DOMAIN
114
    SEARCH_HOST: https://community.osdu-gcp.go3-nrg.projects.epam.com/api/search/v2/
115
116
    STORAGE_HOST: https://community.osdu-gcp.go3-nrg.projects.epam.com/api/storage/v2/
    PARTITION_API: https://community.osdu-gcp.go3-nrg.projects.epam.com/api/partition/v1
117
118
119
    # well delivery
    # NO ENDING SLASH !!!!!
    OSDU_WELL_DELIVERY_SCHEMA_SERVICE_ENDPOINT: https://community.osdu-gcp.go3-nrg.projects.epam.com/api/schema-service/v1
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140

.common: &common
  - gcloud auth activate-service-account --key-file $OSDU_GCP_DEPLOY_FILE
  - gcloud config set project $OSDU_GCP_PROJECT
  - gcloud container clusters get-credentials $OSDU_GCP_CLUSTER --zone $OSDU_GCP_ZONE --project $OSDU_GCP_PROJECT
  - curl https://get.helm.sh/helm-v3.5.4-linux-amd64.tar.gz -s -o helm.tgz; tar -zxf helm.tgz; mv linux-amd64/helm /usr/local/bin/

osdu-gcp-containerize-gitlab:
  stage: containerize
  needs: ["compile-and-unit-test"]
  tags: ["osdu-medium"]
  extends: .osdu-gcp-variables
  image: docker:19.03
  cache: {}
  script:
    - export EXTRA_DOCKER_TAG=""; if [ "$CI_COMMIT_TAG" != "" ] ; then EXTRA_DOCKER_TAG="-t $CI_REGISTRY_IMAGE/osdu-gcp:$CI_COMMIT_TAG" ; elif [ "$CI_COMMIT_REF_NAME" = "master" ] ; then EXTRA_DOCKER_TAG="-t $CI_REGISTRY_IMAGE/osdu-gcp:latest" ; fi
    - docker build -t $OSDU_GCP_LOCAL_IMAGE_TAG_SHA $EXTRA_DOCKER_TAG --file provider/$OSDU_GCP_SERVICE-$OSDU_GCP_VENDOR/cloudbuild/Dockerfile.cloudbuild --build-arg PROVIDER_NAME=$OSDU_GCP_VENDOR --build-arg PORT=$OSDU_GCP_PORT .
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
    - docker push $CI_REGISTRY_IMAGE/osdu-gcp
  only:
    variables:
Oleksii Tsyganov (EPAM)'s avatar
Oleksii Tsyganov (EPAM) committed
141
      - $OSDU_GCP == 'true'
142
143
144
145
146
147
148
149
150
151
152
153
154
155

osdu-gcp-deploy-configmap:
  extends: .osdu-gcp-variables
  image: gcr.io/google.com/cloudsdktool/cloud-sdk
  stage: deploy
  needs: ["compile-and-unit-test"]
  script:
    - *common
    - >
      helm upgrade $OSDU_GCP_HELM_CONFIG_SERVICE $OSDU_GCP_HELM_CONFIG_DIR
      --install
      --create-namespace
      --namespace=$OSDU_GCP_HELM_NAMESPACE
      --wait
156
      --history-max=3
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
      $OSDU_GCP_HELM_CONFIG_SERVICE_VARS
  only:
    variables:
      - $OSDU_GCP == 'true'

osdu-gcp-deploy-deployment:
  extends: .osdu-gcp-variables
  image: gcr.io/google.com/cloudsdktool/cloud-sdk
  stage: deploy
  needs: ["osdu-gcp-containerize-gitlab"]
  script:
    - *common
    - >
      helm upgrade $OSDU_GCP_HELM_DEPLOYMENT_SERVICE $OSDU_GCP_HELM_DEPLOYMENT_DIR
      --install
      --create-namespace
      --namespace=$OSDU_GCP_HELM_NAMESPACE
      --wait
175
      --history-max=3
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
      $OSDU_GCP_HELM_DEPLOYMENT_SERVICE_VARS
    - echo ----- Verify Deployment -----
    - kubectl rollout status deployment.v1.apps/$OSDU_GCP_SERVICE -n $OSDU_GCP_HELM_NAMESPACE --timeout=900s
    - POD=$(kubectl get pod --sort-by=.metadata.creationTimestamp -n $OSDU_GCP_HELM_NAMESPACE | grep $OSDU_GCP_SERVICE | tail -1 | awk '{print $1}')
    - STATUS=$(kubectl wait -n $OSDU_GCP_HELM_NAMESPACE --for=condition=Ready pod/$POD --timeout=300s)
    - echo $STATUS
    - if [[ "$STATUS" != *"met"* ]]; then echo "POD didn't start correctly" ; exit 1 ; fi
  only:
    variables:
      - $OSDU_GCP == 'true'

osdu-gcp-test:
  image: maven:3.3.9-jdk-8
  extends:
    - .osdu-gcp-variables
    - .maven
  stage: integration
  needs: ["osdu-gcp-deploy-deployment"]
  variables:
    INTEGRATION_TESTER: $OSDU_GCP_INTEGRATION_TESTER
    NO_DATA_ACCESS_TESTER: $OSDU_GCP_NO_DATA_ACCESS_TESTER
    SEARCH_INTEGRATION_TESTER: $OSDU_GCP_INTEGRATION_TESTER
    DE_OPS_TESTER: $OSDU_GCP_R3_OPS_TESTER
    DE_ADMIN_TESTER: $OSDU_GCP_R3_ADMIN_TESTER
    DE_EDITOR_TESTER: $OSDU_GCP_R3_EDITOR_TESTER
    DE_NO_ACCESS_TESTER: $OSDU_GCP_R3_NO_ACCESS_TESTER
    ENTITLEMENT_V2_URL: $OSDU_GCP_ENTITLEMENTS_URL
203
    FILE_SERVICE_HOST: $OSDU_GCP_FILE_URL
Oleksandr Kosse (EPAM)'s avatar
Oleksandr Kosse (EPAM) committed
204
205
206
207
    DEPLOY_ENV: empty
    LEGAL_URL: https://community.osdu-gcp.go3-nrg.projects.epam.com/api/legal/v1/
    PUBSUB_TOKEN: $GCP_PUBSUB_TOKEN
    STORAGE_URL: https://community.osdu-gcp.go3-nrg.projects.epam.com/api/storage/v2/
208
    HOST: https://os-schema-attcrcktoa-uc.a.run.app
209
210
211
    # well delivery
    SCHEMA_URL: https://community.osdu-gcp.go3-nrg.projects.epam.com/api/schema-service/v1/
    ENTITY_URL: $OSDU_GCP_WELL_DELIVERY
212
213
214
215
216
217
218
219
220
  script:
    - export GCP_DEPLOY_FILE=$(echo $OSDU_GCP_DEPLOY_FILE)
    - >
      mvn
      --batch-mode
      --settings=$CI_PROJECT_DIR/.mvn/community-maven.settings.xml
      -f $OSDU_GCP_TESTING_DIR/pom.xml
      verify
      --projects $OSDU_GCP_SERVICE-test-core,$OSDU_GCP_SERVICE-test-$OSDU_GCP_VENDOR
221
      --update-snapshots
222
223
224
225
226
227
228
229
230
231
232
233
234
      2>&1 | tee test-results.log
  only:
    variables:
      - $OSDU_GCP == 'true' && $OSDU_GCP_INT_TEST_TYPE != 'python'
  artifacts:
    when: always
    paths:
      - test-results.log
      - $OSDU_GCP_TEST_SUBDIR/target/*/TEST-*.xml
    expire_in: 1 week
    reports:
      junit:
        - $OSDU_GCP_TEST_SUBDIR/target/*/TEST-*.xml
235
236
237
238
239
240
241
242
243
244

osdu-gcp-test-python:
  extends:
    - .osdu-gcp-variables
  variables:
    INTEGRATION_TESTER: $OSDU_GCP_INTEGRATION_TESTER
    MY_TENANT: osdu
    GOOGLE_AUDIENCES: $GOOGLE_AUDIENCE
  stage: integration
  image: python:3.7
245
  needs: ["osdu-gcp-deploy-deployment"]
246
247
248
249
250
251
252
253
  allow_failure: false
  script:
    - cd $OSDU_GCP_PYTHON_INT_TEST_SUBDIR
    - chmod +x ./run-integration-tests.sh
    - ./run-integration-tests.sh
  only:
    variables:
      - $OSDU_GCP == 'true' && $OSDU_GCP_INT_TEST_TYPE == 'python'