osdu-gcp-cloudrun.yml 13.3 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
# Copyright 2020 Google LLC
# Copyright 2017-2019, Schlumberger
# Copyright 2020 EPAM
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

17
18
19
20
21
22
23
# Global Protected Variables:
#
# OSDU_GCP
# OSDU_GCP_DEPLOY_FILE
# OSDU_GCP_SERVICE_ACCOUNT
# OSDU_GCP_INTEGRATION_TESTER
# OSDU_GCP_NO_DATA_ACCESS_TESTER
24

25
26
27
include:
  - local: 'cloud-providers/osdu-gcp-global.yml'

28
.osdu-gcp-variables:
29
  variables:
30
    # services' urls
Aliaksandr Ramanovich (EPAM)'s avatar
Aliaksandr Ramanovich (EPAM) committed
31
    OSDU_GCP_STORAGE_HOSTNAME: os-storage-attcrcktoa-uc.a.run.app
32
    OSDU_GCP_LEGALTAG_API: https://community.osdu-gcp.go3-nrg.projects.epam.com/api/legal/v1
33
    OSDU_GCP_CRS_API: https://crs-converter-attcrcktoa-uc.a.run.app/api/crs/v2
34
    OSDU_GCP_REGISTER_URL: https://os-register-attcrcktoa-uc.a.run.app/
Aliaksandr Ramanovich (EPAM)'s avatar
Aliaksandr Ramanovich (EPAM) committed
35
    OSDU_GCP_ENTITLEMENTS_URL: https://os-entitlements-gcp-attcrcktoa-uc.a.run.app/entitlements/v1/
36
37
    OSDU_GCP_ENTITLEMENTS_V2_URL: https://community.osdu-gcp.go3-nrg.projects.epam.com/api/entitlements/v2/
    OSDU_GCP_ENTITLEMENTS_V2_BASE_URL: https://community.osdu-gcp.go3-nrg.projects.epam.com/api
38
    OSDU_GCP_AUTHORIZE_API: https://os-entitlements-gcp-attcrcktoa-uc.a.run.app/entitlements/v1/
Aliaksandr Ramanovich (EPAM)'s avatar
Aliaksandr Ramanovich (EPAM) committed
39
40
    OSDU_GCP_STORAGE_URL: https://os-storage-attcrcktoa-uc.a.run.app/api/storage/v2/
    OSDU_GCP_RECORDS_ROOT_URL: https://os-storage-attcrcktoa-uc.a.run.app/api/storage/v2/
41
42
    OSDU_GCP_LEGAL_HOST_URL: https://community.osdu-gcp.go3-nrg.projects.epam.com/api/legal/v1/
    OSDU_GCP_LEGAL_HOSTNAME: https://community.osdu-gcp.go3-nrg.projects.epam.com/
43
    OSDU_GCP_AIRFLOW_URL: https://y7ae53ee6eb377feap-tp.appspot.com/
44
    OSDU_GCP_DELIVERY_SERVICE_URL: https://os-delivery-attcrcktoa-uc.a.run.app/api/delivery/v2/
45
46
    OSDU_GCP_INGEST_HOST: https://os-ingest-attcrcktoa-uc.a.run.app/
    OSDU_GCP_WORKFLOW_SERVICE_URL: https://os-workflow-attcrcktoa-uc.a.run.app/
Aliaksandr Ramanovich (EPAM)'s avatar
Aliaksandr Ramanovich (EPAM) committed
47
    OSDU_GCP_SEARCH_HOST: https://os-search-attcrcktoa-uc.a.run.app/api/search/v2/
48
    OSDU_GCP_BACKUP_HOST: https://backup-attcrcktoa-uc.a.run.app
Aliaksandr Ramanovich (EPAM)'s avatar
Aliaksandr Ramanovich (EPAM) committed
49
    OSDU_GCP_SEARCH_QUERY_URL: https://os-search-attcrcktoa-uc.a.run.app/api/search/v2/query/
50
51
    OSDU_GCP_INDEXER_HOST: https://os-indexer-attcrcktoa-uc.a.run.app/
    OSDU_GCP_INDEXER_HOST_SEARCH: https://os-indexer-attcrcktoa-uc.a.run.app/api/indexer/v2/
52
    OSDU_GCP_FILE_URL: https://os-file-attcrcktoa-uc.a.run.app
53
    OSDU_GCP_NOTIFICATION_URL: https://os-notification-attcrcktoa-uc.a.run.app/
54
55
56
57
    OSDU_GCP_STORAGE_QUERY_RECORD_HOST: https://os-storage-attcrcktoa-uc.a.run.app/api/storage/v2/query/records
    OSDU_GCP_STORAGE_SCHEMA_HOST: https://os-storage-attcrcktoa-uc.a.run.app/api/storage/v2/schemas
    OSDU_GCP_STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST: https://os-storage-attcrcktoa-uc.a.run.app/api/storage/v2/query/records:batch
    OSDU_GCP_INDEXER_QUEUE_HOST: https://indexer-queue-attcrcktoa-uc.a.run.app/api/indexer/v1/_dps/task-handlers/enqueue
58
    OSDU_GCP_UNIT_HOSTNAME: os-unit-attcrcktoa-uc.a.run.app
59
    OSDU_GCP_CRS_CONVERTER_HOSTNAME: crs-converter-attcrcktoa-uc.a.run.app
60
    OSDU_GCP_CRS_CATALOG_HOSTNAME: os-crs-catalog-attcrcktoa-uc.a.run.app
61
    OSDU_GCP_SCHEMA_URL: https://os-schema-attcrcktoa-uc.a.run.app
62
    OSDU_GCP_DATASET_URL: https://os-dataset-attcrcktoa-uc.a.run.app/api/dataset/v1/
63
64
    OSDU_GCP_PARTITION_API: https://os-partition-attcrcktoa-uc.a.run.app/api/partition/v1/
    OSDU_GCP_POLICY_API: https://community.osdu-gcp.go3-nrg.projects.epam.com/api/policy/v1/
65
    OSDU_GCP_LOCAL_IMAGE_TAG_SHA: $CI_REGISTRY_IMAGE/osdu-gcp:$CI_COMMIT_SHORT_SHA
66
67
68
69
    REDIS_GROUP_HOST: 10.234.198.27
    REDIS_SEARCH_HOST: 10.99.138.107
    REDIS_STORAGE_HOST: 10.16.59.203
    REDIS_SYNC_HOST: 10.116.62.35
70
    OSDU_GCP_REDIS_SEARCH_PORT: 6379
71
    OSDU_GCP_SPRING_PROFILES_ACTIVE: dev
72
    OSDU_GCP_INDEXER_QUEUE_CLOUDTASK_NAME: os-indexer-queue-osdu
73
74
    # variables for unit deployment
    OSDU_GCP_UNIT_CATALOG_BUCKET: nice-etching-277309-unit-catalog-bucket
75
    # common variables
Aliaksandr Ramanovich (EPAM)'s avatar
Aliaksandr Ramanovich (EPAM) committed
76
77
78
    OSDU_GCP_CLOUDRUN_REGION: us-central1
    OSDU_GCP_PROJECT: nice-etching-277309
    OSDU_GCP_VPC_CONNECTOR: osdu-connector
79
    OSDU_GCP_PORT: 8080
80
81
82
83
84
85
86
    #variables for seismic-store deployment
    OSDU_GCP_CLOUD_PROVIDER: google
    OSDU_GCP_IMP_SERVICE_ACCOUNT_SIGNER: unknown
    OSDU_GCP_SEISTORE_DES_APPKEY: na
    OSDU_GCP_DES_REDIS_INSTANCE_ADDRESS: 10.93.210.3
    OSDU_GCP_DES_REDIS_INSTANCE_PORT: 6379
    OSDU_GCP_LOCKSMAP_REDIS_INSTANCE_ADDRESS: 10.93.210.3
Vladislav Shishko (EPAM)'s avatar
Vladislav Shishko (EPAM) committed
87
88
89
90
    # variables for WKS service:
    OSDU_GCP_WKS_SEARCH_API: https://os-search-attcrcktoa-uc.a.run.app/api/search/v2
    OSDU_GCP_WKS_STORAGE_API: https://os-storage-attcrcktoa-uc.a.run.app/api/storage/v2
    OSDU_GCP_WKS_SCHEMA_API: https://os-schema-attcrcktoa-uc.a.run.app/api/schema-service/v1
91
92
93
    # common variables for tests
    GOOGLE_AUDIENCE: 689762842995-pv217jo3k8j803kk6gqf52qb5amos3a9.apps.googleusercontent.com
    DOMAIN: osdu-gcp.go3-nrg.projects.epam.com
94
    LEGAL_TAG: osdu-demo-legaltag
95
96
97
    OTHER_RELEVANT_DATA_COUNTRIES: US
    DEFAULT_DATA_PARTITION_ID_TENANT1: osdu
    GCLOUD_PROJECT: nice-etching-277309
Aliaksandr Ramanovich (EPAM)'s avatar
Aliaksandr Ramanovich (EPAM) committed
98
99
    # variables for storage tests
    TENANT_NAME: osdu
100
    # variables for workflow tests
101
    OSDU_GCP_ENTITLEMENTS_APPKEY: workflow-service
Oleksandr Kosse (EPAM)'s avatar
Oleksandr Kosse (EPAM) committed
102
    FINISHED_WORKFLOW_ID: fad778da-fbc4-4261-8b3e-deb48be44969
103
    OSDU_GCP_TEST_DAG_NAME: airflow_monitoring
104
    # variables for ingestion tests
105
    TEST_OSDU_FILE_PATH: gs://nice-etching-277309-file/r1/data/provided/well-logs/1013_akm11_1978_comp.las
106
107
108
109
110
    # variables for file deployment and tests
    OSDU_GCP_DATASTORE_NAMESPACE: osdu-namespace
    FILE_BUCKET: nice-etching-277309-file
    ACL_OWNERS: data.default.owners
    ACL_VIEWERS: data.default.viewers
111
112
113
114
115
116
117
    TARGET_AUDIENCE: 689762842995-pv217jo3k8j803kk6gqf52qb5amos3a9.apps.googleusercontent.com
    DATA_PARTITION_ID: osdu
    USER_ID: common-user
    TIME_ZONE: UTC+0
    # variables for delivery tests
    ENTITLEMENTS_DOMAIN: osdu-gcp.go3-nrg.projects.epam.com
    TENANT: osdu
118
    #DEFAULT_DATA_PARTITION_ID_TENANT2: osdu
119
    INTEGRATION_TEST_AUDIENCE: 689762842995-pv217jo3k8j803kk6gqf52qb5amos3a9.apps.googleusercontent.com
120
121
    # variables for register tests, and deployment
    SERVICE_IDENTITY: osdu-gcp-sa
122
    RECORDS_CHANGE_PUBSUB_ENDPOINT: https://os-notification-attcrcktoa-uc.a.run.app/push-handlers/records-changed
123
    CRON_JOB_EXPECTED_IP: 0:0:0:0:0:0:0:1
124
125
126
    SUBSCRIBER_PRIVATE_KEY_ID: $OSDU_GCP_SUBSCRIBER_PRIVATE_KEY_ID
    SUBSCRIBER_SECRET: $OSDU_GCP_SUBSCRIBER_SECRET
    SUBSCRIPTION_ID: $OSDU_GCP_REGISTER_SUBSCRIPTION_ID
127
    REGISTER_CUSTOM_PUSH_URL: https://os-register-attcrcktoa-uc.a.run.app/api/register/v1/test/challenge/1
128
129
130
    CLIENT_TENANT: nonexistenttenant
    OSDU_TENANT: osdu
    REGISTER_BASE_URL: https://os-register-attcrcktoa-uc.a.run.app/
131
    ENVIRONMENT: dev
132
133
134
135
136
    # variables for notification tests
    HMAC_SECRET: $OSDU_GCP_SUBSCRIBER_SECRET
    REGISTER_CUSTOM_PUSH_URL_HMAC: https://os-register-attcrcktoa-uc.a.run.app/api/register/v1/test/challenge/hmac-integration-test
    TOPIC_ID: records-changed
    NOTIFICATION_BASE_URL: $OSDU_GCP_NOTIFICATION_URL
Aliaksandr Ramanovich (EPAM)'s avatar
Aliaksandr Ramanovich (EPAM) committed
137
138
139
140
141
    # variables for legal tests
    MY_TENANT_PROJECT: osdu
    MY_TENANT: osdu
    ENABLE_FULL_BUCKET_NAME: "true"
    SKIP_HTTP_TESTS: "true"
142
143
144
145
146
147
148
149
    # variables for search tests
    DATA_GROUP: osdu
    ENTITLEMENTS_HOST: $OSDU_GCP_ENTITLEMENTS_URL
    ELASTIC_PASSWORD: $OSDU_GCP_ELASTIC_PASSWORD
    ELASTIC_USER_NAME: $OSDU_GCP_ELASTIC_USER
    ELASTIC_HOST: $OSDU_GCP_ELASTIC_HOST
    ELASTIC_PORT: $OSDU_GCP_ELASTIC_PORT
    DATA_PARTITION_ID_TENANT1: osdu
150
151
152
153
154
155
    # variables for schema tests
    VENDOR: $OSDU_GCP_VENDOR
    HOST: $OSDU_GCP_SCHEMA_URL
    PRIVATE_TENANT1: osdu
    PRIVATE_TENANT2: osdu
    SHARED_TENANT: osdu
156
    # variables for partition tests
157
    PARTITION_BASE_URL: https://os-partition-attcrcktoa-uc.a.run.app/
158
159
    # variables for backup tests
    BACKUP_SERVICE_HOST: $OSDU_GCP_BACKUP_HOST
160

161
osdu-gcp-containerize-gcloud:
Aliaksandr Ramanovich (EPAM)'s avatar
Aliaksandr Ramanovich (EPAM) committed
162
  stage: containerize
163
  needs: ["compile-and-unit-test"]
164
  extends: .osdu-gcp-variables
165
  image: gcr.io/google.com/cloudsdktool/cloud-sdk
166
  cache: {}
167
  script:
168
    - gcloud auth activate-service-account --key-file $OSDU_GCP_DEPLOY_FILE
Aliaksandr Ramanovich (EPAM)'s avatar
Aliaksandr Ramanovich (EPAM) committed
169
    - gcloud config set project $OSDU_GCP_PROJECT
170
    - touch .gcloudignore
171
    - gcloud builds submit --config provider/$OSDU_GCP_SERVICE-$OSDU_GCP_VENDOR/cloudbuild/cloudbuild.yaml --substitutions=_GCP_SERVICE=$OSDU_GCP_SERVICE,_APPLICATION_NAME=$OSDU_GCP_APPLICATION_NAME,_PROVIDER_NAME=$OSDU_GCP_VENDOR,_SHORT_SHA=$CI_COMMIT_SHORT_SHA,_PORT=$OSDU_GCP_PORT
172
173
174
175
  only:
    variables:
      - $OSDU_GCP == 'true'

176
177
178
osdu-gcp-containerize-gitlab:
  stage: containerize
  needs: ["compile-and-unit-test"]
179
  tags: ["osdu-medium"]
180
  extends: .osdu-gcp-variables
181
  image: docker:19.03
182
  cache: {}
183
  allow_failure: true
184
185
186
187
  script:
    - export EXTRA_DOCKER_TAG=""; if [ "$CI_COMMIT_TAG" != "" ] ; then EXTRA_DOCKER_TAG="-t $CI_REGISTRY_IMAGE/osdu-gcp:$CI_COMMIT_TAG" ; elif [ "$CI_COMMIT_REF_NAME" = "master" ] ; then EXTRA_DOCKER_TAG="-t $CI_REGISTRY_IMAGE/osdu-gcp:latest" ; fi
    - docker build -t $OSDU_GCP_LOCAL_IMAGE_TAG_SHA $EXTRA_DOCKER_TAG --file provider/$OSDU_GCP_SERVICE-$OSDU_GCP_VENDOR/cloudbuild/Dockerfile.cloudbuild --build-arg PROVIDER_NAME=$OSDU_GCP_VENDOR --build-arg PORT=$OSDU_GCP_PORT .
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
188
    - docker push $CI_REGISTRY_IMAGE/osdu-gcp
189
190
191
192
  only:
    variables:
      - $OSDU_GCP == 'true'

193
osdu-gcp-deploy:
194
  image: gcr.io/google.com/cloudsdktool/cloud-sdk
195
  needs: ["osdu-gcp-containerize-gcloud"]
Aliaksandr Ramanovich (EPAM)'s avatar
Aliaksandr Ramanovich (EPAM) committed
196
  stage: deploy
197
  extends: .osdu-gcp-variables
198
  cache: {}
199
  script:
200
    - gcloud auth activate-service-account --key-file $OSDU_GCP_DEPLOY_FILE
Aliaksandr Ramanovich (EPAM)'s avatar
Aliaksandr Ramanovich (EPAM) committed
201
    - gcloud config set project $OSDU_GCP_PROJECT
202
    - >
203
204
205
      gcloud beta run deploy $OSDU_GCP_APPLICATION_NAME
      --image gcr.io/$OSDU_GCP_PROJECT/$OSDU_GCP_APPLICATION_NAME/$OSDU_GCP_SERVICE-$OSDU_GCP_VENDOR:$CI_COMMIT_SHORT_SHA
      --platform managed
206
      --region $OSDU_GCP_CLOUDRUN_REGION
207
208
209
210
      --allow-unauthenticated
      --service-account $OSDU_GCP_SERVICE_ACCOUNT
      --memory 512M
      $OSDU_GCP_CLOUD_RUN_PARAMETERS
211
      --set-env-vars=LOG_LEVEL=${OSDU_GCP_LOG_LEVEL:="INFO"},$OSDU_GCP_ENV_VARS
Aliaksandr Ramanovich (EPAM)'s avatar
Aliaksandr Ramanovich (EPAM) committed
212
    - gcloud run services update-traffic $OSDU_GCP_APPLICATION_NAME --platform managed --region $OSDU_GCP_CLOUDRUN_REGION --to-latest
213
214
  only:
    variables:
215
      - $OSDU_GCP == 'true'
216
217

osdu-gcp-test:
218
  image: maven:3.3.9-jdk-8
219
220
  extends:
    - .osdu-gcp-variables
Aliaksandr Ramanovich (EPAM)'s avatar
Aliaksandr Ramanovich (EPAM) committed
221
    - .maven
222
  stage: integration
223
  needs: ["osdu-gcp-deploy"]
224
  allow_failure: false
225
226
227
228
229
230
231
232
  variables:
    WORKFLOW_HOST: $OSDU_GCP_WORKFLOW_SERVICE_URL
    INGEST_HOST: $OSDU_GCP_INGEST_HOST
    TEST_FILE_ID: $OSDU_GCP_TEST_FILE_ID
    FILE_SERVICE_HOST: $OSDU_GCP_FILE_URL
    DELIVERY_HOST: $OSDU_GCP_DELIVERY_SERVICE_URL
    SEARCH_HOST: $OSDU_GCP_SEARCH_HOST
    STORAGE_HOST: $OSDU_GCP_STORAGE_URL
233
234
235
236
    DE_OPS_TESTER: $OSDU_GCP_R3_OPS_TESTER
    DE_ADMIN_TESTER: $OSDU_GCP_R3_ADMIN_TESTER
    DE_EDITOR_TESTER: $OSDU_GCP_R3_EDITOR_TESTER
    DE_NO_ACCESS_TESTER: $OSDU_GCP_R3_NO_ACCESS_TESTER
Aliaksandr Ramanovich (EPAM)'s avatar
Aliaksandr Ramanovich (EPAM) committed
237
238
239
240
241
242
243
244
    STORAGE_URL: $OSDU_GCP_STORAGE_URL
    DEPLOY_ENV: empty
    LEGAL_URL: $OSDU_GCP_LEGAL_HOST_URL
    PUBSUB_TOKEN: $GCP_PUBSUB_TOKEN
    INTEGRATION_TESTER: $OSDU_GCP_INTEGRATION_TESTER
    NO_DATA_ACCESS_TESTER: $OSDU_GCP_NO_DATA_ACCESS_TESTER
    SEARCH_INTEGRATION_TESTER: $OSDU_GCP_INTEGRATION_TESTER
    HOST_URL: $OSDU_GCP_LEGAL_HOST_URL
245
246
    INDEXER_HOST: $OSDU_GCP_INDEXER_HOST_SEARCH
    OSDU_GCP_TESTING_DIR: testing
Aliaksandr Ramanovich (EPAM)'s avatar
Aliaksandr Ramanovich (EPAM) committed
247
    TEST_DAG_NAME: $OSDU_GCP_TEST_DAG_NAME
248
  script:
249
    - export GCP_DEPLOY_FILE=$(echo $OSDU_GCP_DEPLOY_FILE)
250
    - >
251
252
253
      mvn
      --batch-mode
      --settings=$CI_PROJECT_DIR/.mvn/community-maven.settings.xml
254
      -f $OSDU_GCP_TESTING_DIR/pom.xml
255
256
257
      verify
      --projects $OSDU_GCP_SERVICE-test-core,$OSDU_GCP_SERVICE-test-gcp
      2>&1 | tee test-results.log
258
259
  only:
    variables:
260
      - $OSDU_GCP == 'true' && $OSDU_GCP_INT_TEST_TYPE != 'python'
261
262
263
264
  artifacts:
    when: always
    paths:
      - test-results.log
Vladislav Shishko (EPAM)'s avatar
Vladislav Shishko (EPAM) committed
265
      - $OSDU_GCP_TEST_SUBDIR/target/*/TEST-*.xml
266
    expire_in: 1 week
267
268
    reports:
      junit:
Vladislav Shishko (EPAM)'s avatar
Vladislav Shishko (EPAM) committed
269
        - $OSDU_GCP_TEST_SUBDIR/target/*/TEST-*.xml
270
271
272
273
274
275
276

osdu-gcp-test-python:
  extends:
    - .osdu-gcp-variables
  variables:
    INTEGRATION_TESTER: $OSDU_GCP_INTEGRATION_TESTER
    MY_TENANT: osdu
277
    GOOGLE_AUDIENCES: $GOOGLE_AUDIENCE
278
279
  stage: integration
  image: python:3.7
280
  needs: ["osdu-gcp-deploy"]
281
282
  allow_failure: false
  script:
283
    - cd $OSDU_GCP_PYTHON_INT_TEST_SUBDIR
284
285
286
287
288
    - chmod +x ./run-integration-tests.sh
    - ./run-integration-tests.sh
  only:
    variables:
      - $OSDU_GCP == 'true' && $OSDU_GCP_INT_TEST_TYPE == 'python'
Aliaksandr Ramanovich (EPAM)'s avatar
Aliaksandr Ramanovich (EPAM) committed
289
290
291
292
293
294
295
296
297
298
299
300

osdu-gcp-tagging:
  image: google/cloud-sdk
  needs: ["osdu-gcp-deploy"]
  stage: integration
  extends: .osdu-gcp-variables
  cache: {}
  script:
    - gcloud auth activate-service-account --key-file $OSDU_GCP_DEPLOY_FILE
    - gcloud config set project $OSDU_GCP_PROJECT
    - gcloud container images add-tag gcr.io/$OSDU_GCP_PROJECT/$OSDU_GCP_APPLICATION_NAME/$OSDU_GCP_SERVICE-$OSDU_GCP_VENDOR:$CI_COMMIT_SHORT_SHA gcr.io/$OSDU_GCP_PROJECT/$OSDU_GCP_APPLICATION_NAME/$OSDU_GCP_SERVICE-$OSDU_GCP_VENDOR:$CI_COMMIT_TAG
  only:
301
    - tags