osdu-gcp-cloudrun.yml 7.88 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
# Copyright 2020 Google LLC
# Copyright 2017-2019, Schlumberger
# Copyright 2020 EPAM
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

17
18
19
20
21
22
23
# Global Protected Variables:
#
# OSDU_GCP
# OSDU_GCP_DEPLOY_FILE
# OSDU_GCP_SERVICE_ACCOUNT
# OSDU_GCP_INTEGRATION_TESTER
# OSDU_GCP_NO_DATA_ACCESS_TESTER
24
.osdu-gcp-variables:
25
  variables:
Aliaksandr Ramanovich (EPAM)'s avatar
Aliaksandr Ramanovich (EPAM) committed
26
27
28
    OSDU_GCP_STORAGE_HOSTNAME: os-storage-attcrcktoa-uc.a.run.app
    OSDU_GCP_LEGALTAG_API: https://os-legal-attcrcktoa-uc.a.run.app/api/legal/v1
    OSDU_GCP_CRS_API: https://os-crs-converter-gae-dot-nice-etching-277309.appspot.com/api/crs/v2
29
    OSDU_GCP_REGISTER_URL: https://os-register-attcrcktoa-uc.a.run.app/
Aliaksandr Ramanovich (EPAM)'s avatar
Aliaksandr Ramanovich (EPAM) committed
30
    OSDU_GCP_ENTITLEMENTS_URL: https://os-entitlements-gcp-attcrcktoa-uc.a.run.app/entitlements/v1/
31
    OSDU_GCP_AUTHORIZE_API: https://os-entitlements-gcp-attcrcktoa-uc.a.run.app/entitlements/v1/
Aliaksandr Ramanovich (EPAM)'s avatar
Aliaksandr Ramanovich (EPAM) committed
32
33
34
35
    OSDU_GCP_STORAGE_URL: https://os-storage-attcrcktoa-uc.a.run.app/api/storage/v2/
    OSDU_GCP_RECORDS_ROOT_URL: https://os-storage-attcrcktoa-uc.a.run.app/api/storage/v2/
    OSDU_GCP_LEGAL_HOST_URL: https://os-legal-attcrcktoa-uc.a.run.app/api/legal/v1/
    OSDU_GCP_LEGAL_HOSTNAME: https://os-legal-attcrcktoa-uc.a.run.app/
36
    OSDU_GCP_AIRFLOW_URL: https://c410714dc037b0e02p-tp.appspot.com/
37
    OSDU_GCP_DELIVERY_SERVICE_URL: https://os-delivery-attcrcktoa-uc.a.run.app/api/delivery/v2/
38
39
    OSDU_GCP_INGEST_HOST: https://os-ingest-attcrcktoa-uc.a.run.app/
    OSDU_GCP_WORKFLOW_SERVICE_URL: https://os-workflow-attcrcktoa-uc.a.run.app/
Aliaksandr Ramanovich (EPAM)'s avatar
Aliaksandr Ramanovich (EPAM) committed
40
41
    OSDU_GCP_SEARCH_HOST: https://os-search-attcrcktoa-uc.a.run.app/api/search/v2/
    OSDU_GCP_SEARCH_QUERY_URL: https://os-search-attcrcktoa-uc.a.run.app/api/search/v2/query/
42
    OSDU_GCP_INDEXER_HOST: https://os-indexer-dot-nice-etching-277309.uc.r.appspot.com/
43
    OSDU_GCP_FILE_URL: https://os-file-attcrcktoa-uc.a.run.app/
44
    OSDU_GCP_NOTIFICATION_URL: https://os-notification-attcrcktoa-uc.a.run.app/
45
46
47
48
    REDIS_GROUP_HOST: 10.234.198.27
    REDIS_SEARCH_HOST: 10.99.138.107
    REDIS_STORAGE_HOST: 10.16.59.203
    REDIS_SYNC_HOST: 10.116.62.35
49
    # common variables
Aliaksandr Ramanovich (EPAM)'s avatar
Aliaksandr Ramanovich (EPAM) committed
50
51
52
    OSDU_GCP_CLOUDRUN_REGION: us-central1
    OSDU_GCP_PROJECT: nice-etching-277309
    OSDU_GCP_VPC_CONNECTOR: osdu-connector
53
    OSDU_GCP_PORT: 8080
54
55
56
    # common variables for tests
    GOOGLE_AUDIENCE: 689762842995-pv217jo3k8j803kk6gqf52qb5amos3a9.apps.googleusercontent.com
    DOMAIN: osdu-gcp.go3-nrg.projects.epam.com
57
    LEGAL_TAG: osdu-demo-legaltag
58
59
60
    OTHER_RELEVANT_DATA_COUNTRIES: US
    DEFAULT_DATA_PARTITION_ID_TENANT1: osdu
    GCLOUD_PROJECT: nice-etching-277309
Aliaksandr Ramanovich (EPAM)'s avatar
Aliaksandr Ramanovich (EPAM) committed
61
62
    # variables for storage tests
    TENANT_NAME: osdu
63
    # variables for workflow tests
Oleksandr Kosse (EPAM)'s avatar
Oleksandr Kosse (EPAM) committed
64
    FINISHED_WORKFLOW_ID: fad778da-fbc4-4261-8b3e-deb48be44969
65
    # variables for ingestion tests
66
    TEST_OSDU_FILE_PATH: gs://nice-etching-277309-file/r1/data/provided/well-logs/1013_akm11_1978_comp.las
67
68
69
70
71
72
73
74
75
76
77
    # variables for file tests
    TARGET_AUDIENCE: 689762842995-pv217jo3k8j803kk6gqf52qb5amos3a9.apps.googleusercontent.com
    DATA_PARTITION_ID: osdu
    USER_ID: common-user
    TIME_ZONE: UTC+0
    FILE_BUCKET: nice-etching-277309-file
    # variables for delivery tests
    ENTITLEMENTS_DOMAIN: osdu-gcp.go3-nrg.projects.epam.com
    TENANT: osdu
    DEFAULT_DATA_PARTITION_ID_TENANT2: osdu
    INTEGRATION_TEST_AUDIENCE: 689762842995-pv217jo3k8j803kk6gqf52qb5amos3a9.apps.googleusercontent.com
78
79
    # variables for register tests, and deployment
    SERVICE_IDENTITY: osdu-gcp-sa
80
    RECORDS_CHANGE_PUBSUB_ENDPOINT: https://os-notification-attcrcktoa-uc.a.run.app/push-handlers/records-changed
81
    CRON_JOB_EXPECTED_IP: 0:0:0:0:0:0:0:1
82
83
84
85
86
87
    SUBSCRIBER_PRIVATE_KEY_ID: $OSDU_GCP_SUBSCRIBER_PRIVATE_KEY_ID
    SUBSCRIBER_SECRET: $OSDU_GCP_SUBSCRIBER_SECRET
    DE_OPS_TESTER: $OSDU_GCP_R3_OPS_TESTER
    DE_ADMIN_TESTER: $OSDU_GCP_R3_ADMIN_TESTER
    DE_EDITOR_TESTER: $OSDU_GCP_R3_EDITOR_TESTER
    DE_NO_ACCESS_TESTER: $OSDU_GCP_R3_NO_ACCESS_TESTER
88
    REGISTER_BASE_URL: https://os-register-attcrcktoa-uc.a.run.app/
89
    CLIENT_TENANT: nonexistenttenant
90
    OSDU_TENANT: osdu
91
    SUBSCRIPTION_ID: $OSDU_GCP_REGISTER_SUBSCRIPTION_ID
92
    REGISTER_CUSTOM_PUSH_URL: https://os-register-attcrcktoa-uc.a.run.app/api/register/v1/test/challenge/1
Aliaksandr Ramanovich (EPAM)'s avatar
Aliaksandr Ramanovich (EPAM) committed
93
94
95
96
97
    # variables for legal tests
    MY_TENANT_PROJECT: osdu
    MY_TENANT: osdu
    ENABLE_FULL_BUCKET_NAME: "true"
    SKIP_HTTP_TESTS: "true"
98

Aliaksandr Ramanovich (EPAM)'s avatar
Aliaksandr Ramanovich (EPAM) committed
99
osdu-gcp-containerize:
Aliaksandr Ramanovich (EPAM)'s avatar
Aliaksandr Ramanovich (EPAM) committed
100
  stage: containerize
101
  needs: [ "compile-and-unit-test" ]
102
  extends: .osdu-gcp-variables
103
  image: google/cloud-sdk
104
  cache: { }
105
  script:
106
    - gcloud auth activate-service-account --key-file $OSDU_GCP_DEPLOY_FILE
Aliaksandr Ramanovich (EPAM)'s avatar
Aliaksandr Ramanovich (EPAM) committed
107
    - gcloud config set project $OSDU_GCP_PROJECT
108
    - touch .gcloudignore
Aliaksandr Ramanovich (EPAM)'s avatar
Aliaksandr Ramanovich (EPAM) committed
109
    - gcloud builds submit --config provider/$OSDU_GCP_SERVICE-$OSDU_GCP_VENDOR/cloudbuild/cloudbuild.yaml --substitutions=_PROVIDER_NAME=$OSDU_GCP_VENDOR,_SHORT_SHA=$CI_COMMIT_SHORT_SHA,_PORT=$OSDU_GCP_PORT
110
111
112
113
  only:
    variables:
      - $OSDU_GCP == 'true'

114
osdu-gcp-deploy:
115
  image: google/cloud-sdk
116
  needs: [ "osdu-gcp-containerize" ]
Aliaksandr Ramanovich (EPAM)'s avatar
Aliaksandr Ramanovich (EPAM) committed
117
  stage: deploy
118
  extends: .osdu-gcp-variables
119
  cache: { }
120
  script:
121
    - gcloud auth activate-service-account --key-file $OSDU_GCP_DEPLOY_FILE
Aliaksandr Ramanovich (EPAM)'s avatar
Aliaksandr Ramanovich (EPAM) committed
122
    - gcloud config set project $OSDU_GCP_PROJECT
123
    - gcloud run deploy $OSDU_GCP_APPLICATION_NAME --image gcr.io/$OSDU_GCP_PROJECT/$OSDU_GCP_APPLICATION_NAME/$OSDU_GCP_SERVICE-$OSDU_GCP_VENDOR:$CI_COMMIT_SHORT_SHA --platform managed --region $OSDU_GCP_CLOUDRUN_REGION --allow-unauthenticated --service-account $OSDU_GCP_SERVICE_ACCOUNT --memory 512M --set-env-vars=$OSDU_GCP_ENV_VARS
Aliaksandr Ramanovich (EPAM)'s avatar
Aliaksandr Ramanovich (EPAM) committed
124
    - gcloud run services update-traffic $OSDU_GCP_APPLICATION_NAME --platform managed --region $OSDU_GCP_CLOUDRUN_REGION --to-latest
125
126
  only:
    variables:
127
      - $OSDU_GCP == 'true'
128
129

osdu-gcp-test:
130
  image: maven:3.3.9-jdk-8
131
132
  extends:
    - .osdu-gcp-variables
Aliaksandr Ramanovich (EPAM)'s avatar
Aliaksandr Ramanovich (EPAM) committed
133
    - .maven
134
  stage: integration
135
  needs: [ "osdu-gcp-deploy" ]
136
  allow_failure: true
137
138
139
140
141
142
143
144
  variables:
    WORKFLOW_HOST: $OSDU_GCP_WORKFLOW_SERVICE_URL
    INGEST_HOST: $OSDU_GCP_INGEST_HOST
    TEST_FILE_ID: $OSDU_GCP_TEST_FILE_ID
    FILE_SERVICE_HOST: $OSDU_GCP_FILE_URL
    DELIVERY_HOST: $OSDU_GCP_DELIVERY_SERVICE_URL
    SEARCH_HOST: $OSDU_GCP_SEARCH_HOST
    STORAGE_HOST: $OSDU_GCP_STORAGE_URL
Aliaksandr Ramanovich (EPAM)'s avatar
Aliaksandr Ramanovich (EPAM) committed
145
146
147
148
149
150
151
152
    STORAGE_URL: $OSDU_GCP_STORAGE_URL
    DEPLOY_ENV: empty
    LEGAL_URL: $OSDU_GCP_LEGAL_HOST_URL
    PUBSUB_TOKEN: $GCP_PUBSUB_TOKEN
    INTEGRATION_TESTER: $OSDU_GCP_INTEGRATION_TESTER
    NO_DATA_ACCESS_TESTER: $OSDU_GCP_NO_DATA_ACCESS_TESTER
    SEARCH_INTEGRATION_TESTER: $OSDU_GCP_INTEGRATION_TESTER
    HOST_URL: $OSDU_GCP_LEGAL_HOST_URL
153
  script:
154
    - export GCP_DEPLOY_FILE=$(echo $OSDU_GCP_DEPLOY_FILE)
155
156
157
158
159
160
161
162
    - >
        mvn
        --batch-mode
        --settings=$CI_PROJECT_DIR/.mvn/community-maven.settings.xml
        -f testing/pom.xml
        verify
        --projects $OSDU_GCP_SERVICE-test-core,$OSDU_GCP_SERVICE-test-gcp
        2>&1 | tee test-results.log
163
164
  only:
    variables:
165
      - $OSDU_GCP == 'true' && $OSDU_GCP_INT_TEST_TYPE != 'python'
166
167
168
169
170
  artifacts:
    when: always
    paths:
      - test-results.log
    expire_in: 1 week
171
172
173
174
175
176
177
178
179
180
181

osdu-gcp-test-python:
  extends:
    - .osdu-gcp-variables
  variables:
    INTEGRATION_TESTER: $OSDU_GCP_INTEGRATION_TESTER
    BASE_URL: /api/unit
    VIRTUAL_SERVICE_HOST_NAME: os-unit-attcrcktoa-uc.a.run.app
    MY_TENANT: osdu
  stage: integration
  image: python:3.7
182
  needs: [ "osdu-gcp-deploy" ]
183
184
185
186
187
188
189
190
  allow_failure: false
  script:
    - cd testing/unit_test_gcp
    - chmod +x ./run-integration-tests.sh
    - ./run-integration-tests.sh
  only:
    variables:
      - $OSDU_GCP == 'true' && $OSDU_GCP_INT_TEST_TYPE == 'python'