Skip to content
GitLab
Commit 49fe3b3b authored by Oleksandr Kosse (EPAM)'s avatar Oleksandr Kosse (EPAM)
Browse files

Merge branch 'OSDU-GCP-CRun-to-r4' into 'release/0.4' 

Osdu gcp CRun to r4

See merge request osdu/platform/ci-cd-pipelines!136
parents 7b871b4f 65f5bd53
Hide whitespace changes
Inline Side-by-side
cloud-providers/osdu-gcp-cloudrun.yml
View file @ 49fe3b3b
......@@ -24,27 +24,41 @@
.osdu-gcp-variables:
variables:
#services' urls
# placeholder(stub) for register service
# services' urls
OSDU_GCP_STORAGE_HOSTNAME: os-storage-attcrcktoa-uc.a.run.app
OSDU_GCP_LEGALTAG_API: https://os-legal-attcrcktoa-uc.a.run.app/api/legal/v1
OSDU_GCP_CRS_API: https://crs-converter-attcrcktoa-uc.a.run.app/api/crs/v2
OSDU_GCP_REGISTER_URL: https://os-register-attcrcktoa-uc.a.run.app/
OSDU_GCP_ENTITLEMENTS_URL: https://os-entitlements-gcp-attcrcktoa-uc.a.run.app/entitlements/v1/
OSDU_GCP_AUTHORIZE_API: https://os-entitlements-gcp-attcrcktoa-uc.a.run.app/entitlements/v1/
OSDU_GCP_STORAGE_URL: https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/
OSDU_GCP_RECORDS_ROOT_URL: https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/
OSDU_GCP_LEGAL_HOST_URL: https://os-legal-dot-nice-etching-277309.uc.r.appspot.com/api/legal/v1/
OSDU_GCP_STORAGE_URL: https://os-storage-attcrcktoa-uc.a.run.app/api/storage/v2/
OSDU_GCP_RECORDS_ROOT_URL: https://os-storage-attcrcktoa-uc.a.run.app/api/storage/v2/
OSDU_GCP_LEGAL_HOST_URL: https://os-legal-attcrcktoa-uc.a.run.app/api/legal/v1/
OSDU_GCP_LEGAL_HOSTNAME: https://os-legal-attcrcktoa-uc.a.run.app/
OSDU_GCP_AIRFLOW_URL: https://c410714dc037b0e02p-tp.appspot.com/
OSDU_GCP_DELIVERY_SERVICE_URL: https://os-delivery-attcrcktoa-uc.a.run.app/api/delivery/v2/
OSDU_GCP_INGEST_HOST: https://os-ingest-attcrcktoa-uc.a.run.app/
OSDU_GCP_WORKFLOW_SERVICE_URL: https://os-workflow-attcrcktoa-uc.a.run.app/
OSDU_GCP_SEARCH_HOST: https://os-search-dot-nice-etching-277309.uc.r.appspot.com/api/search/v2/
OSDU_GCP_SEARCH_QUERY_URL: https://os-search-dot-nice-etching-277309.uc.r.appspot.com//api/search/v2/query/
OSDU_GCP_INDEXER_HOST: https://os-indexer-dot-nice-etching-277309.uc.r.appspot.com/
OSDU_GCP_SEARCH_HOST: https://os-search-attcrcktoa-uc.a.run.app/api/search/v2/
OSDU_GCP_SEARCH_QUERY_URL: https://os-search-attcrcktoa-uc.a.run.app/api/search/v2/query/
OSDU_GCP_INDEXER_HOST: https://os-indexer-attcrcktoa-uc.a.run.app/
OSDU_GCP_INDEXER_HOST_SEARCH: https://os-indexer-attcrcktoa-uc.a.run.app/api/indexer/v2/
OSDU_GCP_FILE_URL: https://os-file-attcrcktoa-uc.a.run.app/
OSDU_GCP_NOTIFICATION_URL: https://os-notification-attcrcktoa-uc.a.run.app/
OSDU_GCP_STORAGE_QUERY_RECORD_HOST: https://os-storage-attcrcktoa-uc.a.run.app/api/storage/v2/query/records
OSDU_GCP_STORAGE_SCHEMA_HOST: https://os-storage-attcrcktoa-uc.a.run.app/api/storage/v2/schemas
OSDU_GCP_STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST: https://os-storage-attcrcktoa-uc.a.run.app/api/storage/v2/query/records:batch
OSDU_GCP_INDEXER_QUEUE_HOST: https://indexer-queue-attcrcktoa-uc.a.run.app/api/indexer/v1/_dps/task-handlers/enqueue
OSDU_GCP_UNIT_HOSTNAME: os-unit-attcrcktoa-uc.a.run.app
OSDU_GCP_CRS_CATALOG_HOSTNAME: os-crs-catalog-attcrcktoa-uc.a.run.app
REDIS_GROUP_HOST: 10.234.198.27
REDIS_SEARCH_HOST: 10.99.138.107
REDIS_STORAGE_HOST: 10.16.59.203
REDIS_SYNC_HOST: 10.116.62.35
OSDU_GCP_REDIS_SEARCH_PORT: 6379
OSDU_GCP_SPRING_PROFILES_ACTIVE: dev
# variables for unit deployment
OSDU_GCP_UNIT_CATALOG_BUCKET: nice-etching-277309-unit-catalog-bucket
# common variables
OSDU_GCP_CLOUDRUN_REGION: us-central1
OSDU_GCP_PROJECT: nice-etching-277309
......@@ -57,6 +71,8 @@
OTHER_RELEVANT_DATA_COUNTRIES: US
DEFAULT_DATA_PARTITION_ID_TENANT1: osdu
GCLOUD_PROJECT: nice-etching-277309
# variables for storage tests
TENANT_NAME: osdu
# variables for workflow tests
FINISHED_WORKFLOW_ID: fad778da-fbc4-4261-8b3e-deb48be44969
# variables for ingestion tests
......@@ -70,7 +86,7 @@
# variables for delivery tests
ENTITLEMENTS_DOMAIN: osdu-gcp.go3-nrg.projects.epam.com
TENANT: osdu
DEFAULT_DATA_PARTITION_ID_TENANT2: osdu
#DEFAULT_DATA_PARTITION_ID_TENANT2: osdu
INTEGRATION_TEST_AUDIENCE: 689762842995-pv217jo3k8j803kk6gqf52qb5amos3a9.apps.googleusercontent.com
# variables for register tests, and deployment
SERVICE_IDENTITY: osdu-gcp-sa
......@@ -78,22 +94,37 @@
CRON_JOB_EXPECTED_IP: 0:0:0:0:0:0:0:1
SUBSCRIBER_PRIVATE_KEY_ID: $OSDU_GCP_SUBSCRIBER_PRIVATE_KEY_ID
SUBSCRIBER_SECRET: $OSDU_GCP_SUBSCRIBER_SECRET
DE_OPS_TESTER: $OSDU_GCP_R3_OPS_TESTER
DE_ADMIN_TESTER: $OSDU_GCP_R3_ADMIN_TESTER
DE_EDITOR_TESTER: $OSDU_GCP_R3_EDITOR_TESTER
DE_NO_ACCESS_TESTER: $OSDU_GCP_R3_NO_ACCESS_TESTER
REGISTER_BASE_URL: https://os-register-attcrcktoa-uc.a.run.app/
CLIENT_TENANT: nonexistenttenant
OSDU_TENANT: osdu
SUBSCRIPTION_ID: $OSDU_GCP_REGISTER_SUBSCRIPTION_ID
REGISTER_CUSTOM_PUSH_URL: https://os-register-attcrcktoa-uc.a.run.app/api/register/v1/test/challenge/1
CLIENT_TENANT: nonexistenttenant
OSDU_TENANT: osdu
REGISTER_BASE_URL: https://os-register-attcrcktoa-uc.a.run.app/
# variables for notification tests
HMAC_SECRET: $OSDU_GCP_SUBSCRIBER_SECRET
REGISTER_CUSTOM_PUSH_URL_HMAC: https://os-register-attcrcktoa-uc.a.run.app/api/register/v1/test/challenge/hmac-integration-test
ENVIRONMENT: DEV_GKE
TOPIC_ID: records-changed
NOTIFICATION_BASE_URL: $OSDU_GCP_NOTIFICATION_URL
# variables for legal tests
MY_TENANT_PROJECT: osdu
MY_TENANT: osdu
ENABLE_FULL_BUCKET_NAME: "true"
SKIP_HTTP_TESTS: "true"
# variables for search tests
DATA_GROUP: osdu
ENTITLEMENTS_HOST: $OSDU_GCP_ENTITLEMENTS_URL
ELASTIC_PASSWORD: $OSDU_GCP_ELASTIC_PASSWORD
ELASTIC_USER_NAME: $OSDU_GCP_ELASTIC_USER
ELASTIC_HOST: $OSDU_GCP_ELASTIC_HOST
ELASTIC_PORT: $OSDU_GCP_ELASTIC_PORT
DATA_PARTITION_ID_TENANT1: osdu
osdu-gcp-containerize:
stage: containerize
needs: [ "compile-and-unit-test" ]
needs: ["compile-and-unit-test"]
extends: .osdu-gcp-variables
image: google/cloud-sdk
cache: { }
cache: {}
script:
- gcloud auth activate-service-account --key-file $OSDU_GCP_DEPLOY_FILE
- gcloud config set project $OSDU_GCP_PROJECT
......@@ -105,14 +136,24 @@ osdu-gcp-containerize:
osdu-gcp-deploy:
image: google/cloud-sdk
needs: [ "osdu-gcp-containerize" ]
needs: ["osdu-gcp-containerize"]
stage: deploy
extends: .osdu-gcp-variables
cache: { }
cache: {}
script:
- gcloud auth activate-service-account --key-file $OSDU_GCP_DEPLOY_FILE
- gcloud config set project $OSDU_GCP_PROJECT
- gcloud run deploy $OSDU_GCP_APPLICATION_NAME --image gcr.io/$OSDU_GCP_PROJECT/$OSDU_GCP_APPLICATION_NAME/$OSDU_GCP_SERVICE-$OSDU_GCP_VENDOR:$CI_COMMIT_SHORT_SHA --platform managed --region $OSDU_GCP_CLOUDRUN_REGION --allow-unauthenticated --service-account $OSDU_GCP_SERVICE_ACCOUNT --memory 512M --set-env-vars=$OSDU_GCP_ENV_VARS
- >
gcloud beta run deploy $OSDU_GCP_APPLICATION_NAME
--image gcr.io/$OSDU_GCP_PROJECT/$OSDU_GCP_APPLICATION_NAME/$OSDU_GCP_SERVICE-$OSDU_GCP_VENDOR:$CI_COMMIT_SHORT_SHA
--platform managed
--region $OSDU_GCP_CLOUDRUN_REGION
--allow-unauthenticated
--service-account $OSDU_GCP_SERVICE_ACCOUNT
--memory 512M
$OSDU_GCP_CLOUD_RUN_PARAMETERS
--set-env-vars=$OSDU_GCP_ENV_VARS
- gcloud run services update-traffic $OSDU_GCP_APPLICATION_NAME --platform managed --region $OSDU_GCP_CLOUDRUN_REGION --to-latest
only:
variables:
- $OSDU_GCP == 'true'
......@@ -121,8 +162,9 @@ osdu-gcp-test:
image: maven:3.3.9-jdk-8
extends:
- .osdu-gcp-variables
- .maven
stage: integration
needs: [ "osdu-gcp-deploy" ]
needs: ["osdu-gcp-deploy"]
allow_failure: true
variables:
WORKFLOW_HOST: $OSDU_GCP_WORKFLOW_SERVICE_URL
......@@ -132,23 +174,30 @@ osdu-gcp-test:
DELIVERY_HOST: $OSDU_GCP_DELIVERY_SERVICE_URL
SEARCH_HOST: $OSDU_GCP_SEARCH_HOST
STORAGE_HOST: $OSDU_GCP_STORAGE_URL
DE_OPS_TESTER: $OSDU_GCP_R3_OPS_TESTER
DE_ADMIN_TESTER: $OSDU_GCP_R3_ADMIN_TESTER
DE_EDITOR_TESTER: $OSDU_GCP_R3_EDITOR_TESTER
DE_NO_ACCESS_TESTER: $OSDU_GCP_R3_NO_ACCESS_TESTER
STORAGE_URL: $OSDU_GCP_STORAGE_URL
DEPLOY_ENV: empty
LEGAL_URL: $OSDU_GCP_LEGAL_HOST_URL
PUBSUB_TOKEN: $GCP_PUBSUB_TOKEN
INTEGRATION_TESTER: $OSDU_GCP_INTEGRATION_TESTER
NO_DATA_ACCESS_TESTER: $OSDU_GCP_NO_DATA_ACCESS_TESTER
SEARCH_INTEGRATION_TESTER: $OSDU_GCP_INTEGRATION_TESTER
HOST_URL: $OSDU_GCP_LEGAL_HOST_URL
INDEXER_HOST: $OSDU_GCP_INDEXER_HOST_SEARCH
OSDU_GCP_TESTING_DIR: testing
script:
- echo $TEST_FILE_ID
- echo $INGEST_HOST
- echo $DELIVERY_HOST
- export INTEGRATION_TESTER=$(echo $OSDU_GCP_R3_INTEGRATION_TESTER)
- export NO_DATA_ACCESS_TESTER=$(echo $OSDU_GCP_R3_NO_DATA_ACCESS_TESTER)
- export GCP_DEPLOY_FILE=$(echo $OSDU_GCP_DEPLOY_FILE)
- export SEARCH_INTEGRATION_TESTER=$(echo $OSDU_GCP_R3_INTEGRATION_TESTER)
- >
mvn
--batch-mode
--settings=$CI_PROJECT_DIR/.mvn/community-maven.settings.xml
-f testing/pom.xml
verify
--projects $OSDU_GCP_SERVICE-test-core,$OSDU_GCP_SERVICE-test-gcp
2>&1 | tee test-results.log
mvn
--batch-mode
--settings=$CI_PROJECT_DIR/.mvn/community-maven.settings.xml
-f $OSDU_GCP_TESTING_DIR/pom.xml
verify
--projects $OSDU_GCP_SERVICE-test-core,$OSDU_GCP_SERVICE-test-gcp
2>&1 | tee test-results.log
only:
variables:
- $OSDU_GCP == 'true' && $OSDU_GCP_INT_TEST_TYPE != 'python'
......@@ -163,17 +212,28 @@ osdu-gcp-test-python:
- .osdu-gcp-variables
variables:
INTEGRATION_TESTER: $OSDU_GCP_INTEGRATION_TESTER
BASE_URL: /api/unit
VIRTUAL_SERVICE_HOST_NAME: os-unit-attcrcktoa-uc.a.run.app
MY_TENANT: osdu
stage: integration
image: python:3.7
needs: [ "osdu-gcp-deploy" ]
needs: ["osdu-gcp-deploy"]
allow_failure: false
script:
- cd testing/unit_test_gcp
- cd $OSDU_GCP_PYTHON_INT_TEST_SUBDIR
- chmod +x ./run-integration-tests.sh
- ./run-integration-tests.sh
only:
variables:
- $OSDU_GCP == 'true' && $OSDU_GCP_INT_TEST_TYPE == 'python'
tag_gcp_image:
image: google/cloud-sdk
needs:
- job: osdu-gcp-deploy
stage: Integration
cache: {}
script:
- gcloud auth activate-service-account --key-file $OSDU_GCP_DEPLOY_FILE
- gcloud config set project $OSDU_GCP_PROJECT
- gcloud container images add-tag gcr.io/$OSDU_GCP_PROJECT/$OSDU_GCP_APPLICATION_NAME/$OSDU_GCP_SERVICE-$OSDU_GCP_VENDOR:$CI_COMMIT_SHORT_SHA gcr.io/$OSDU_GCP_PROJECT/$OSDU_GCP_APPLICATION_NAME/$OSDU_GCP_SERVICE-$OSDU_GCP_VENDOR:$CI_COMMIT_TAG
only:
- tags
cloud-providers/osdu-gcp.yml deleted 100644 → 0
View file @ 7b871b4f
.OSDU_GCP_variables:
variables:
# service's URLs
OSDU_GCP_ENTITLEMENTS_URL: https://os-entitlements-gcp-attcrcktoa-uc.a.run.app/entitlements/v1/
OSDU_GCP_STORAGE_URL: https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/
OSDU_GCP_RECORDS_ROOT_URL: https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/
OSDU_GCP_LEGAL_HOST_URL: https://os-legal-dot-nice-etching-277309.uc.r.appspot.com/api/legal/v1/
OSDU_GCP_AIRFLOW_URL: https://c410714dc037b0e02p-tp.appspot.com/
OSDU_GCP_DELIVERY_SERVICE_URL: https://os-delivery-attcrcktoa-uc.a.run.app/api/delivery/v2/
OSDU_GCP_INGEST_HOST: https://os-ingest-attcrcktoa-uc.a.run.app/
OSDU_GCP_WORKFLOW_SERVICE_URL: https://os-workflow-attcrcktoa-uc.a.run.app/
OSDU_GCP_SEARCH_HOST: https://os-search-dot-nice-etching-277309.uc.r.appspot.com/api/search/v2/
OSDU_GCP_SEARCH_QUERY_URL: https://os-search-dot-nice-etching-277309.uc.r.appspot.com//api/search/v2/query/
OSDU_GCP_INDEXER_HOST: https://os-indexer-dot-nice-etching-277309.uc.r.appspot.com/
OSDU_GCP_FILE_URL: https://os-file-attcrcktoa-uc.a.run.app/
REDIS_GROUP_HOST: 10.234.198.27
REDIS_SEARCH_HOST: 10.99.138.107
REDIS_STORAGE_HOST: 10.16.59.203
REDIS_SYNC_HOST: 10.116.62.35
# variables for integration tests
DEPLOY_ENV: empty
DOMAIN: osdu-gcp.go3-nrg.projects.epam.com
ENTITLEMENTS_DOMAIN: osdu-gcp.go3-nrg.projects.epam.com
TENANT_NAME: osdu
MY_TENANT_PROJECT: osdu
MY_TENANT: osdu
DATA_GROUP: osdu
DEFAULT_DATA_PARTITION_ID_TENANT1: osdu
DEFAULT_DATA_PARTITION_ID_TENANT2: osdudeploy
INTEGRATION_TEST_AUDIENCE: 689762842995-pv217jo3k8j803kk6gqf52qb5amos3a9.apps.googleusercontent.com
INT_TEST_VENDOR: gcp
PROJECT: nice-etching-277309
GCLOUD_PROJECT: nice-etching-277309
LEGAL_TAG: osdu-demo-legaltag
OTHER_RELEVANT_DATA_COUNTRIES: US
STORAGE_HOST: https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/
.osdu-gcp:
tags: ["osdu-medium"]
image: google/cloud-sdk
only:
variables:
- $OSDU_GCP_BUILD_SUBDIR && $OSDU_GCP_INT_TEST_SUBDIR && $OSDU_GCP_APPLICATION_NAME && $OSDU_GCP_ENVIRONMENT && $OSDU_GCP_PROJECT && $OSDU_GCP_DEPLOY_FILE
variables:
OSDU_GCP_DEPLOY_DIR: OSDU_GCP_deploy_dir
before_script:
- mkdir -p "$OSDU_GCP_DEPLOY_DIR"
- gcloud auth activate-service-account --key-file="$OSDU_GCP_DEPLOY_FILE"
osdu-gcp-deploy:
extends: .osdu-gcp
stage: deploy
needs: ["compile-and-unit-test"]
script:
- cat $OSDU_GCP_APP_YAML > $OSDU_GCP_DEPLOY_DIR/app.yaml
- cp -v "${OSDU_GCP_BUILD_SUBDIR}"/target/*-spring-boot.jar "$OSDU_GCP_DEPLOY_DIR"
- cd "$OSDU_GCP_DEPLOY_DIR"
- echo --version="$CI_COMMIT_SHORT_SHA" --project="$OSDU_GCP_PROJECT" app.yaml
- gcloud auth list
- gcloud app deploy --quiet --version="$CI_COMMIT_SHORT_SHA" --project="$OSDU_GCP_PROJECT" app.yaml
only:
variables:
- $OSDU_GCP == 'true'
osdu-gcp-test:
extends:
- .osdu-gcp
- .maven
- .OSDU_GCP_variables
stage: integration
needs: ["osdu-gcp-deploy"]
variables:
# integration test variables for search service
ENTITLEMENTS_HOST: $OSDU_GCP_ENTITLEMENTS_URL
ELASTIC_PASSWORD: $OSDU_GCP_ELASTIC_PASSWORD
ELASTIC_USER_NAME: $OSDU_GCP_ELASTIC_USER
ELASTIC_HOST: $OSDU_GCP_ELASTIC_HOST
ELASTIC_PORT: $OSDU_GCP_ELASTIC_PORT
SEARCH_ON_BEHALF_INTEGRATION_TESTER: $OSDU_GCP_INTEGRATION_TESTER
INDEXER_HOST: $OSDU_GCP_INDEXER_HOST
SEARCH_INTEGRATION_TESTER: $OSDU_GCP_INTEGRATION_TESTER
SEARCH_HOST: $OSDU_GCP_SEARCH_HOST
# integration test variables for search service
PUBSUB_TOKEN: $OSDU_GCP_PUBSUB_TOKEN
STORAGE_URL: $OSDU_GCP_STORAGE_URL
AUTHORIZE_API: $OSDU_GCP_ENTITLEMENTS_URL
HOST_URL: $OSDU_GCP_LEGAL_HOST_URL
LEGAL_URL: $OSDU_GCP_LEGAL_HOST_URL
ENTITLEMENT_URL: $OSDU_GCP_ENTITLEMENT_URL
INTEGRATION_TESTER: $OSDU_GCP_INTEGRATION_TESTER
NO_DATA_ACCESS_TESTER: $OSDU_GCP_NO_DATA_ACCESS_TESTER
script:
- export GCP_DEPLOY_FILE=$(echo $OSDU_GCP_DEPLOY_FILE)
- cd $OSDU_GCP_INT_TEST_SUBDIR
# start tests
- $MAVEN test 2>&1 | tee test-results.log
only:
variables:
- $OSDU_GCP == 'true'
artifacts:
when: always
paths:
- $OSDU_GCP_INT_TEST_SUBDIR/test-results.log
expire_in: 1 week
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment