Commit 57f0b0e1 authored by Siarhei Khaletski (EPAM)'s avatar Siarhei Khaletski (EPAM) 🚩
Browse files

Merge branch 'GONRG-3453_update_cicd' into 'master'

GONRG-3453 update GCP related CI CD

See merge request !42
parents be0649aa a4ee54ed
Pipeline #79238 skipped with stages
......@@ -16,18 +16,16 @@
image: docker:19.03
variables:
OSDU_AIRFLOW_IMAGE: johnybear/osdu-airflow:python36-1
OSDU_GCP_VENDOR: gcp
OSDU_GCP_APPLICATION_NAME: energistics
OSDU_GCP_BASE_URL: https://community.osdu-gcp.go3-nrg.projects.epam.com
OSDU_GCP_MOCK_BASE_URL: localhost
WORKDIR: energistics
SRCDIR: energistics/src/
TESTDIR: energistics/tests
VENDOR_NAME: energistics
BASE_IMAGE_TAG: $CI_REGISTRY_IMAGE/witsml-parser:$CI_COMMIT_SHA
OSDU_GCP_IMAGE_TAG: $CI_REGISTRY_IMAGE/osdu-gcp-witsml-parser:$CI_COMMIT_SHA
BASE_IMAGE: $CI_REGISTRY_IMAGE/witsml-parser
OSDU_GCP_IMAGE: $CI_REGISTRY_IMAGE/osdu-gcp-witsml-parser
BASE_IMAGE_TAG: $BASE_IMAGE:$CI_COMMIT_SHORT_SHA
stages:
- test
......@@ -36,14 +34,21 @@ stages:
- deploy
- end_to_end_postman_tests
.add_system_deps: &add_system_deps
- echo ---- ---- ---- SYSTEM DEPENDENCIES ---- ---- ----
- apk update
- apk add git
.run_tests: &run_tests
- echo ---- ---- ---- TESTS ---- ---- ----
- docker build -t witsml:test --rm -f ./build/test/Dockerfile --build-arg base_image=$BASE_IMAGE_TAG .
- docker run witsml:test || exit $?
build_base_image:
stage: containerize
tags: ["osdu-medium"]
script:
- echo ---- ---- ---- SYSTEM DEPENDENCIES ---- ---- ----
- apk update
- apk add git
- *add_system_deps
- echo ---- ---- ---- CLONE REPOSITORIES ---- ---- ----
- >
cd energistics/src/witsml_parser/osdu
......@@ -52,123 +57,122 @@ build_base_image:
&& cd - && ls
- echo ---- ---- ---- BUILD IMAGE ---- ---- ----
- docker build -t $BASE_IMAGE_TAG --rm -f ./build/Dockerfile .
- docker build -t witsml:test --rm -f ./build/test/Dockerfile --build-arg base_image=$BASE_IMAGE_TAG .
- echo ---- ---- ---- TESTS ---- ---- ----
- docker run witsml:test || exit $?
- *run_tests
- echo ---- ---- ---- PUSH IMAGE ---- ---- ----
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- docker push $BASE_IMAGE_TAG
when: manual
.gcp_update_ini_file: &gcp_update_ini_file
- echo ---- ---- ---- UPDATE ENV VARS ---- ---- ----
- sed -i "s|%(DATASET_BASE_URL)s|$OSDU_GCP_BASE_URL|" build/providers/gcp/osdu_api.ini
- sed -i "s|%(STORAGE_BASE_URL)s|$OSDU_GCP_BASE_URL|" build/providers/gcp/osdu_api.ini
- sed -i "s|%([_A-Z]*_BASE_URL)s|$OSDU_GCP_MOCK_BASE_URL|" build/providers/gcp/osdu_api.ini
- sed -i "s|%([_A-Z]*_BASE_URL)s|$OSDU_GCP_BASE_URL|" build/providers/gcp/osdu_api.ini
build_osdu_gcp_image:
.gcp_update_airflow_config: &gcp_update_airflow_config
- echo ---- ---- ---- UPDATE AIRFLOW CONFIG ---- ---- ----
- sed -i "s|IMAGE|$OSDU_GCP_IMAGE_TAG|" devops/osdu-gcp/airflow_configs.yaml
.build_osdu_gcp:
stage: csp_containerize
tags: ["osdu-medium"]
needs: ["build_base_image"]
script:
- echo ---- ---- ---- SYSTEM DEPENDENCIES ---- ---- ----
- apk update
- apk add git
- *add_system_deps
- *gcp_update_ini_file
- echo ---- ---- ---- BUILD IMAGE ---- ---- ----
- docker build -t $OSDU_GCP_IMAGE_TAG --rm -f ./build/providers/$OSDU_GCP_VENDOR/Dockerfile.energistics --build-arg PROVIDER_NAME=$OSDU_GCP_VENDOR --build-arg base_image=$BASE_IMAGE_TAG .
- echo ---- ---- ---- PUSH IMAGE ---- ---- ----
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- echo ---- ---- PUSH IMAGE
- docker push $OSDU_GCP_IMAGE_TAG
- sed -i "s|IMAGE|$OSDU_GCP_IMAGE_TAG|" devops/osdu-gcp/airflow_configs.yaml
when: manual
artifacts:
when: always
paths:
- devops/osdu-gcp/airflow_configs.yaml
expire_in: 1 week
build_osdu_gcp_image:
extends: .build_osdu_gcp
variables:
OSDU_GCP_BASE_URL: https://community.osdu-gcp.go3-nrg.projects.epam.com
OSDU_GCP_IMAGE_TAG: $OSDU_GCP_IMAGE:latest
after_script:
- *gcp_update_airflow_config
except:
- /^release\/*/
build_osdu_gcp_release_image:
stage: csp_containerize
tags: ["osdu-medium"]
needs: ["build_base_image"]
extends: .build_osdu_gcp
variables:
OSDU_GCP_IMAGE_TAG: $CI_REGISTRY_IMAGE/osdu-gcp-witsml-parser
script:
- echo ---- ---- ---- SYSTEM DEPENDENCIES ---- ---- ----
- apk update
- apk add git
- *gcp_update_ini_file
- echo ---- ---- ---- BUILD IMAGE ---- ---- ----
- docker build -t $OSDU_GCP_IMAGE_TAG:release-latest --rm -f ./build/providers/$OSDU_GCP_VENDOR/Dockerfile.energistics --build-arg PROVIDER_NAME=$OSDU_GCP_VENDOR --build-arg base_image=$BASE_IMAGE_TAG .
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- echo ---- ---- PUSH IMAGE
- docker push $OSDU_GCP_IMAGE_TAG:release-latest
- if [ -n "$CI_COMMIT_TAG" ]; then docker tag $OSDU_GCP_IMAGE_TAG:release-latest $OSDU_GCP_IMAGE_TAG:$CI_COMMIT_TAG && docker push $OSDU_GCP_IMAGE_TAG:$CI_COMMIT_TAG; fi
when: manual
OSDU_GCP_BASE_URL: https://preship-asm.osdu-gcp.go3-nrg.projects.epam.com
OSDU_GCP_IMAGE_TAG: $OSDU_GCP_IMAGE:release-latest
after_script:
- if [ -n "$CI_COMMIT_TAG" ]; then docker tag $OSDU_GCP_IMAGE_TAG $OSDU_GCP_IMAGE:$CI_COMMIT_TAG && docker push $OSDU_GCP_IMAGE:$CI_COMMIT_TAG && OSDU_GCP_IMAGE_TAG=$OSDU_GCP_IMAGE:$CI_COMMIT_TAG; fi
- *gcp_update_airflow_config
only:
- tags
- /^release\/*/
osdu_gcp_dags_rsync:
.gcp_rsync:
image: gcr.io/google.com/cloudsdktool/cloud-sdk
stage: deploy
needs: [build_osdu_gcp_image]
script:
- gcloud auth activate-service-account --key-file $OSDU_GCP_DEPLOY_FILE
- gcloud config set project "$OSDU_GCP_PROJECT"
- sed -i "s|KEY_PATH|$OSDU_GCP_KEY_PATH|" devops/osdu-gcp/airflow_configs.yaml
- gcloud auth activate-service-account --key-file $GCP_KEY_FILE
- gcloud config set project "$GCP_PROJECT"
- sed -i "s|KEY_PATH|$GCP_KEY_PATH|" devops/osdu-gcp/airflow_configs.yaml
- pushd "$SRCDIR"
- ls
- echo *******
- gsutil -m rsync -d -R dags/"$VENDOR_NAME"/ "$OSDU_GCP_DEPL_TARGET_V2"/dags/"$VENDOR_NAME"
- gsutil -m rsync -d -R dags/"$VENDOR_NAME"/ "$GCP_DEPLOY_TARGET"/dags/"$VENDOR_NAME"
- popd
- gsutil cp devops/osdu-gcp/airflow_configs.yaml "$OSDU_GCP_DEPL_TARGET_V2"/dags/configs/
- gsutil cp devops/osdu-gcp/airflow_configs.yaml "$GCP_DEPLOY_TARGET"/dags/configs/
osdu_gcp_dags_rsync_comm:
extends: .gcp_rsync
needs: [build_osdu_gcp_image]
variables:
GCP_DEPLOY_TARGET: $OSDU_GCP_DEPL_TARGET_V2
GCP_KEY_FILE: $OSDU_GCP_DEPLOY_FILE
GCP_KEY_PATH: $OSDU_GCP_KEY_PATH
GCP_PROJECT: $OSDU_GCP_PROJECT
only:
- $CI_DEFAULT_BRANCH
- /^trusted-/
when: manual
osdu_gcp_dags_prod_rsync:
image: gcr.io/google.com/cloudsdktool/cloud-sdk
stage: deploy
osdu_gcp_dags_rsync_prod:
extends: .gcp_rsync
needs: [build_osdu_gcp_release_image]
script:
- gcloud auth activate-service-account --key-file $OSDU_GCP_PROD_SVC_ACCT
- gcloud config set project "$OSDU_GCP_PROD_PROJECT"
- sed -i "s|KEY_PATH|$OSDU_GCP_PROD_KEY_PATH|" devops/osdu-gcp/airflow_configs.yaml
- pushd "$SRCDIR"
- ls
- echo *******
- gsutil -m rsync -d -R dags/"$VENDOR_NAME"/ "$OSDU_GCP_DEPL_TARGET_PREPROD"/dags/"$VENDOR_NAME"
- popd
- gsutil cp devops/osdu-gcp/airflow_configs.yaml "$OSDU_GCP_DEPL_TARGET_PREPROD"/dags/configs/
variables:
GCP_DEPL_TARGET: $OSDU_GCP_DEPL_TARGET_PREPROD
GCP_KEY_FILE: $OSDU_GCP_PROD_SVC_ACCT
GCP_KEY_PATH: $OSDU_GCP_PROD_KEY_PATH
GCP_PROJECT: $OSDU_GCP_PROD_PROJECT
only:
- tags
- /^release\/*/
when: manual
gcp_e2e_test_dag:
.gcp_e2e_test_dag:
image: gcr.io/google.com/cloudsdktool/cloud-sdk
stage: end_to_end_postman_tests
needs: [osdu_gcp_dags_rsync]
script:
- chmod +x tests/e2e_postman_tests/gcp/run_e2e_tests.sh
- tests/e2e_postman_tests/gcp/./run_e2e_tests.sh "COMMUNITY"
- tests/e2e_postman_tests/gcp/./run_e2e_tests.sh $GCP_TEST_ENV
gcp_e2e_test_dag_comm:
extends: .gcp_e2e_test_dag
needs: [osdu_gcp_dags_rsync_comm]
variables:
GCP_TEST_ENV: "COMMUNITY"
only:
- $CI_DEFAULT_BRANCH
- /^trusted-/
gcp_e2e_test_dag_pre_prod:
image: gcr.io/google.com/cloudsdktool/cloud-sdk
stage: end_to_end_postman_tests
needs: [osdu_gcp_dags_prod_rsync]
script:
- chmod +x tests/e2e_postman_tests/gcp/run_e2e_tests.sh
- tests/e2e_postman_tests/gcp/./run_e2e_tests.sh "PREPROD"
gcp_e2e_test_dag_prod:
extends: .gcp_e2e_test_dag
needs: [osdu_gcp_dags_rsync_prod]
variables:
GCP_TEST_ENV: "PREPROD"
only:
- tags
- /^release\/*/
......@@ -24,7 +24,7 @@ dataset_url=%(DATASET_BASE_URL)s/api/dataset/v1
entitlements_url=%(ENTITLEMENTS_BASE_URL)s/api/entitlements/v1
schema_url=%(SCHEMA_BASE_URL)s/api/schema-service/v1
ingestion_workflow_url=%(INGESTION_WORKFLOW_BASE_URL)s/api/workflow/v1
partition_url=$(PARTITION_BASE_URL)s/api/partition/v1
partition_url=%(PARTITION_BASE_URL)s/api/partition/v1
use_service_principal=False
[provider]
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment