Commit eb5381c8 authored by Yan Sushchynski (EPAM)'s avatar Yan Sushchynski (EPAM)
Browse files

GONRG-3779: Common pipeline for dag

parent 57f0b0e1
Pipeline #78771 passed with stages
in 33 seconds
......@@ -15,6 +15,11 @@
image: docker:19.03
include:
- project: "osdu/platform/ci-cd-pipelines"
ref: "GONRG-3779_Common_pipeline_for_dag"
file: "cloud-providers/osdu-gcp-dag.yaml"
variables:
OSDU_GCP_VENDOR: gcp
OSDU_GCP_APPLICATION_NAME: energistics
......@@ -44,6 +49,16 @@ stages:
- docker build -t witsml:test --rm -f ./build/test/Dockerfile --build-arg base_image=$BASE_IMAGE_TAG .
- docker run witsml:test || exit $?
osdu_gcp_bootstrap_dag_community:
stage: deploy
variables:
OSDU_GCP_SKIP_BOOTSTRAP: "true"
osdu_gcp_bootstrap_dag_pre_prod:
stage: deploy
variables:
OSDU_GCP_SKIP_BOOTSTRAP: "true"
build_base_image:
stage: containerize
tags: ["osdu-medium"]
......@@ -112,67 +127,55 @@ build_osdu_gcp_release_image:
- tags
- /^release\/*/
.gcp_rsync:
image: gcr.io/google.com/cloudsdktool/cloud-sdk
osdu_gcp_deploy_composer_community:
stage: deploy
script:
- gcloud auth activate-service-account --key-file $GCP_KEY_FILE
- gcloud config set project "$GCP_PROJECT"
- sed -i "s|KEY_PATH|$GCP_KEY_PATH|" devops/osdu-gcp/airflow_configs.yaml
- pushd "$SRCDIR"
- ls
- echo *******
- gsutil -m rsync -d -R dags/"$VENDOR_NAME"/ "$GCP_DEPLOY_TARGET"/dags/"$VENDOR_NAME"
- popd
- gsutil cp devops/osdu-gcp/airflow_configs.yaml "$GCP_DEPLOY_TARGET"/dags/configs/
osdu_gcp_dags_rsync_comm:
extends: .gcp_rsync
needs: [build_osdu_gcp_image]
variables:
GCP_DEPLOY_TARGET: $OSDU_GCP_DEPL_TARGET_V2
GCP_KEY_FILE: $OSDU_GCP_DEPLOY_FILE
GCP_KEY_PATH: $OSDU_GCP_KEY_PATH
GCP_PROJECT: $OSDU_GCP_PROJECT
DAG_FOLDER: "energistics/src/dags/energistics"
OSDU_GCP_DAG_FOLDER: "energistics"
OSDU_GCP_DEPLOY_TARGET: ${OSDU_GCP_DEPL_TARGET_V2}
OSDU_GCP_DEPLOY_FILE: ${GCP_KEY_FILE}
before_script:
- gcloud auth activate-service-account --key-file ${OSDU_GCP_DEPLOY_FILE}
- sed -i "s|KEY_PATH|$OSDU_GCP_KEY_PATH|" devops/osdu-gcp/airflow_configs.yaml
- gsutil cp devops/osdu-gcp/airflow_configs.yaml "$GCP_DEPLOY_TARGET"/dags/configs/
only:
- $CI_DEFAULT_BRANCH
- /^trusted-/
when: manual
osdu_gcp_dags_rsync_prod:
extends: .gcp_rsync
osdu_gcp_deploy_composer_pre_prod:
stage: deploy
needs: [build_osdu_gcp_release_image]
before_script:
- gcloud auth activate-service-account --key-file ${OSDU_GCP_DEPLOY_FILE}
- sed -i "s|KEY_PATH|$OSDU_GCP_KEY_PATH|" devops/osdu-gcp/airflow_configs.yaml
- gsutil cp devops/osdu-gcp/airflow_configs.yaml "$GCP_DEPLOY_TARGET"/dags/configs/
variables:
GCP_DEPL_TARGET: $OSDU_GCP_DEPL_TARGET_PREPROD
GCP_KEY_FILE: $OSDU_GCP_PROD_SVC_ACCT
GCP_KEY_PATH: $OSDU_GCP_PROD_KEY_PATH
GCP_PROJECT: $OSDU_GCP_PROD_PROJECT
only:
- tags
- /^release\/*/
when: manual
DAG_FOLDER: "energistics/src/dags/energistics"
OSDU_GCP_DAG_FOLDER: "energistics"
OSDU_GCP_DEPLOY_TARGET: ${OSDU_GCP_DEPL_TARGET_PREPROD}
OSDU_GCP_DEPLOY_FILE: ${OSDU_GCP_PROD_SVC_ACCT}
.gcp_e2e_test_dag:
image: gcr.io/google.com/cloudsdktool/cloud-sdk
osdu_gcp_postman_tests_community:
stage: end_to_end_postman_tests
script:
- chmod +x tests/e2e_postman_tests/gcp/run_e2e_tests.sh
- tests/e2e_postman_tests/gcp/./run_e2e_tests.sh $GCP_TEST_ENV
gcp_e2e_test_dag_comm:
extends: .gcp_e2e_test_dag
needs: [osdu_gcp_dags_rsync_comm]
before_script:
# download a file for File Service upload in the collection.
- curl 'https://community.opengroup.org/osdu/platform/testing/-/raw/master/Dev/41_CICD_WITSML/Trajectory.xml?inline=false' > Trajectory.xml
- export POSTMAN_ENV_URL=${OSDU_GCP_POSTMAN_ENV_COMMUNITY_URL}
variables:
GCP_TEST_ENV: "COMMUNITY"
only:
- $CI_DEFAULT_BRANCH
- /^trusted-/
CLIENT_ID: $OSDU_GCP_COMMUNITY_CLIENT_ID
CLIENT_SECRET: $OSDU_GCP_COMMUNITY_CLIENT_SECRET
REFRESH_TOKEN: $OSDU_GCP_COMMUNITY_REFRESH_TOKEN
COLLECTION_PATH: "Dev/41_CICD_WITSML/WITSML CI-CD v1.0.postman_collection.json"
gcp_e2e_test_dag_prod:
extends: .gcp_e2e_test_dag
needs: [osdu_gcp_dags_rsync_prod]
osdu_gcp_postman_tests_pre_prod:
stage: end_to_end_postman_tests
before_script:
# download a file for File Service upload in the collection.
- curl 'https://community.opengroup.org/osdu/platform/testing/-/raw/master/Dev/41_CICD_WITSML/Trajectory.xml?inline=false' > Trajectory.xml
- export POSTMAN_ENV_URL=${OSDU_GCP_POSTMAN_ENV_PRE_PROD_URL}
variables:
GCP_TEST_ENV: "PREPROD"
only:
- tags
- /^release\/*/
CLIENT_ID: $OSDU_GCP_PRE_PROD_CLIENT_ID
CLIENT_SECRET: $OSDU_GCP_PRE_PROD_CLIENT_SECRET
REFRESH_TOKEN: $OSDU_GCP_PRE_PROD_REFRESH_TOKEN
COLLECTION_PATH: "Dev/41_CICD_WITSML/WITSML CI-CD v1.0.postman_collection.json"
\ No newline at end of file
# /usr/bin/sh
apt-get update
apt-get install jq -y
apt-get install nodejs -y > /dev/null 2>&1
node -v
apt install npm -y > /dev/null 2>&1
PATH="$PATH"
npm install -g n
n latest
PATH="$PATH"
npm install -g newman -y
# clone collections
mkdir platform_testing
git clone --depth 1 -b master https://community.opengroup.org/osdu/platform/testing.git platform_testing
COLLECTION="platform_testing/Dev/41_CICD_WITSML/WITSML CI-CD v1.0.postman_collection.json"
COLLECTION_FOLDER="platform_testing/Dev/41_CICD_WITSML"
case $1 in
"PREPROD")
ENVIRONMENT="platform_testing/Postman Collection/00_CICD_Setup_Environment/gcp.OSDU R3 PROD v2.4.postman_environment.json"
;;
"COMMUNITY")
ENVIRONMENT="tests/e2e_postman_tests/gcp/environments/OSDU_Community.postman_environment.json"
;;
*)
echo "Env option must be 'PREPROD' or 'COMMUNITY'"
exit 1
;;
esac
# Get access token for GCP SA.
echo $OSDU_GCP_INTEGRATION_TESTER | base64 -d > file.json
gcloud auth activate-service-account --key-file file.json
gcloud config set project $OSDU_GCP_PROJECT
export ACCESS_TOKEN=`gcloud auth print-access-token`
# As we use acces_token of Integration Tester SA File, we must skip Refresh Token step
cat "$COLLECTION" | jq 'del(.item[] | select (.name=="Configure collection"))' > collection.json
cat collection.json > "$COLLECTION"
# Wait for DAGs initialization
sleep 120
newman run "$COLLECTION" --working-dir "$COLLECTION_FOLDER" --env-var access_token="$ACCESS_TOKEN" -g "$ENVIRONMENT" || EXIT_CODE=$?
exit ${EXIT_CODE}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment