Commit 160ea44d authored by Yan Sushchynski (EPAM)'s avatar Yan Sushchynski (EPAM)
Browse files

GONRG-3779: Common pipeline for dag

parent f4f2c225
Pipeline #80913 passed with stages
in 1 minute and 53 seconds
.vscode/*
.idea/*
env/**
venv/**
energistics/src/witsml_parser/osdu/*
!energistics/src/witsml_parser/osdu/.gitkeep
# Copyright 2020 Google LLC
# Copyright 2020 EPAM Systems
# Copyright 2021 Google LLC
# Copyright 2021 EPAM Systems
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
......@@ -21,38 +21,22 @@ variables:
OSDU_GCP_BASE_IMAGE_TAG: $CI_REGISTRY_IMAGE/osdu-gcp:$CI_COMMIT_SHORT_SHA
WORKDIR: energistics
SRCDIR: energistics/src/
SRCDIR: energistics/src
TESTDIR: energistics/tests
VENDOR_NAME: energistics
include:
- project: "osdu/platform/ci-cd-pipelines"
ref: "GONRG-3779_Common_pipeline_for_dag"
file: "cloud-providers/osdu-gcp-dag.yaml"
stages:
- build_and_test
- deploy
- end_to_end_postman_tests
.gcp_rsync_deploy:
image: gcr.io/google.com/cloudsdktool/cloud-sdk
stage: deploy
script:
- echo $GCP_KEY_FILE > key.json
- gcloud auth activate-service-account --key-file key.json
- gcloud config set project "$GCP_PROJECT"
- sed -i "s|IMAGE|$OSDU_GCP_BASE_IMAGE_TAG|" devops/osdu-gcp/airflow_configs.yaml
- pushd "$SRCDIR"
- ls
- echo *******
- gsutil -m rsync -d -R dags/"$VENDOR_NAME"/ "$GCP_DEPLOY_TARGET"/dags/"$VENDOR_NAME"
- popd
- gsutil cp devops/osdu-gcp/airflow_configs.yaml "$GCP_DEPLOY_TARGET"/dags/configs/
.gcp_e2e_test_dag:
image: gcr.io/google.com/cloudsdktool/cloud-sdk
stage: end_to_end_postman_tests
script:
- chmod +x tests/e2e_postman_tests/gcp/run_e2e_tests.sh
- tests/e2e_postman_tests/gcp/./run_e2e_tests.sh $GCP_TEST_ENV
build_and_test:
stage: build_and_test
tags: ["osdu-medium"]
......@@ -65,42 +49,48 @@ build_and_test:
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- docker push $CI_REGISTRY_IMAGE/osdu-gcp
osdu_gcp_dags_rsync_comm:
extends: .gcp_rsync_deploy
osdu_gcp_bootstrap_dag_community:
stage: deploy
needs: ["build_and_test"]
variables:
OSDU_GCP_SKIP_BOOTSTRAP: "true"
osdu_gcp_bootstrap_dag_pre_ship:
stage: deploy
needs: ["build_and_test"]
variables:
OSDU_GCP_SKIP_BOOTSTRAP: "true"
osdu_gcp_deploy_composer_community:
stage: deploy
after_script:
- sed -i "s|IMAGE|$OSDU_GCP_BASE_IMAGE_TAG|" devops/osdu-gcp/airflow_configs.yaml
- gsutil cp devops/osdu-gcp/airflow_configs.yaml "$GCP_DEPLOY_TARGET"/dags/configs/
variables:
DAG_FOLDER: $SRCDIR/dags/$VENDOR_NAME
GCP_DEPLOY_TARGET: $OSDU_GCP_DEPL_TARGET_V2
GCP_KEY_FILE: $OSDU_GCP_DEPLOY_FILE
GCP_KEY_PATH: $OSDU_GCP_KEY_PATH
GCP_PROJECT: $OSDU_GCP_PROJECT
only:
- $CI_DEFAULT_BRANCH
- /^trusted-/
osdu_gcp_dags_rsync_prod:
extends: .gcp_rsync_deploy
osdu_gcp_deploy_composer_pre_ship:
stage: deploy
after_script:
- sed -i "s|IMAGE|$OSDU_GCP_BASE_IMAGE_TAG|" devops/osdu-gcp/airflow_configs.yaml
- gsutil cp devops/osdu-gcp/airflow_configs.yaml "$GCP_DEPLOY_TARGET"/dags/configs/
variables:
DAG_FOLDER: $SRCDIR/dags/$VENDOR_NAME
GCP_DEPL_TARGET: $OSDU_GCP_DEPL_TARGET_PREPROD
GCP_KEY_FILE: $OSDU_GCP_PROD_SVC_ACCT
GCP_KEY_PATH: $OSDU_GCP_PROD_KEY_PATH
GCP_PROJECT: $OSDU_GCP_PROD_PROJECT
only:
- tags
- /^release\/*/
gcp_e2e_test_dag_comm:
extends: .gcp_e2e_test_dag
needs: [osdu_gcp_dags_rsync_comm]
osdu_gcp_postman_tests_community:
stage: end_to_end_postman_tests
variables:
GCP_TEST_ENV: "COMMUNITY"
only:
- $CI_DEFAULT_BRANCH
- /^trusted-/
COLLECTION_PATH: "Dev/41_CICD_WITSML/WITSML CI-CD v1.0.postman_collection.json"
TEST_SOURCE_FILES_DIR: "Dev/41_CICD_WITSML"
gcp_e2e_test_dag_prod:
extends: .gcp_e2e_test_dag
needs: [osdu_gcp_dags_rsync_prod]
osdu_gcp_postman_tests_pre_ship:
stage: end_to_end_postman_tests
variables:
GCP_TEST_ENV: "PREPROD"
only:
- tags
- /^release\/*/
COLLECTION_PATH: "Dev/41_CICD_WITSML/WITSML CI-CD v1.0.postman_collection.json"
TEST_SOURCE_FILES_DIR: "Dev/41_CICD_WITSML"
# /usr/bin/sh
apt-get update
apt-get install jq -y
apt-get install nodejs -y > /dev/null 2>&1
node -v
apt install npm -y > /dev/null 2>&1
PATH="$PATH"
npm install -g n
n latest
PATH="$PATH"
npm install -g newman -y
# clone collections
mkdir platform_testing
git clone --depth 1 -b master https://community.opengroup.org/osdu/platform/testing.git platform_testing
COLLECTION="platform_testing/Dev/41_CICD_WITSML/WITSML CI-CD v1.0.postman_collection.json"
COLLECTION_FOLDER="platform_testing/Dev/41_CICD_WITSML"
case $1 in
"PREPROD")
ENVIRONMENT="platform_testing/Postman Collection/00_CICD_Setup_Environment/gcp.OSDU R3 PROD v2.4.postman_environment.json"
;;
"COMMUNITY")
ENVIRONMENT="tests/e2e_postman_tests/gcp/environments/OSDU_Community.postman_environment.json"
;;
*)
echo "Env option must be 'PREPROD' or 'COMMUNITY'"
exit 1
;;
esac
# Get access token for GCP SA.
echo $OSDU_GCP_INTEGRATION_TESTER | base64 -d > file.json
gcloud auth activate-service-account --key-file file.json
gcloud config set project $OSDU_GCP_PROJECT
export ACCESS_TOKEN=`gcloud auth print-access-token`
# As we use acces_token of Integration Tester SA File, we must skip Refresh Token step
cat "$COLLECTION" | jq 'del(.item[] | select (.name=="Configure collection"))' > collection.json
cat collection.json > "$COLLECTION"
# Wait for DAGs initialization
sleep 120
newman run "$COLLECTION" --working-dir "$COLLECTION_FOLDER" --env-var access_token="$ACCESS_TOKEN" -g "$ENVIRONMENT" || EXIT_CODE=$?
exit ${EXIT_CODE}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment