GONRG-3779: Common pipeline for dag

.gitignore

 .vscode/*
 .idea/*
 
+env/**
+venv/**
+
 energistics/src/witsml_parser/osdu/*
 !energistics/src/witsml_parser/osdu/.gitkeep

.gitlab-ci.yml

-#  Copyright 2020 Google LLC
-#  Copyright 2020 EPAM Systems
+#  Copyright 2021 Google LLC
+#  Copyright 2021 EPAM Systems
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
@@ -21,38 +21,22 @@ variables:
   OSDU_GCP_BASE_IMAGE_TAG: $CI_REGISTRY_IMAGE/osdu-gcp:$CI_COMMIT_SHORT_SHA
 
   WORKDIR: energistics
-  SRCDIR:  energistics/src/
+  SRCDIR:  energistics/src
   TESTDIR: energistics/tests
   VENDOR_NAME: energistics
 
+include:
+
+  - project: "osdu/platform/ci-cd-pipelines"
+    ref: "master"
+    file: "cloud-providers/osdu-gcp-dag.yaml"
+
 
 stages:
   - build_and_test
   - deploy
   - end_to_end_postman_tests
 
-.gcp_rsync_deploy:
-  image: gcr.io/google.com/cloudsdktool/cloud-sdk
-  stage: deploy
-  script:
-    - echo $GCP_KEY_FILE > key.json
-    - gcloud auth activate-service-account --key-file key.json
-    - gcloud config set project "$GCP_PROJECT"
-    - sed -i "s|IMAGE|$OSDU_GCP_BASE_IMAGE_TAG|" devops/osdu-gcp/airflow_configs.yaml
-    - pushd "$SRCDIR"
-    - ls
-    - echo *******
-    - gsutil -m rsync -d -R dags/"$VENDOR_NAME"/ "$GCP_DEPLOY_TARGET"/dags/"$VENDOR_NAME"
-    - popd
-    - gsutil cp devops/osdu-gcp/airflow_configs.yaml "$GCP_DEPLOY_TARGET"/dags/configs/
-
-.gcp_e2e_test_dag:
-  image: gcr.io/google.com/cloudsdktool/cloud-sdk
-  stage: end_to_end_postman_tests
-  script:
-    - chmod +x tests/e2e_postman_tests/gcp/run_e2e_tests.sh
-    - tests/e2e_postman_tests/gcp/./run_e2e_tests.sh $GCP_TEST_ENV
-
 build_and_test:
   stage: build_and_test
   tags: ["osdu-medium"]
@@ -65,42 +49,42 @@ build_and_test:
     - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
     - docker push $CI_REGISTRY_IMAGE/osdu-gcp
 
-osdu_gcp_dags_rsync_comm:
-  extends: .gcp_rsync_deploy
+osdu_gcp_bootstrap_dag_community:
+  stage: deploy
+  variables: 
+    OSDU_GCP_SKIP_BOOTSTRAP: "true"
+
+osdu_gcp_bootstrap_dag_pre_ship:
+  stage: deploy
+  variables: 
+    OSDU_GCP_SKIP_BOOTSTRAP: "true"
+
+osdu_gcp_deploy_composer_community:
+  stage: deploy
+  after_script:
+    - sed -i "s|IMAGE|$OSDU_GCP_BASE_IMAGE_TAG|" devops/osdu-gcp/airflow_configs.yaml
+    - gsutil cp devops/osdu-gcp/airflow_configs.yaml ${GCP_DEPLOY_TARGET}/dags/configs/airflow_configs.yaml || exit $?
   variables:
-    GCP_DEPLOY_TARGET: $OSDU_GCP_DEPL_TARGET_V2
-    GCP_KEY_FILE: $OSDU_GCP_DEPLOY_FILE
-    GCP_KEY_PATH: $OSDU_GCP_KEY_PATH
-    GCP_PROJECT: $OSDU_GCP_PROJECT
-  only:
-    - $CI_DEFAULT_BRANCH
-    - /^trusted-/
+    DAG_FOLDER: $SRCDIR/dags/$VENDOR_NAME
+    OSDU_GCP_DAG_FOLDER: energistics
 
-osdu_gcp_dags_rsync_prod:
-  extends: .gcp_rsync_deploy
+osdu_gcp_deploy_composer_pre_ship:
+  stage: deploy
+  after_script:
+    - sed -i "s|IMAGE|$OSDU_GCP_BASE_IMAGE_TAG|" devops/osdu-gcp/airflow_configs.yaml
+    - gsutil cp devops/osdu-gcp/airflow_configs.yaml ${GCP_DEPLOY_TARGET}/dags/configs/airflow_configs.yaml || exit $?
   variables:
-    GCP_DEPL_TARGET: $OSDU_GCP_DEPL_TARGET_PREPROD
-    GCP_KEY_FILE: $OSDU_GCP_PROD_SVC_ACCT
-    GCP_KEY_PATH: $OSDU_GCP_PROD_KEY_PATH
-    GCP_PROJECT: $OSDU_GCP_PROD_PROJECT
-  only:
-    - tags
-    - /^release\/*/
+    DAG_FOLDER: $SRCDIR/dags/$VENDOR_NAME
+    OSDU_GCP_DAG_FOLDER: energistics
 
-gcp_e2e_test_dag_comm:
-  extends: .gcp_e2e_test_dag
-  needs: [osdu_gcp_dags_rsync_comm]
+osdu_gcp_postman_tests_community:
+  stage: end_to_end_postman_tests
   variables:
-    GCP_TEST_ENV: "COMMUNITY"
-  only:
-    - $CI_DEFAULT_BRANCH
-    - /^trusted-/
+    COLLECTION_PATH: "Dev/41_CICD_WITSML/WITSML CI-CD v1.0.postman_collection.json"
+    TEST_SOURCE_FILES_DIR: "Dev/41_CICD_WITSML"
 
-gcp_e2e_test_dag_prod:
-  extends: .gcp_e2e_test_dag
-  needs: [osdu_gcp_dags_rsync_prod]
+osdu_gcp_postman_tests_pre_ship:
+  stage: end_to_end_postman_tests
   variables:
-    GCP_TEST_ENV: "PREPROD"
-  only:
-    - tags
-    - /^release\/*/
+    COLLECTION_PATH: "Dev/41_CICD_WITSML/WITSML CI-CD v1.0.postman_collection.json"
+    TEST_SOURCE_FILES_DIR: "Dev/41_CICD_WITSML"

tests/e2e_postman_tests/gcp/run_e2e_tests.sh

-# /usr/bin/sh
-
-apt-get update
-apt-get install jq -y
-apt-get install nodejs -y > /dev/null 2>&1
-node -v
-apt install npm -y > /dev/null 2>&1
-PATH="$PATH"
-npm install -g n
-n latest
-PATH="$PATH"
-npm install -g newman -y
-
-# clone collections
-mkdir platform_testing
-git clone --depth 1 -b master https://community.opengroup.org/osdu/platform/testing.git platform_testing
-COLLECTION="platform_testing/Dev/41_CICD_WITSML/WITSML CI-CD v1.0.postman_collection.json"
-COLLECTION_FOLDER="platform_testing/Dev/41_CICD_WITSML"
-
-case $1 in
-    "PREPROD") 
-        ENVIRONMENT="platform_testing/Postman Collection/00_CICD_Setup_Environment/gcp.OSDU R3 PROD v2.4.postman_environment.json"
-        ;;
-    "COMMUNITY")
-        ENVIRONMENT="tests/e2e_postman_tests/gcp/environments/OSDU_Community.postman_environment.json"
-        ;;
-    *)
-        echo "Env option must be 'PREPROD' or 'COMMUNITY'"
-        exit 1
-        ;;
-esac
-
-# Get access token for GCP SA.
-echo $OSDU_GCP_INTEGRATION_TESTER | base64 -d > file.json
-gcloud auth activate-service-account --key-file file.json
-gcloud config set project $OSDU_GCP_PROJECT
-export ACCESS_TOKEN=`gcloud auth print-access-token`
-
-# As we use acces_token of Integration Tester SA File, we must skip Refresh Token step
-cat "$COLLECTION" | jq 'del(.item[] | select (.name=="Configure collection"))' > collection.json
-cat collection.json > "$COLLECTION"
-
-
-# Wait for DAGs initialization
-sleep 120
-
-newman run "$COLLECTION" --working-dir "$COLLECTION_FOLDER" --env-var access_token="$ACCESS_TOKEN" -g "$ENVIRONMENT" || EXIT_CODE=$?
-
-
-exit ${EXIT_CODE}