GCP common pipeline for DAGs [GONRG-5296]

.gitlab-ci.yml

-#  Copyright 2021 Google LLC
-#  Copyright 2021 EPAM Systems
+#  Copyright 2022 Google LLC
+#  Copyright 2022 EPAM Systems
 #  Copyright 2020 Microsoft
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
@@ -21,41 +21,30 @@ variables:
   AZURE_CLIENT_SECRET: ${AZURE_PRINCIPAL_SECRET}
   AZURE_AD_APP_RESOURCE_ID: ${AZURE_APP_ID}
 
-  OSDU_GCP_ENABLE_BOOTSTRAP: "true"
-  OSDU_GCP_SERVICE: segy-to-vds-conversion
-  OSDU_GCP_VENDOR: gcp
-  OSDU_GCP_GCR_REGISTRY: gcr.io/osdu-anthos
-  SEGY_VDS_DOCKER_IMAGE_GITLAB: $CI_REGISTRY_IMAGE/seismic/open-vds/openvds-ingestion:latest
-  SEGY_VDS_DOCKER_IMAGE_GCR: $OSDU_GCP_GCR_REGISTRY/seismic/open-vds/openvds-ingestion:$CI_COMMIT_TAG
-  OSDU_GCP_OUTPUT_DAG_DIR: bootstrapped_dag
-  OSDU_ANTHOS_OUTPUT_DAG_DIR: bootstraped_dag_anthos
-
   # AWS env vars
   AWS_DEPLOY_TARGET: EKS
   # AWS collection vars
   AWS_COLLECTION_PATH: https://community.opengroup.org/osdu/platform/testing/-/raw/master/Dev/43_CICD_SEGY_VDS_SSDMS/OpenVDS%20SSDMS_to_SSDMS%20conversion%20CI-CD.postman_collection.json
   AWS_COLLECTION_ENV_PATH: https://community.opengroup.org/osdu/platform/testing/-/raw/master/Postman%20Collection/00_CICD_Setup_Environment/aws.OSDU%20R3%20PROD%20v2.4.postman_environment.json
-  AWS_COLLECTION_TEST_PATH: ''
-  AWS_TEST_FILE_INPUT: ''
-  AWS_TEST_FILE_KEY: ''
+  AWS_COLLECTION_TEST_PATH: ""
+  AWS_TEST_FILE_INPUT: ""
+  AWS_TEST_FILE_KEY: ""
   #Skip stages Variables
-  SKIP_BOOTSTRAP: 'true'
-  SKIP_PUBLISH: 'true'
-  SKIP_END_TO_END: 'false'
+  SKIP_BOOTSTRAP: "true"
+  SKIP_PUBLISH: "true"
+  SKIP_END_TO_END: "false"
   AWS_POSTMAN_TESTS_TO_SKIP: "3 Ingest Seismic|3.2 Check the triggered Ingestion workflow status|4.3 Check the triggered OpenVDS workflow status|5 Validate Ingestion|6 Get VDS info|7 Clean-up"
 
-
-
 image: google/cloud-sdk:alpine
 
 stages:
   - review
   - linters
+  - build
   - containerize
   - bootstrap_dag
   - publish_dag
   - deploy
-  - build
   - scan
   - bootstrap
   - end_to_end_postman_tests
@@ -70,11 +59,8 @@ include:
   - project: "osdu/platform/ci-cd-pipelines"
     file: "cloud-providers/osdu-gcp-dag.yaml"
 
-  - project: "osdu/platform/ci-cd-pipelines"
-    file: "cloud-providers/osdu-gcp-bootstrap.yml"
-
   - local: "/devops/azure/override-stages.yml"
-  - local: "/devops/gcp/override-stages.yml"
+  - local: "/devops/gcp/pipeline/override-stages.yml"
   - local: "/devops/ibm/ibm-stages.yml"
 
   - project: "osdu/platform/ci-cd-pipelines"
@@ -82,18 +68,3 @@ include:
 
   - project: "osdu/platform/ci-cd-pipelines"
     file: "cloud-providers/aws-dag.yml"
-
-
-osdu-gcp-containerize-bootstrap-gitlab:
-  needs: []
-  variables:
-    IMAGE_BOOTSTRAP_NAME: osdu-gcp-bootstrap-segy-to-vds
-    BUILD_ARGS: --build-arg SEGY_VDS_DOCKER_IMAGE_ARG=${SEGY_VDS_DOCKER_IMAGE_GITLAB}
-    BUILD_BOOTSTRAP_PATH : deployments/scripts/gcp/Dockerfile
-
-osdu-gcp-containerize-bootstrap-gcr:
-  needs: []
-  variables:
-    IMAGE_BOOTSTRAP_NAME: osdu-gcp-bootstrap-segy-to-vds
-    BUILD_ARGS: --build-arg SEGY_VDS_DOCKER_IMAGE_ARG=${SEGY_VDS_DOCKER_IMAGE_GCR}
-    BUILD_BOOTSTRAP_PATH : deployments/scripts/gcp/Dockerfile

deployments/scripts/anthos/README.md

@@ -5,8 +5,8 @@
 pip install Jinja2==2.10.1
 
 # Setup Variables
-ANTHOS_DEPLOYMENTS_SUBDIR="<path_to_folder>" # ie: path to GCP deployment scripts folder
-ANTHOS_DAG_DIRECTORY="<path_to_folder>" # ie: segy-to-vds-conversion/airflow/workflow-svc-v2
+OSDU_GCP_DEPLOYMENTS_SUBDIR="<path_to_folder>" # ie: path to GCP deployment scripts folder
+OSDU_GCP_DAG_DIRECTORY="<path_to_folder>" # ie: segy-to-vds-conversion/airflow/workflow-svc-v2
 
 OUTPUT_FILE="<path_to_output_file>" # ie. to avoid original file overwriting
 
@@ -27,4 +27,4 @@ Workflow Payload Example:
         "persistent_id": "<string>",
         "id_token": "<id_token>"
     }
-}
\ No newline at end of file
+}

deployments/scripts/anthos/bootstrap.sh

@@ -22,10 +22,10 @@ echo "------------------------------------";
 
 if [ -z $OUTPUT_FILE ]; then
   echo "THE ORIGINAL DAG FILE WILL BE OVERWRITTEN"
-  python3 $ANTHOS_DEPLOYMENTS_SUBDIR/render_dag_file.py -f $ANTHOS_DAG_DIRECTORY/segy_to_vds_ssdms_conversion_dag.py
+  python3 $OSDU_GCP_DEPLOYMENTS_SUBDIR/render_dag_file.py -f $OSDU_GCP_DAG_DIRECTORY/segy_to_vds_ssdms_conversion_dag.py
 else
   echo "RENDERED OUTPUT WILL BE SAVED INTO ${OUTPUT_FILE}"
-  python3 $ANTHOS_DEPLOYMENTS_SUBDIR/render_dag_file.py -f $ANTHOS_DAG_DIRECTORY/segy_to_vds_ssdms_conversion_dag.py -o ${OUTPUT_FILE}
+  python3 $OSDU_GCP_DEPLOYMENTS_SUBDIR/render_dag_file.py -f $OSDU_GCP_DAG_DIRECTORY/segy_to_vds_ssdms_conversion_dag.py -o ${OUTPUT_FILE}
 fi
 
 if test $? = '1'; then
@@ -41,4 +41,3 @@ else
   echo "------------------------------------";
   exit 0
 fi
-

deployments/scripts/anthos/render_dag_file.py

@@ -39,7 +39,7 @@ class DAGFileRenderer:
 
     def _render_template(self, file) -> str:
         env = Environment(
-            loader=FileSystemLoader(searchpath=os.getenv("GCP_DAG_DIRECTORY", default="src/dags")),
+            loader=FileSystemLoader(searchpath=os.getenv("OSDU_GCP_DAG_DIRECTORY", default="src/dags")),
             variable_start_string='{|', variable_end_string='|}',
         )
         template = env.get_template(os.path.basename(file.name))

deployments/scripts/gcp/Dockerfile

@@ -6,8 +6,8 @@ COPY ./src /opt
 ARG SEGY_VDS_DOCKER_IMAGE_ARG
 
 ENV SEGY_VDS_DOCKER_IMAGE=$SEGY_VDS_DOCKER_IMAGE_ARG
-ENV GCP_DEPLOYMENTS_SUBDIR=/opt
-ENV GCP_DAG_DIRECTORY=/opt/dags
+ENV OSDU_GCP_DEPLOYMENTS_SUBDIR=/opt
+ENV OSDU_GCP_DAG_DIRECTORY=/opt/dags
 ENV PREFIX=$(pwd)
 
 RUN apt-get update && apt-get install -yq jq && pip3 install -r /opt/requirements.txt && chmod 775 /opt/bootstrap_segy_to_vds.sh

deployments/scripts/gcp/README.md

@@ -10,8 +10,8 @@ export SEGY_VDS_DOCKER_IMAGE="<image_tag>"
 export K8S_NODE_POOLS=<node_pools>  # i.e. pool-1,pool-2  the value will be split by comma
 export SD_SVC_URL=<ssdms_host>  # i.e https://<ssdms_host>/seistore-svc/api/v3
 
-GCP_DEPLOYMENTS_SUBDIR="<path_to_folder>" # ie: path to GCP deployment scripts folder
-GCP_DAG_DIRECTORY="<path_to_folder>" # ie: segy-to-vds-conversion/airflow/workflow-svc-v2
+OSDU_GCP_DEPLOYMENTS_SUBDIR="<path_to_folder>" # ie: path to GCP deployment scripts folder
+OSDU_GCP_DAG_DIRECTORY="<path_to_folder>" # ie: segy-to-vds-conversion/airflow/workflow-svc-v2
 
 OUTPUT_FILE="<path_to_output_file>" # ie. to avoid original file overwriting
 
@@ -32,4 +32,4 @@ Workflow Payload Example:
         "persistent_id": "<string>",
         "id_token": "<id_token>"
     }
-}
\ No newline at end of file
+}

deployments/scripts/gcp/bootstrap.sh

@@ -22,10 +22,10 @@ echo "------------------------------------";
 
 if [ -z $OUTPUT_FILE ]; then
   echo "THE ORIGINAL DAG FILE WILL BE OVERWRITTEN"
-  python3 $GCP_DEPLOYMENTS_SUBDIR/render_dag_file.py -f $GCP_DAG_DIRECTORY/segy_to_vds_ssdms_conversion_dag.py
+  python3 $OSDU_GCP_DEPLOYMENTS_SUBDIR/render_dag_file.py -f $OSDU_GCP_DAG_DIRECTORY/segy_to_vds_ssdms_conversion_dag.py
 else
   echo "RENDERED OUTPUT WILL BE SAVED INTO ${OUTPUT_FILE}"
-  python3 $GCP_DEPLOYMENTS_SUBDIR/render_dag_file.py -f $GCP_DAG_DIRECTORY/segy_to_vds_ssdms_conversion_dag.py -o ${OUTPUT_FILE}
+  python3 $OSDU_GCP_DEPLOYMENTS_SUBDIR/render_dag_file.py -f $OSDU_GCP_DAG_DIRECTORY/segy_to_vds_ssdms_conversion_dag.py -o ${OUTPUT_FILE}
 fi
 
 if test $? = '1'; then
@@ -41,4 +41,3 @@ else
   echo "------------------------------------";
   exit 0
 fi
-

deployments/scripts/gcp/bootstrap_segy_to_vds.sh

@@ -50,10 +50,10 @@ echo "------------------------------------";
 
 if [ -z $OUTPUT_FILE ]; then
   echo "THE ORIGINAL DAG FILE WILL BE OVERWRITTEN"
-  python3 $GCP_DEPLOYMENTS_SUBDIR/render_dag_file.py -f $GCP_DAG_DIRECTORY/segy_to_vds_ssdms_conversion_dag.py
+  python3 $OSDU_GCP_DEPLOYMENTS_SUBDIR/render_dag_file.py -f $OSDU_GCP_DAG_DIRECTORY/segy_to_vds_ssdms_conversion_dag.py
 else
   echo "RENDERED OUTPUT WILL BE SAVED INTO ${OUTPUT_FILE}"
-  python3 $GCP_DEPLOYMENTS_SUBDIR/render_dag_file.py -f $GCP_DAG_DIRECTORY/segy_to_vds_ssdms_conversion_dag.py -o ${OUTPUT_FILE}
+  python3 $OSDU_GCP_DEPLOYMENTS_SUBDIR/render_dag_file.py -f $OSDU_GCP_DAG_DIRECTORY/segy_to_vds_ssdms_conversion_dag.py -o ${OUTPUT_FILE}
 fi
 
 # get airflow bucket variable

deployments/scripts/gcp/render_dag_file.py

@@ -31,7 +31,7 @@ class DAGFileRenderer:
 
     def _render_template(self, file) -> str:
         env = Environment(
-            loader=FileSystemLoader(searchpath=os.getenv("GCP_DAG_DIRECTORY", default="src/dags")),
+            loader=FileSystemLoader(searchpath=os.getenv("OSDU_GCP_DAG_DIRECTORY", default="src/dags")),
             variable_start_string='{|', variable_end_string='|}',
         )
         template = env.get_template(os.path.basename(file.name))

devops/gcp/override-stages.yml

-.osdu_gcp_common_bootstrap_variables: &osdu_gcp_common_bootstrap_variables
-  GCP_DAG_DIRECTORY: $CI_PROJECT_DIR/src/dags
-  GCP_DEPLOYMENTS_SUBDIR: deployments/scripts/gcp
-  OSDU_GCP_BOOTSTRAP_REQUIREMENTS: deployments/scripts/gcp/requirements.txt
-  OSDU_GCP_BOOTSTRAP_SCRIPT: deployments/scripts/gcp/bootstrap.sh
-
-  OUTPUT_FILE: ${OSDU_GCP_OUTPUT_DAG_DIR}/segy_to_vds_ssdms_conversion_dag.py
-
-osdu_gcp_bootstrap_dag_community:
-  stage: bootstrap_dag
-  variables:
-    <<: *osdu_gcp_common_bootstrap_variables
-
-osdu_gcp_bootstrap_dag_pre_ship:
-  stage: bootstrap_dag
-  variables:
-    <<: *osdu_gcp_common_bootstrap_variables
-
-osdu_anthos_bootstrap_dag:
-  stage: bootstrap_dag
-  variables:
-    ANTHOS_DAG_DIRECTORY: $CI_PROJECT_DIR/src/dags
-    ANTHOS_DEPLOYMENTS_SUBDIR: deployments/scripts/anthos
-    OSDU_ANTHOS_BOOTSTRAP_REQUIREMENTS: deployments/scripts/anthos/requirements.txt
-    OSDU_ANTHOS_BOOTSTRAP_SCRIPT: deployments/scripts/anthos/bootstrap.sh
-    OUTPUT_FILE: ${OSDU_ANTHOS_OUTPUT_DAG_DIR}/segy_to_vds_ssdms_conversion_dag.py
-
-osdu_gcp_publish_dag:
-  stage: publish_dag
-  needs: 
-  - job: osdu_gcp_bootstrap_dag_community
-    optional: true
-  - job: osdu_gcp_bootstrap_dag_pre_ship
-    optional: true
-  variables:
-    BOOTSTRAPPED_DAG_PATH: ${OSDU_GCP_OUTPUT_DAG_DIR}/segy_to_vds_ssdms_conversion_dag.py
-    DAG_FILE: segy_to_vds_ssdms_conversion_dag.py
-
-osdu_anthos_publish_dag:
-  stage: publish_dag
-  needs: ["osdu_anthos_bootstrap_dag"]
-  variables:
-    BOOTSTRAPPED_DAG_PATH: ${OSDU_ANTHOS_OUTPUT_DAG_DIR}/segy_to_vds_ssdms_conversion_dag.py
-    DAG_FILE: segy_to_vds_ssdms_conversion_dag.py
-
-osdu_gcp_deploy_composer_community:
-  stage: deploy
-  needs: ["osdu_gcp_bootstrap_dag_community"]
-  variables:
-    DAG_FOLDER: $OSDU_GCP_OUTPUT_DAG_DIR
-    OSDU_GCP_DAG_FOLDER: segy_to_vds
-    IMAGE_TAG: $SEGY_VDS_DOCKER_IMAGE_GITLAB
-    IMAGE_TAG_VARIABLE_NAME: gcp__image__segy_to_vds_converter  
-
-osdu_gcp_deploy_composer_pre_ship:
-  stage: deploy
-  needs: ["osdu_gcp_bootstrap_dag_pre_ship"]
-  variables:
-    DAG_FOLDER: $OSDU_GCP_OUTPUT_DAG_DIR
-    OSDU_GCP_DAG_FOLDER: segy_to_vds
-    # TODO: Create TAG image
-    IMAGE_TAG: $SEGY_VDS_DOCKER_IMAGE_GITLAB:$CI_COMMIT_TAG
-    IMAGE_TAG_VARIABLE_NAME: gcp__image__segy_to_vds_converter  
-
-osdu_gcp_deploy_composer_anthos_env:
-  stage: deploy
-  needs: ["osdu_anthos_bootstrap_dag"]
-  variables:
-    DAG_FOLDER: $OSDU_ANTHOS_OUTPUT_DAG_DIR
-    OSDU_GCP_DAG_FOLDER: segy_to_vds
-    IMAGE_TAG: $SEGY_VDS_DOCKER_IMAGE_GITLAB
-    IMAGE_TAG_VARIABLE_NAME: anthos__image__segy_to_vds_converter  
-
-osdu_gcp_postman_tests_community:
-  stage: integration
-  variables:
-    COLLECTION_PATH: "Dev/43_CICD_SEGY_VDS_SSDMS/OpenVDS SSDMS_to_SSDMS conversion CI-CD.postman_collection.json"
-
-osdu_gcp_postman_tests_pre_ship:
-  stage: integration
-  variables:
-    COLLECTION_PATH: "Dev/43_CICD_SEGY_VDS_SSDMS/OpenVDS SSDMS_to_SSDMS conversion CI-CD.postman_collection.json"
-
-osdu_anthos_postman_tests:
-  stage: integration
-  variables:
-    COLLECTION_PATH: "Dev/43_CICD_SEGY_VDS_SSDMS/OpenVDS SSDMS_to_SSDMS conversion CI-CD.postman_collection.json"

devops/gcp/pipeline/override-stages.yml

+variables:
+  OSDU_GCP_ENABLE_BOOTSTRAP: "true"
+  OSDU_GCP_SERVICE: segy-to-vds
+  OSDU_GCP_DAG_DIRECTORY: src/dags
+  OSDU_GCP_DAG_FILE: segy_to_vds_ssdms_conversion_dag.py
+  OUTPUT_FILE: $OSDU_GCP_RENDERED_DAG_FOLDER/$OSDU_GCP_DAG_FILE
+  IMAGE_TAG_VARIABLE_NAME: gcp__image__segy_to_vds_converter
+  OSDU_GCP_GCR_REGISTRY: gcr.io/osdu-anthos
+  # Postman tests
+  COLLECTION_PATH: "Dev/43_CICD_SEGY_VDS_SSDMS/OpenVDS SSDMS_to_SSDMS conversion CI-CD.postman_collection.json"
+
+# FIXME: delete after package registry usage is implemented in DF
+osdu-gcp-containerize-bootstrap-gitlab:
+  variables:
+    IMAGE_BOOTSTRAP_NAME: osdu-gcp-bootstrap-segy-to-vds
+    BUILD_ARGS: --build-arg SEGY_VDS_DOCKER_IMAGE_ARG=$OSDU_GCP_COMPOSER_TAG
+    BUILD_BOOTSTRAP_PATH: deployments/scripts/gcp/Dockerfile
+
+# FIXME: delete after package registry usage is implemented in DF
+osdu-gcp-containerize-bootstrap-gcr:
+  variables:
+    IMAGE_BOOTSTRAP_NAME: osdu-gcp-bootstrap-segy-to-vds
+    BUILD_ARGS: --build-arg SEGY_VDS_DOCKER_IMAGE_ARG=$OSDU_GCP_GCR_REGISTRY/seismic/open-vds/openvds-ingestion:$CI_COMMIT_TAG
+    BUILD_BOOTSTRAP_PATH: deployments/scripts/gcp/Dockerfile
+
+# Community, Dev2, Preship
+osdu-gcp-render-dag:
+  needs: ["osdu-gcp-containerize-bootstrap-gitlab"]
+
+# Anthos
+osdu-gcp-render-dag:anthos:
+  needs: ["osdu-gcp-containerize-bootstrap-gitlab"]
+
+osdu-gcp-deploy-dag:community:
+  variables:
+    # FIXME: an open-vds image is created in a different repo, need to implement a new logic on how to update the open-vds image in composer with sha/latest/release/tag versions
+    OSDU_GCP_COMPOSER_TAG: community.opengroup.org:5555/osdu/platform/domain-data-mgmt-services/seismic/open-vds/openvds-ingestion:latest
+
+osdu-gcp-deploy-dag:anthos:
+  variables:
+    IMAGE_TAG_VARIABLE_NAME: anthos__image__segy_to_vds_converter
+    # FIXME: an open-vds image is created in a different repo, need to implement a new logic on how to update the open-vds image in composer with sha/latest/release/tag versions
+    OSDU_GCP_COMPOSER_TAG: community.opengroup.org:5555/osdu/platform/domain-data-mgmt-services/seismic/open-vds/openvds-ingestion:latest
+
+# FIXME: an open-vds image is created in a different repo, need to implement a new logic on how to update the open-vds image in composer with sha/latest/release/tag versions
+# osdu-gcp-deploy-dag:dev2:
+#   variables:
+#     OSDU_GCP_COMPOSER_TAG: $OSDU_GCP_GCR_REGISTRY/seismic/open-vds/openvds-ingestion:$CI_COMMIT_TAG
+
+osdu-gcp-deploy-dag:preship:
+  variables:
+    # FIXME: an open-vds image is created in a different repo, need to implement a new logic on how to update the open-vds image in composer with sha/latest/release/tag versions
+    OSDU_GCP_COMPOSER_TAG: $OSDU_GCP_GCR_REGISTRY/seismic/open-vds/openvds-ingestion:$CI_COMMIT_TAG