Merge branch 'gc-cicd-ing-ref-dag' into 'master'

GONRG-7623: pipeline for osdu-ingest-r3-by-reference dag

See merge request !156

devops/gc/pipeline/override-stages.yml

@@ -9,10 +9,125 @@ variables:
   TEST_SOURCE_FILES_DIR: "Postman Collection/29_CICD_Setup_Ingestion"
   COLLECTION_PATH: "Postman%20Collection/29_CICD_Setup_Ingestion/R3%20Full%20manifest-based%20ingestion.postman_collection.json"
 
-# Community, Dev2, Preship
+# Community, Dev2, Preship -- osdu-ingest-r3.py
 gc-render-dag:
   needs: ["test_dags"]
 
-# Baremetal
+# Baremetal -- osdu-ingest-r3.py
 gc-render-dag:baremetal:
   needs: ["test_dags"]
+
+# osdu-ingest-r3-by-reference.py
+.gc_common_render_dag_ref:
+  tags: ["osdu-small"]
+  stage: bootstrap_dag
+  image: python:3.8.12-slim
+  variables:
+    GC_DAG_FILE: osdu-ingest-r3-by-reference.py
+  script:
+    - mkdir -p $GC_RENDERED_DAG_FOLDER
+    - pip install osdu-dag-versioning==1.2.0 --index-url https://community.opengroup.org/api/v4/projects/1341/packages/pypi/simple
+    - add-dag-version -f $GC_DAG_DIRECTORY/$GC_DAG_FILE -o $OUTPUT_FILE
+  artifacts:
+    paths:
+      - $OUTPUT_FILE
+  rules:
+    - if: "$GC_SKIP_RENDER == 'true'"
+      when: never
+    - if: '$GC == "1"'
+      when: on_success
+
+.gc_common_publish_dag_ref:
+  tags: ["osdu-small"]
+  stage: publish_dag
+  image: registry.gitlab.com/gitlab-ci-utils/curl-jq:latest
+  variables:
+    GC_DAG_FILE: osdu-ingest-r3-by-reference.py
+  script:
+    - export EXTRA_PACKAGE_VERSION="latest"
+    - >
+      if [[ "$CI_COMMIT_TAG" != "" ]]; then
+        export EXTRA_PACKAGE_VERSION="$CI_COMMIT_TAG";
+      elif [[ "$CI_COMMIT_REF_NAME" = "$CI_DEFAULT_BRANCH" ]]; then
+        export EXTRA_PACKAGE_VERSION="master";
+      fi
+    - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file $OUTPUT_FILE "$CI_API_V4_URL/projects/$CI_PROJECT_ID/packages/generic/$GC_PACKAGE_CONTAINER_NAME/$EXTRA_PACKAGE_VERSION/$GC_DAG_FILE"'
+    - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file $OUTPUT_FILE "$CI_API_V4_URL/projects/$CI_PROJECT_ID/packages/generic/$GC_PACKAGE_CONTAINER_NAME/$CI_COMMIT_SHORT_SHA/$GC_DAG_FILE"'
+  rules:
+    - if: "$GC_SKIP_PUBLISH == 'true'"
+      when: never
+    - if: '$GC == "1"'
+      when: on_success
+
+.gc_common_deploy_dag_ref:
+  tags: ["osdu-small"]
+  stage: deploy
+  image: gcr.io/google.com/cloudsdktool/cloud-sdk:409.0.0-alpine # FIXME return to latest when bug is fixed
+  retry: 1
+  variables:
+    GC_SERVICE: manifest-by-reference
+  script:
+    - gcloud auth activate-service-account --key-file $GC_KEY_FILE
+    - gcloud config set project $GC_PROJECT
+    - gsutil -m rsync -R $GC_RENDERED_DAG_FOLDER $GC_COMPOSER_DEPLOY_TARGET/dags/$GC_SERVICE
+
+# Community 
+gc-render-osdu-ingest-r3-by-reference:
+  needs: ["test_dags"]
+  extends:
+    - .gc-variables
+    - .gc_common_render_dag_ref
+
+gc-publish-osdu-ingest-r3-by-reference:
+  needs: ["gc-render-osdu-ingest-r3-by-reference"]
+  extends:
+    - .gc-variables
+    - .gc_common_publish_dag_ref
+
+gc-deploy-osdu-ingest-r3-by-reference:community:
+  needs: ["gc-render-osdu-ingest-r3-by-reference"]
+  extends:
+    - .gc-variables
+    - .gc_common_deploy_dag_ref
+  rules:
+    - if: "$CI_COMMIT_BRANCH =~ /^release/"
+      when: never
+    - if: "$CI_COMMIT_TAG"
+      when: never
+    - if: '$GC == "1"'
+      when: on_success
+
+# Baremetal
+gc-render-osdu-ingest-r3-by-reference:baremetal:
+  needs: ["test_dags"]
+  extends:
+    - .gc-baremetal-variables
+    - .gc_common_render_dag_ref
+
+gc-publish-osdu-ingest-r3-by-reference:baremetal:
+  needs: ["gc-render-osdu-ingest-r3-by-reference:baremetal"]
+  extends:
+    - .gc-baremetal-variables
+    - .gc_common_publish_dag_ref
+
+gc-deploy-osdu-ingest-r3-by-reference:baremetal:
+  needs: ["gc-render-osdu-ingest-r3-by-reference:baremetal"]
+  tags: ["osdu-small"]
+  extends:
+    - .gc-baremetal-variables
+  stage: deploy
+  image: gcr.io/google.com/cloudsdktool/cloud-sdk:409.0.0-alpine # FIXME return to latest when bug is fixed
+  retry: 1
+  variables:
+    GC_SERVICE: manifest-by-reference
+  script:
+    - gcloud auth activate-service-account --key-file $GC_KEY_FILE
+    - gcloud config set project $GC_PROJECT
+    - gsutil -m rsync -R $GC_RENDERED_DAG_FOLDER $GC_AIRFLOW_DAG_BUCKET/$GC_SERVICE
+  rules:
+    - if: "$CI_COMMIT_BRANCH =~ /^release/"
+      when: never
+    - if: "$CI_COMMIT_TAG"
+      when: never
+    - if: '$GC == "1"'
+      when: on_success