GONRG-3453 update gitlab ci

.gitlab-ci.yml

@@ -16,18 +16,16 @@
 image: docker:19.03
 
 variables:
-  OSDU_AIRFLOW_IMAGE: johnybear/osdu-airflow:python36-1
   OSDU_GCP_VENDOR: gcp
   OSDU_GCP_APPLICATION_NAME: energistics
-  OSDU_GCP_BASE_URL: https://community.osdu-gcp.go3-nrg.projects.epam.com
-  OSDU_GCP_MOCK_BASE_URL: localhost
 
   WORKDIR: energistics
   SRCDIR:  energistics/src/
   TESTDIR: energistics/tests
   VENDOR_NAME: energistics
-  BASE_IMAGE_TAG: $CI_REGISTRY_IMAGE/witsml-parser:$CI_COMMIT_SHA
-  OSDU_GCP_IMAGE_TAG: $CI_REGISTRY_IMAGE/osdu-gcp-witsml-parser:$CI_COMMIT_SHA
+  BASE_IMAGE: $CI_REGISTRY_IMAGE/witsml-parser
+  OSDU_GCP_IMAGE: $CI_REGISTRY_IMAGE/osdu-gcp-witsml-parser
+  BASE_IMAGE_TAG: $BASE_IMAGE:$CI_COMMIT_SHORT_SHA
 
 stages:
   - test
@@ -36,14 +34,21 @@ stages:
   - deploy
   - end_to_end_postman_tests
 
+.add_system_deps: &add_system_deps
+  - echo ---- ---- ---- SYSTEM DEPENDENCIES ---- ---- ----
+  - apk update
+  - apk add git
+
+.run_tests: &run_tests
+  - echo ---- ---- ---- TESTS ---- ---- ----
+  - docker build -t witsml:test --rm -f ./build/test/Dockerfile --build-arg base_image=$BASE_IMAGE_TAG .
+  - docker run  witsml:test || exit $?
 
 build_base_image:
   stage: containerize
   tags: ["osdu-medium"]
   script:
-    - echo ---- ---- ---- SYSTEM DEPENDENCIES ---- ---- ----
-    - apk update
-    - apk add git
+    - *add_system_deps
     - echo ---- ---- ---- CLONE REPOSITORIES ---- ---- ----
     - >
       cd energistics/src/witsml_parser/osdu 
@@ -52,123 +57,122 @@ build_base_image:
       && cd - && ls
     - echo ---- ---- ---- BUILD IMAGE ---- ---- ----
     - docker build -t $BASE_IMAGE_TAG --rm -f ./build/Dockerfile .
-    - docker build -t witsml:test --rm -f ./build/test/Dockerfile --build-arg base_image=$BASE_IMAGE_TAG .
-    - echo ---- ---- ---- TESTS ---- ---- ----
-    - docker run  witsml:test || exit $?
+    - *run_tests
     - echo ---- ---- ---- PUSH IMAGE ---- ---- ----
     - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
     - docker push $BASE_IMAGE_TAG
   when: manual
-  
 
 .gcp_update_ini_file: &gcp_update_ini_file
   - echo ---- ---- ---- UPDATE ENV VARS ---- ---- ----
-  - sed -i "s|%(DATASET_BASE_URL)s|$OSDU_GCP_BASE_URL|" build/providers/gcp/osdu_api.ini
-  - sed -i "s|%(STORAGE_BASE_URL)s|$OSDU_GCP_BASE_URL|" build/providers/gcp/osdu_api.ini
-  - sed -i "s|%([_A-Z]*_BASE_URL)s|$OSDU_GCP_MOCK_BASE_URL|" build/providers/gcp/osdu_api.ini
+  - sed -i "s|%([_A-Z]*_BASE_URL)s|$OSDU_GCP_BASE_URL|" build/providers/gcp/osdu_api.ini
 
-build_osdu_gcp_image:
+.gcp_update_airflow_config: &gcp_update_airflow_config
+  - echo ---- ---- ---- UPDATE AIRFLOW CONFIG ---- ---- ----
+  - sed -i "s|IMAGE|$OSDU_GCP_IMAGE_TAG|" devops/osdu-gcp/airflow_configs.yaml
+
+.build_osdu_gcp:
   stage: csp_containerize
   tags: ["osdu-medium"]
   needs: ["build_base_image"]
   script:
-    - echo ---- ---- ---- SYSTEM DEPENDENCIES ---- ---- ----
-    - apk update
-    - apk add git
+    - *add_system_deps
     - *gcp_update_ini_file
     - echo ---- ---- ---- BUILD IMAGE ---- ---- ----
     - docker build -t $OSDU_GCP_IMAGE_TAG --rm -f ./build/providers/$OSDU_GCP_VENDOR/Dockerfile.energistics --build-arg PROVIDER_NAME=$OSDU_GCP_VENDOR --build-arg base_image=$BASE_IMAGE_TAG .
+    - echo ---- ---- ---- PUSH IMAGE ---- ---- ----
     - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
-    - echo ---- ---- PUSH IMAGE
     - docker push $OSDU_GCP_IMAGE_TAG
-    - sed -i "s|IMAGE|$OSDU_GCP_IMAGE_TAG|" devops/osdu-gcp/airflow_configs.yaml
   when: manual
   artifacts:
     when: always
     paths:
       - devops/osdu-gcp/airflow_configs.yaml
     expire_in: 1 week
+
+build_osdu_gcp_image:
+  extends: .build_osdu_gcp
+  variables:
+    OSDU_GCP_BASE_URL: https://community.osdu-gcp.go3-nrg.projects.epam.com
+    OSDU_GCP_IMAGE_TAG: $OSDU_GCP_IMAGE:latest
+  after_script:
+    - *gcp_update_airflow_config
   except:
     - /^release\/*/
 
 build_osdu_gcp_release_image:
-  stage: csp_containerize
-  tags: ["osdu-medium"]
-  needs: ["build_base_image"]
+  extends: .build_osdu_gcp
   variables:
-    OSDU_GCP_IMAGE_TAG: $CI_REGISTRY_IMAGE/osdu-gcp-witsml-parser
-  script:
-    - echo ---- ---- ---- SYSTEM DEPENDENCIES ---- ---- ----
-    - apk update
-    - apk add git
-    - *gcp_update_ini_file
-    - echo ---- ---- ---- BUILD IMAGE ---- ---- ----
-    - docker build -t $OSDU_GCP_IMAGE_TAG:release-latest --rm -f ./build/providers/$OSDU_GCP_VENDOR/Dockerfile.energistics --build-arg PROVIDER_NAME=$OSDU_GCP_VENDOR --build-arg base_image=$BASE_IMAGE_TAG .
-    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
-    - echo ---- ---- PUSH IMAGE
-    - docker push $OSDU_GCP_IMAGE_TAG:release-latest
-    - if [ -n "$CI_COMMIT_TAG" ]; then docker tag $OSDU_GCP_IMAGE_TAG:release-latest $OSDU_GCP_IMAGE_TAG:$CI_COMMIT_TAG && docker push $OSDU_GCP_IMAGE_TAG:$CI_COMMIT_TAG; fi
-  when: manual
+    OSDU_GCP_BASE_URL: https://preship-asm.osdu-gcp.go3-nrg.projects.epam.com
+    OSDU_GCP_IMAGE_TAG: $OSDU_GCP_IMAGE:release-latest
+  after_script:
+    - if [ -n "$CI_COMMIT_TAG" ]; then docker tag $OSDU_GCP_IMAGE_TAG $OSDU_GCP_IMAGE:$CI_COMMIT_TAG && docker push $OSDU_GCP_IMAGE:$CI_COMMIT_TAG && OSDU_GCP_IMAGE_TAG=$OSDU_GCP_IMAGE:$CI_COMMIT_TAG; fi
+    - *gcp_update_airflow_config
   only: 
     - tags
     - /^release\/*/
 
-osdu_gcp_dags_rsync:
+.gcp_rsync:
   image: gcr.io/google.com/cloudsdktool/cloud-sdk
   stage: deploy
-  needs: [build_osdu_gcp_image]
   script:
-    - gcloud auth activate-service-account --key-file $OSDU_GCP_DEPLOY_FILE
-    - gcloud config set project "$OSDU_GCP_PROJECT"
-    - sed -i "s|KEY_PATH|$OSDU_GCP_KEY_PATH|" devops/osdu-gcp/airflow_configs.yaml
+    - gcloud auth activate-service-account --key-file $GCP_KEY_FILE
+    - gcloud config set project "$GCP_PROJECT"
+    - sed -i "s|KEY_PATH|$GCP_KEY_PATH|" devops/osdu-gcp/airflow_configs.yaml
     - pushd "$SRCDIR"
     - ls
     - echo *******
-    - gsutil -m rsync -d -R dags/"$VENDOR_NAME"/ "$OSDU_GCP_DEPL_TARGET_V2"/dags/"$VENDOR_NAME"
+    - gsutil -m rsync -d -R dags/"$VENDOR_NAME"/ "$GCP_DEPLOY_TARGET"/dags/"$VENDOR_NAME"
     - popd
-    - gsutil cp devops/osdu-gcp/airflow_configs.yaml "$OSDU_GCP_DEPL_TARGET_V2"/dags/configs/
+    - gsutil cp devops/osdu-gcp/airflow_configs.yaml "$GCP_DEPLOY_TARGET"/dags/configs/
+
+osdu_gcp_dags_rsync_comm:
+  extends: .gcp_rsync
+  needs: [build_osdu_gcp_image]
+  variables:
+    GCP_DEPLOY_TARGET: $OSDU_GCP_DEPL_TARGET_V2
+    GCP_KEY_FILE: $OSDU_GCP_DEPLOY_FILE
+    GCP_KEY_PATH: $OSDU_GCP_KEY_PATH
+    GCP_PROJECT: $OSDU_GCP_PROJECT
   only:
     - $CI_DEFAULT_BRANCH
     - /^trusted-/
   when: manual
 
-osdu_gcp_dags_prod_rsync:
-  image: gcr.io/google.com/cloudsdktool/cloud-sdk
-  stage: deploy
+osdu_gcp_dags_rsync_prod:
+  extends: .gcp_rsync
   needs: [build_osdu_gcp_release_image]
-  script:
-    - gcloud auth activate-service-account --key-file $OSDU_GCP_PROD_SVC_ACCT
-    - gcloud config set project "$OSDU_GCP_PROD_PROJECT"
-    - sed -i "s|KEY_PATH|$OSDU_GCP_PROD_KEY_PATH|" devops/osdu-gcp/airflow_configs.yaml
-    - pushd "$SRCDIR"
-    - ls
-    - echo *******
-    - gsutil -m rsync -d -R dags/"$VENDOR_NAME"/ "$OSDU_GCP_DEPL_TARGET_PREPROD"/dags/"$VENDOR_NAME"
-    - popd
-    - gsutil cp devops/osdu-gcp/airflow_configs.yaml "$OSDU_GCP_DEPL_TARGET_PREPROD"/dags/configs/
+  variables:
+    GCP_DEPL_TARGET: $OSDU_GCP_DEPL_TARGET_PREPROD
+    GCP_KEY_FILE: $OSDU_GCP_PROD_SVC_ACCT
+    GCP_KEY_PATH: $OSDU_GCP_PROD_KEY_PATH
+    GCP_PROJECT: $OSDU_GCP_PROD_PROJECT
   only:
     - tags
     - /^release\/*/
   when: manual
 
-gcp_e2e_test_dag:
+.gcp_e2e_test_dag:
   image: gcr.io/google.com/cloudsdktool/cloud-sdk
   stage: end_to_end_postman_tests
-  needs: [osdu_gcp_dags_rsync]
   script:
     - chmod +x tests/e2e_postman_tests/gcp/run_e2e_tests.sh
-    - tests/e2e_postman_tests/gcp/./run_e2e_tests.sh "COMMUNITY"
+    - tests/e2e_postman_tests/gcp/./run_e2e_tests.sh $GCP_TEST_ENV
+
+gcp_e2e_test_dag_comm:
+  extends: .gcp_e2e_test_dag
+  needs: [osdu_gcp_dags_rsync_comm]
+  variables:
+    GCP_TEST_ENV: "COMMUNITY"
   only:
     - $CI_DEFAULT_BRANCH
     - /^trusted-/
 
-gcp_e2e_test_dag_pre_prod:
-  image: gcr.io/google.com/cloudsdktool/cloud-sdk
-  stage: end_to_end_postman_tests
-  needs: [osdu_gcp_dags_prod_rsync]
-  script:
-    - chmod +x tests/e2e_postman_tests/gcp/run_e2e_tests.sh
-    - tests/e2e_postman_tests/gcp/./run_e2e_tests.sh "PREPROD" 
+gcp_e2e_test_dag_prod:
+  extends: .gcp_e2e_test_dag
+  needs: [osdu_gcp_dags_rsync_prod]
+  variables:
+    GCP_TEST_ENV: "PREPROD"
   only:
     - tags
     - /^release\/*/

build/providers/gcp/osdu_api.ini

@@ -24,7 +24,7 @@ dataset_url=%(DATASET_BASE_URL)s/api/dataset/v1
 entitlements_url=%(ENTITLEMENTS_BASE_URL)s/api/entitlements/v1
 schema_url=%(SCHEMA_BASE_URL)s/api/schema-service/v1
 ingestion_workflow_url=%(INGESTION_WORKFLOW_BASE_URL)s/api/workflow/v1
-partition_url=$(PARTITION_BASE_URL)s/api/partition/v1
+partition_url=%(PARTITION_BASE_URL)s/api/partition/v1
 use_service_principal=False
 
 [provider]