Merge branch 'refactor-ci-cd' into 'master'

(GONRG-3892) Refactor ci cd See merge request !46

Merge branch 'refactor-ci-cd' into 'master'
(GONRG-3892) Refactor ci cd See merge request !46
f4f2c225 · Oleksandr Kosse (EPAM) · 57f0b0e1 · 38003fe6 · f4f2c225 · f4f2c225
Commit f4f2c225 authored Dec 07, 2021 by Oleksandr Kosse (EPAM)
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -18,107 +18,27 @@ image: docker:19.03
 variables:
  OSDU_GCP_VENDOR: gcp
  OSDU_GCP_APPLICATION_NAME: energistics
+  OSDU_GCP_BASE_IMAGE_TAG: $CI_REGISTRY_IMAGE/osdu-gcp:$CI_COMMIT_SHORT_SHA

  WORKDIR: energistics
  SRCDIR:  energistics/src/
  TESTDIR: energistics/tests
  VENDOR_NAME: energistics
-  BASE_IMAGE: $CI_REGISTRY_IMAGE/witsml-parser
-  OSDU_GCP_IMAGE: $CI_REGISTRY_IMAGE/osdu-gcp-witsml-parser
-  BASE_IMAGE_TAG: $BASE_IMAGE:$CI_COMMIT_SHORT_SHA
+

 stages:
-  - test
-  - containerize
-  - csp_containerize
+  - build_and_test
  - deploy
  - end_to_end_postman_tests

-.add_system_deps: &add_system_deps
-  - echo ---- ---- ---- SYSTEM DEPENDENCIES ---- ---- ----
-  - apk update
-  - apk add git
-
-.run_tests: &run_tests
-  - echo ---- ---- ---- TESTS ---- ---- ----
-  - docker build -t witsml:test --rm -f ./build/test/Dockerfile --build-arg base_image=$BASE_IMAGE_TAG .
-  - docker run  witsml:test || exit $?
-
-build_base_image:
-  stage: containerize
-  tags: ["osdu-medium"]
-  script:
-    - *add_system_deps
-    - echo ---- ---- ---- CLONE REPOSITORIES ---- ---- ----
-    - >
-      cd energistics/src/witsml_parser/osdu 
-      && git clone https://community.opengroup.org/osdu/platform/system/sdks/common-python-sdk.git
-      && git clone https://community.opengroup.org/osdu/platform/data-flow/ingestion/ingestion-dags.git
-      && cd - && ls
-    - echo ---- ---- ---- BUILD IMAGE ---- ---- ----
-    - docker build -t $BASE_IMAGE_TAG --rm -f ./build/Dockerfile .
-    - *run_tests
-    - echo ---- ---- ---- PUSH IMAGE ---- ---- ----
-    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
-    - docker push $BASE_IMAGE_TAG
-  when: manual
-
-.gcp_update_ini_file: &gcp_update_ini_file
-  - echo ---- ---- ---- UPDATE ENV VARS ---- ---- ----
-  - sed -i "s|%([_A-Z]*_BASE_URL)s|$OSDU_GCP_BASE_URL|" build/providers/gcp/osdu_api.ini
-
-.gcp_update_airflow_config: &gcp_update_airflow_config
-  - echo ---- ---- ---- UPDATE AIRFLOW CONFIG ---- ---- ----
-  - sed -i "s|IMAGE|$OSDU_GCP_IMAGE_TAG|" devops/osdu-gcp/airflow_configs.yaml
-
-.build_osdu_gcp:
-  stage: csp_containerize
-  tags: ["osdu-medium"]
-  needs: ["build_base_image"]
-  script:
-    - *add_system_deps
-    - *gcp_update_ini_file
-    - echo ---- ---- ---- BUILD IMAGE ---- ---- ----
-    - docker build -t $OSDU_GCP_IMAGE_TAG --rm -f ./build/providers/$OSDU_GCP_VENDOR/Dockerfile.energistics --build-arg PROVIDER_NAME=$OSDU_GCP_VENDOR --build-arg base_image=$BASE_IMAGE_TAG .
-    - echo ---- ---- ---- PUSH IMAGE ---- ---- ----
-    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
-    - docker push $OSDU_GCP_IMAGE_TAG
-  when: manual
-  artifacts:
-    when: always
-    paths:
-      - devops/osdu-gcp/airflow_configs.yaml
-    expire_in: 1 week
-
-build_osdu_gcp_image:
-  extends: .build_osdu_gcp
-  variables:
-    OSDU_GCP_BASE_URL: https://community.osdu-gcp.go3-nrg.projects.epam.com
-    OSDU_GCP_IMAGE_TAG: $OSDU_GCP_IMAGE:latest
-  after_script:
-    - *gcp_update_airflow_config
-  except:
-    - /^release\/*/
-
-build_osdu_gcp_release_image:
-  extends: .build_osdu_gcp
-  variables:
-    OSDU_GCP_BASE_URL: https://preship-asm.osdu-gcp.go3-nrg.projects.epam.com
-    OSDU_GCP_IMAGE_TAG: $OSDU_GCP_IMAGE:release-latest
-  after_script:
-    - if [ -n "$CI_COMMIT_TAG" ]; then docker tag $OSDU_GCP_IMAGE_TAG $OSDU_GCP_IMAGE:$CI_COMMIT_TAG && docker push $OSDU_GCP_IMAGE:$CI_COMMIT_TAG && OSDU_GCP_IMAGE_TAG=$OSDU_GCP_IMAGE:$CI_COMMIT_TAG; fi
-    - *gcp_update_airflow_config
-  only: 
-    - tags
-    - /^release\/*/
-
-.gcp_rsync:
+.gcp_rsync_deploy:
  image: gcr.io/google.com/cloudsdktool/cloud-sdk
  stage: deploy
  script:
-    - gcloud auth activate-service-account --key-file $GCP_KEY_FILE
+    - echo $GCP_KEY_FILE > key.json
+    - gcloud auth activate-service-account --key-file key.json
    - gcloud config set project "$GCP_PROJECT"
-    - sed -i "s|KEY_PATH|$GCP_KEY_PATH|" devops/osdu-gcp/airflow_configs.yaml
+    - sed -i "s|IMAGE|$OSDU_GCP_BASE_IMAGE_TAG|" devops/osdu-gcp/airflow_configs.yaml
    - pushd "$SRCDIR"
    - ls
    - echo *******
@@ -126,9 +46,27 @@ build_osdu_gcp_release_image:
    - popd
    - gsutil cp devops/osdu-gcp/airflow_configs.yaml "$GCP_DEPLOY_TARGET"/dags/configs/

+.gcp_e2e_test_dag:
+  image: gcr.io/google.com/cloudsdktool/cloud-sdk
+  stage: end_to_end_postman_tests
+  script:
+    - chmod +x tests/e2e_postman_tests/gcp/run_e2e_tests.sh
+    - tests/e2e_postman_tests/gcp/./run_e2e_tests.sh $GCP_TEST_ENV
+
+build_and_test:
+  stage: build_and_test
+  tags: ["osdu-medium"]
+  script:
+    - export EXTRA_DOCKER_TAG=""; if [ "$CI_COMMIT_TAG" != "" ] ; then EXTRA_DOCKER_TAG="-t $CI_REGISTRY_IMAGE/osdu-gcp:$CI_COMMIT_TAG" ; elif [ "$CI_COMMIT_REF_NAME" = "master" ] ; then EXTRA_DOCKER_TAG="-t $CI_REGISTRY_IMAGE/osdu-gcp:latest" ; fi
+    - docker build --target gcp -t $OSDU_GCP_BASE_IMAGE_TAG $EXTRA_DOCKER_TAG --rm -f ./build/Dockerfile .
+    - docker build --target test -t witsml:test --rm -f ./build/Dockerfile .
+    - docker run witsml:test || exit $?
+    - docker rmi -f witsml:test
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+    - docker push $CI_REGISTRY_IMAGE/osdu-gcp
+
 osdu_gcp_dags_rsync_comm:
-  extends: .gcp_rsync
-  needs: [build_osdu_gcp_image]
+  extends: .gcp_rsync_deploy
  variables:
    GCP_DEPLOY_TARGET: $OSDU_GCP_DEPL_TARGET_V2
    GCP_KEY_FILE: $OSDU_GCP_DEPLOY_FILE
@@ -137,11 +75,9 @@ osdu_gcp_dags_rsync_comm:
  only:
    - $CI_DEFAULT_BRANCH
    - /^trusted-/
-  when: manual

 osdu_gcp_dags_rsync_prod:
-  extends: .gcp_rsync
-  needs: [build_osdu_gcp_release_image]
+  extends: .gcp_rsync_deploy
  variables:
    GCP_DEPL_TARGET: $OSDU_GCP_DEPL_TARGET_PREPROD
    GCP_KEY_FILE: $OSDU_GCP_PROD_SVC_ACCT
@@ -150,14 +86,6 @@ osdu_gcp_dags_rsync_prod:
  only:
    - tags
    - /^release\/*/
-  when: manual
-
-.gcp_e2e_test_dag:
-  image: gcr.io/google.com/cloudsdktool/cloud-sdk
-  stage: end_to_end_postman_tests
-  script:
-    - chmod +x tests/e2e_postman_tests/gcp/run_e2e_tests.sh
-    - tests/e2e_postman_tests/gcp/./run_e2e_tests.sh $GCP_TEST_ENV

 gcp_e2e_test_dag_comm:
  extends: .gcp_e2e_test_dag

--- a/build/Dockerfile
+++ b/build/Dockerfile
@@ -14,7 +14,7 @@
 # limitations under the License.


-FROM python:3.6-slim
+FROM python:3.8-slim as base

 ENV HOME="/home/witsml_parser"

@@ -29,6 +29,7 @@ ENV OSDU_CORE_PROVIDERS_DIR="${WITSML_PARSER_DIR}/osdu/ingestion-dags/src/dags/p
 COPY ./${BUILD_DIR}/requirements.txt ./
 COPY ./${WITSML_PARSER_DIR}/*.py ./
 COPY ./${ENERGISTICS_LIBS} ./energistics
+COPY ./${BUILD_DIR}/providers/gcp/osdu_api.ini ./

 # TODO: Configure Package Registry for Python SDK
 RUN pip install --upgrade pip && \
@@ -36,3 +37,22 @@ RUN pip install --upgrade pip && \
    pip install -r "./requirements.txt"

 ENV PYTHONPATH="$PYTHONPATH:$HOME"
+
+## test
+FROM base as test
+
+COPY ./${BUILD_DIR}/requirements_test.txt ./
+COPY "./energistics/src/docker_test" ./test
+RUN pip install -r "requirements_test.txt"
+
+CMD pytest test
+
+## gcp
+FROM base as gcp
+
+ARG PROVIDER_NAME=gcp
+ENV CLOUD_PROVIDER $PROVIDER_NAME
+
+COPY "build/providers/${PROVIDER_NAME}/requirements.txt" ./
+RUN pip install -r "requirements.txt"
+COPY build/providers/gcp/osdu_api.ini ./
--- a/build/providers/gcp/Dockerfile.energistics
+++ b/build/providers/gcp/Dockerfile.energistics
-# Copyright 2021 Google LLC
-# Copyright 2021 EPAM
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-ARG base_image
-FROM ${base_image}
-
-ENV HOME="/home/witsml_parser"
-WORKDIR $HOME
-ARG PROVIDER_NAME
-ENV CLOUD_PROVIDER $PROVIDER_NAME
-
-COPY "build/providers/${PROVIDER_NAME}/requirements.txt" ./
-RUN pip install -r "requirements.txt"
-COPY build/providers/gcp/osdu_api.ini ./
--- a/build/providers/gcp/cloudbuild/energistics.cloudbuild.yaml
+++ b/build/providers/gcp/cloudbuild/energistics.cloudbuild.yaml
-# Copyright 2021 Google LLC
-# Copyright 2017-2019, Schlumberger
-# Copyright 2021 EPAM
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-steps:
-  - id: base
-    name: 'gcr.io/cloud-builders/docker'
-    args: [
-        'build',
-        '--build-arg', 'PROVIDER_NAME=${_PROVIDER_NAME}',
-        '-t', 'gcr.io/$PROJECT_ID/${_APPLICATION_NAME}/${_GCP_SERVICE}-${_PROVIDER_NAME}-builder:witsml',
-        '-f', 'build/Dockerfile',
-        '.'
-    ]
-    waitFor: ['-']
-
-  - name: 'gcr.io/cloud-builders/docker'
-    args: [
-        'build',
-        '--build-arg', 'PROVIDER_NAME=${_PROVIDER_NAME}',
-        '--build-arg', 'builder_image=gcr.io/$PROJECT_ID/${_APPLICATION_NAME}/${_GCP_SERVICE}-${_PROVIDER_NAME}-builder:witsml',
-        '-t', 'gcr.io/$PROJECT_ID/${_APPLICATION_NAME}/${_GCP_SERVICE}-${_PROVIDER_NAME}:${_SHORT_SHA}',
-        '-t', 'gcr.io/$PROJECT_ID/${_APPLICATION_NAME}/${_GCP_SERVICE}-${_PROVIDER_NAME}:latest',
-        '-f', 'build/providers/${_PROVIDER_NAME}/Dockerfile.energistics',
-        '.'
-    ]
-    waitFor: ["base"]
-
-images:
-  - 'gcr.io/$PROJECT_ID/${_APPLICATION_NAME}/${_GCP_SERVICE}-${_PROVIDER_NAME}-builder'
-  - 'gcr.io/$PROJECT_ID/${_APPLICATION_NAME}/${_GCP_SERVICE}-${_PROVIDER_NAME}'
\ No newline at end of file
--- a/build/providers/gcp/osdu_api.ini
+++ b/build/providers/gcp/osdu_api.ini
@@ -14,17 +14,17 @@
 # limitations under the License.

 [environment]
-data_partition_id=osdu
-storage_url=%(STORAGE_BASE_URL)s/api/storage/v2
-search_url=%(SEARCH_BASE_URL)s/api/search/v2
-legal_url=%(LEGAL_BASE_URL)s/api/legal/v1
-data_workflow_url=%(DATA_WORKFLOW_BASE_URL)s/api/data-workflow/v1
-file_dms_url=%(FILE_DMS_BASE_URL)s/api/filedms/v2
-dataset_url=%(DATASET_BASE_URL)s/api/dataset/v1
-entitlements_url=%(ENTITLEMENTS_BASE_URL)s/api/entitlements/v1
-schema_url=%(SCHEMA_BASE_URL)s/api/schema-service/v1
-ingestion_workflow_url=%(INGESTION_WORKFLOW_BASE_URL)s/api/workflow/v1
-partition_url=%(PARTITION_BASE_URL)s/api/partition/v1
+data_partition_id=%(DATA_PARTITION)s
+storage_url=%(OSDU_GCP_BASE_URL)s/api/storage/v2
+search_url=%(OSDU_GCP_BASE_URL)s/api/search/v2
+legal_url=%(OSDU_GCP_BASE_URL)s/api/legal/v1
+data_workflow_url=%(OSDU_GCP_BASE_URL)s/api/data-workflow/v1
+file_dms_url=%(OSDU_GCP_BASE_URL)s/api/filedms/v2
+dataset_url=%(OSDU_GCP_BASE_URL)s/api/dataset/v1
+entitlements_url=%(OSDU_GCP_BASE_URL)s/api/entitlements/v1
+schema_url=%(OSDU_GCP_BASE_URL)s/api/schema-service/v1
+ingestion_workflow_url=%(OSDU_GCP_BASE_URL)s/api/workflow/v1
+partition_url=%(OSDU_GCP_BASE_URL)s/api/partition/v1
 use_service_principal=False

 [provider]

--- a/build/requirements_test.txt
+++ b/build/requirements_test.txt
+jsonschema
+pytest
--- a/devops/osdu-gcp/airflow_configs.yaml
+++ b/devops/osdu-gcp/airflow_configs.yaml
@@ -11,7 +11,10 @@ energistics:
                  values:
                    - pool-1
    env_vars:
-      SA_FILE_PATH: "KEY_PATH"
+      SA_FILE_PATH: "{{ var.value.core__config__sa_file_path }}"
+      OSDU_GCP_BASE_URL: "{{ var.value.core__config__osdu_gcp_base_url }}"
+      DATA_PARTITION: "{{ var.value.core__config__data_partition }}"
+      CLOUD_PROVIDER: gcp
    resources:
      request_memory: "1Gi"
      request_cpu: 200m