Commit 38003fe6 authored by Vladislav Shishko (EPAM)'s avatar Vladislav Shishko (EPAM) Committed by Oleksandr Kosse (EPAM)
Browse files

(GONRG-3892) Refactor ci cd

parent 57f0b0e1
......@@ -18,107 +18,27 @@ image: docker:19.03
variables:
OSDU_GCP_VENDOR: gcp
OSDU_GCP_APPLICATION_NAME: energistics
OSDU_GCP_BASE_IMAGE_TAG: $CI_REGISTRY_IMAGE/osdu-gcp:$CI_COMMIT_SHORT_SHA
WORKDIR: energistics
SRCDIR: energistics/src/
TESTDIR: energistics/tests
VENDOR_NAME: energistics
BASE_IMAGE: $CI_REGISTRY_IMAGE/witsml-parser
OSDU_GCP_IMAGE: $CI_REGISTRY_IMAGE/osdu-gcp-witsml-parser
BASE_IMAGE_TAG: $BASE_IMAGE:$CI_COMMIT_SHORT_SHA
stages:
- test
- containerize
- csp_containerize
- build_and_test
- deploy
- end_to_end_postman_tests
.add_system_deps: &add_system_deps
- echo ---- ---- ---- SYSTEM DEPENDENCIES ---- ---- ----
- apk update
- apk add git
.run_tests: &run_tests
- echo ---- ---- ---- TESTS ---- ---- ----
- docker build -t witsml:test --rm -f ./build/test/Dockerfile --build-arg base_image=$BASE_IMAGE_TAG .
- docker run witsml:test || exit $?
build_base_image:
stage: containerize
tags: ["osdu-medium"]
script:
- *add_system_deps
- echo ---- ---- ---- CLONE REPOSITORIES ---- ---- ----
- >
cd energistics/src/witsml_parser/osdu
&& git clone https://community.opengroup.org/osdu/platform/system/sdks/common-python-sdk.git
&& git clone https://community.opengroup.org/osdu/platform/data-flow/ingestion/ingestion-dags.git
&& cd - && ls
- echo ---- ---- ---- BUILD IMAGE ---- ---- ----
- docker build -t $BASE_IMAGE_TAG --rm -f ./build/Dockerfile .
- *run_tests
- echo ---- ---- ---- PUSH IMAGE ---- ---- ----
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- docker push $BASE_IMAGE_TAG
when: manual
.gcp_update_ini_file: &gcp_update_ini_file
- echo ---- ---- ---- UPDATE ENV VARS ---- ---- ----
- sed -i "s|%([_A-Z]*_BASE_URL)s|$OSDU_GCP_BASE_URL|" build/providers/gcp/osdu_api.ini
.gcp_update_airflow_config: &gcp_update_airflow_config
- echo ---- ---- ---- UPDATE AIRFLOW CONFIG ---- ---- ----
- sed -i "s|IMAGE|$OSDU_GCP_IMAGE_TAG|" devops/osdu-gcp/airflow_configs.yaml
.build_osdu_gcp:
stage: csp_containerize
tags: ["osdu-medium"]
needs: ["build_base_image"]
script:
- *add_system_deps
- *gcp_update_ini_file
- echo ---- ---- ---- BUILD IMAGE ---- ---- ----
- docker build -t $OSDU_GCP_IMAGE_TAG --rm -f ./build/providers/$OSDU_GCP_VENDOR/Dockerfile.energistics --build-arg PROVIDER_NAME=$OSDU_GCP_VENDOR --build-arg base_image=$BASE_IMAGE_TAG .
- echo ---- ---- ---- PUSH IMAGE ---- ---- ----
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- docker push $OSDU_GCP_IMAGE_TAG
when: manual
artifacts:
when: always
paths:
- devops/osdu-gcp/airflow_configs.yaml
expire_in: 1 week
build_osdu_gcp_image:
extends: .build_osdu_gcp
variables:
OSDU_GCP_BASE_URL: https://community.osdu-gcp.go3-nrg.projects.epam.com
OSDU_GCP_IMAGE_TAG: $OSDU_GCP_IMAGE:latest
after_script:
- *gcp_update_airflow_config
except:
- /^release\/*/
build_osdu_gcp_release_image:
extends: .build_osdu_gcp
variables:
OSDU_GCP_BASE_URL: https://preship-asm.osdu-gcp.go3-nrg.projects.epam.com
OSDU_GCP_IMAGE_TAG: $OSDU_GCP_IMAGE:release-latest
after_script:
- if [ -n "$CI_COMMIT_TAG" ]; then docker tag $OSDU_GCP_IMAGE_TAG $OSDU_GCP_IMAGE:$CI_COMMIT_TAG && docker push $OSDU_GCP_IMAGE:$CI_COMMIT_TAG && OSDU_GCP_IMAGE_TAG=$OSDU_GCP_IMAGE:$CI_COMMIT_TAG; fi
- *gcp_update_airflow_config
only:
- tags
- /^release\/*/
.gcp_rsync:
.gcp_rsync_deploy:
image: gcr.io/google.com/cloudsdktool/cloud-sdk
stage: deploy
script:
- gcloud auth activate-service-account --key-file $GCP_KEY_FILE
- echo $GCP_KEY_FILE > key.json
- gcloud auth activate-service-account --key-file key.json
- gcloud config set project "$GCP_PROJECT"
- sed -i "s|KEY_PATH|$GCP_KEY_PATH|" devops/osdu-gcp/airflow_configs.yaml
- sed -i "s|IMAGE|$OSDU_GCP_BASE_IMAGE_TAG|" devops/osdu-gcp/airflow_configs.yaml
- pushd "$SRCDIR"
- ls
- echo *******
......@@ -126,9 +46,27 @@ build_osdu_gcp_release_image:
- popd
- gsutil cp devops/osdu-gcp/airflow_configs.yaml "$GCP_DEPLOY_TARGET"/dags/configs/
.gcp_e2e_test_dag:
image: gcr.io/google.com/cloudsdktool/cloud-sdk
stage: end_to_end_postman_tests
script:
- chmod +x tests/e2e_postman_tests/gcp/run_e2e_tests.sh
- tests/e2e_postman_tests/gcp/./run_e2e_tests.sh $GCP_TEST_ENV
build_and_test:
stage: build_and_test
tags: ["osdu-medium"]
script:
- export EXTRA_DOCKER_TAG=""; if [ "$CI_COMMIT_TAG" != "" ] ; then EXTRA_DOCKER_TAG="-t $CI_REGISTRY_IMAGE/osdu-gcp:$CI_COMMIT_TAG" ; elif [ "$CI_COMMIT_REF_NAME" = "master" ] ; then EXTRA_DOCKER_TAG="-t $CI_REGISTRY_IMAGE/osdu-gcp:latest" ; fi
- docker build --target gcp -t $OSDU_GCP_BASE_IMAGE_TAG $EXTRA_DOCKER_TAG --rm -f ./build/Dockerfile .
- docker build --target test -t witsml:test --rm -f ./build/Dockerfile .
- docker run witsml:test || exit $?
- docker rmi -f witsml:test
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- docker push $CI_REGISTRY_IMAGE/osdu-gcp
osdu_gcp_dags_rsync_comm:
extends: .gcp_rsync
needs: [build_osdu_gcp_image]
extends: .gcp_rsync_deploy
variables:
GCP_DEPLOY_TARGET: $OSDU_GCP_DEPL_TARGET_V2
GCP_KEY_FILE: $OSDU_GCP_DEPLOY_FILE
......@@ -137,11 +75,9 @@ osdu_gcp_dags_rsync_comm:
only:
- $CI_DEFAULT_BRANCH
- /^trusted-/
when: manual
osdu_gcp_dags_rsync_prod:
extends: .gcp_rsync
needs: [build_osdu_gcp_release_image]
extends: .gcp_rsync_deploy
variables:
GCP_DEPL_TARGET: $OSDU_GCP_DEPL_TARGET_PREPROD
GCP_KEY_FILE: $OSDU_GCP_PROD_SVC_ACCT
......@@ -150,14 +86,6 @@ osdu_gcp_dags_rsync_prod:
only:
- tags
- /^release\/*/
when: manual
.gcp_e2e_test_dag:
image: gcr.io/google.com/cloudsdktool/cloud-sdk
stage: end_to_end_postman_tests
script:
- chmod +x tests/e2e_postman_tests/gcp/run_e2e_tests.sh
- tests/e2e_postman_tests/gcp/./run_e2e_tests.sh $GCP_TEST_ENV
gcp_e2e_test_dag_comm:
extends: .gcp_e2e_test_dag
......
......@@ -14,7 +14,7 @@
# limitations under the License.
FROM python:3.6-slim
FROM python:3.8-slim as base
ENV HOME="/home/witsml_parser"
......@@ -29,6 +29,7 @@ ENV OSDU_CORE_PROVIDERS_DIR="${WITSML_PARSER_DIR}/osdu/ingestion-dags/src/dags/p
COPY ./${BUILD_DIR}/requirements.txt ./
COPY ./${WITSML_PARSER_DIR}/*.py ./
COPY ./${ENERGISTICS_LIBS} ./energistics
COPY ./${BUILD_DIR}/providers/gcp/osdu_api.ini ./
# TODO: Configure Package Registry for Python SDK
RUN pip install --upgrade pip && \
......@@ -36,3 +37,22 @@ RUN pip install --upgrade pip && \
pip install -r "./requirements.txt"
ENV PYTHONPATH="$PYTHONPATH:$HOME"
## test
FROM base as test
COPY ./${BUILD_DIR}/requirements_test.txt ./
COPY "./energistics/src/docker_test" ./test
RUN pip install -r "requirements_test.txt"
CMD pytest test
## gcp
FROM base as gcp
ARG PROVIDER_NAME=gcp
ENV CLOUD_PROVIDER $PROVIDER_NAME
COPY "build/providers/${PROVIDER_NAME}/requirements.txt" ./
RUN pip install -r "requirements.txt"
COPY build/providers/gcp/osdu_api.ini ./
# Copyright 2021 Google LLC
# Copyright 2021 EPAM
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
ARG base_image
FROM ${base_image}
ENV HOME="/home/witsml_parser"
WORKDIR $HOME
ARG PROVIDER_NAME
ENV CLOUD_PROVIDER $PROVIDER_NAME
COPY "build/providers/${PROVIDER_NAME}/requirements.txt" ./
RUN pip install -r "requirements.txt"
COPY build/providers/gcp/osdu_api.ini ./
# Copyright 2021 Google LLC
# Copyright 2017-2019, Schlumberger
# Copyright 2021 EPAM
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
steps:
- id: base
name: 'gcr.io/cloud-builders/docker'
args: [
'build',
'--build-arg', 'PROVIDER_NAME=${_PROVIDER_NAME}',
'-t', 'gcr.io/$PROJECT_ID/${_APPLICATION_NAME}/${_GCP_SERVICE}-${_PROVIDER_NAME}-builder:witsml',
'-f', 'build/Dockerfile',
'.'
]
waitFor: ['-']
- name: 'gcr.io/cloud-builders/docker'
args: [
'build',
'--build-arg', 'PROVIDER_NAME=${_PROVIDER_NAME}',
'--build-arg', 'builder_image=gcr.io/$PROJECT_ID/${_APPLICATION_NAME}/${_GCP_SERVICE}-${_PROVIDER_NAME}-builder:witsml',
'-t', 'gcr.io/$PROJECT_ID/${_APPLICATION_NAME}/${_GCP_SERVICE}-${_PROVIDER_NAME}:${_SHORT_SHA}',
'-t', 'gcr.io/$PROJECT_ID/${_APPLICATION_NAME}/${_GCP_SERVICE}-${_PROVIDER_NAME}:latest',
'-f', 'build/providers/${_PROVIDER_NAME}/Dockerfile.energistics',
'.'
]
waitFor: ["base"]
images:
- 'gcr.io/$PROJECT_ID/${_APPLICATION_NAME}/${_GCP_SERVICE}-${_PROVIDER_NAME}-builder'
- 'gcr.io/$PROJECT_ID/${_APPLICATION_NAME}/${_GCP_SERVICE}-${_PROVIDER_NAME}'
\ No newline at end of file
......@@ -14,17 +14,17 @@
# limitations under the License.
[environment]
data_partition_id=osdu
storage_url=%(STORAGE_BASE_URL)s/api/storage/v2
search_url=%(SEARCH_BASE_URL)s/api/search/v2
legal_url=%(LEGAL_BASE_URL)s/api/legal/v1
data_workflow_url=%(DATA_WORKFLOW_BASE_URL)s/api/data-workflow/v1
file_dms_url=%(FILE_DMS_BASE_URL)s/api/filedms/v2
dataset_url=%(DATASET_BASE_URL)s/api/dataset/v1
entitlements_url=%(ENTITLEMENTS_BASE_URL)s/api/entitlements/v1
schema_url=%(SCHEMA_BASE_URL)s/api/schema-service/v1
ingestion_workflow_url=%(INGESTION_WORKFLOW_BASE_URL)s/api/workflow/v1
partition_url=%(PARTITION_BASE_URL)s/api/partition/v1
data_partition_id=%(DATA_PARTITION)s
storage_url=%(OSDU_GCP_BASE_URL)s/api/storage/v2
search_url=%(OSDU_GCP_BASE_URL)s/api/search/v2
legal_url=%(OSDU_GCP_BASE_URL)s/api/legal/v1
data_workflow_url=%(OSDU_GCP_BASE_URL)s/api/data-workflow/v1
file_dms_url=%(OSDU_GCP_BASE_URL)s/api/filedms/v2
dataset_url=%(OSDU_GCP_BASE_URL)s/api/dataset/v1
entitlements_url=%(OSDU_GCP_BASE_URL)s/api/entitlements/v1
schema_url=%(OSDU_GCP_BASE_URL)s/api/schema-service/v1
ingestion_workflow_url=%(OSDU_GCP_BASE_URL)s/api/workflow/v1
partition_url=%(OSDU_GCP_BASE_URL)s/api/partition/v1
use_service_principal=False
[provider]
......
......@@ -11,7 +11,10 @@ energistics:
values:
- pool-1
env_vars:
SA_FILE_PATH: "KEY_PATH"
SA_FILE_PATH: "{{ var.value.core__config__sa_file_path }}"
OSDU_GCP_BASE_URL: "{{ var.value.core__config__osdu_gcp_base_url }}"
DATA_PARTITION: "{{ var.value.core__config__data_partition }}"
CLOUD_PROVIDER: gcp
resources:
request_memory: "1Gi"
request_cpu: 200m
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment