Skip to content
Snippets Groups Projects
.gitlab-ci.yml 6.19 KiB
Newer Older
#  Copyright 2022 Google LLC
#  Copyright 2022 EPAM Systems
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

image: docker:19.03
workflow:
  rules:
  # If the pipelines are from release or tag commits then set azure deployment env variable value   
  - if: $CI_COMMIT_BRANCH =~ /^release/ || $CI_COMMIT_TAG
    variables:
      DEPLOY_ENVIRONMENT_AZURE: "AZURE_STAGE"
  - when: always


variables:
  OSDU_AZURE_POSTFIX: osdu-azure
  OSDU_AZURE_BASE_IMAGE_TAG: $CI_REGISTRY_IMAGE/$OSDU_AZURE_POSTFIX:$CI_COMMIT_SHORT_SHA
  AZURE_WITSML_IMAGE_NAME: ${CI_PROJECT_NAME}:${CI_COMMIT_SHA}
  AZURE_PUBLISH_PARSER_IMAGE: "true" 
Mohammed Ali's avatar
Mohammed Ali committed
  WORKDIR: energistics
  SRCDIR: energistics/src
Mohammed Ali's avatar
Mohammed Ali committed
  TESTDIR: energistics/tests
  VENDOR_NAME: energistics
Elsi Abraham's avatar
Elsi Abraham committed
  #AWS
  AWS_DEPLOY_TARGET: TF
Elsi Abraham's avatar
Elsi Abraham committed
  # AWS skip stages
  SKIP_BOOTSTRAP: "true"
  SKIP_PUBLISH: "true"
  SKIP_END_TO_END: "false"
Elsi Abraham's avatar
Elsi Abraham committed
  # AWS collection vars
Abhishek Patil's avatar
Abhishek Patil committed
  AWS_COLLECTION_PATH: https://community.opengroup.org/osdu/qa/-/raw/main/Postman%20Collection/41_CICD_Setup_WITSMLIngestion/WITSML%20Energistics%20XML%20Ingest%20CI-CD%20v3.0.postman_collection.json
  AWS_COLLECTION_ENV_PATH: https://community.opengroup.org/osdu/qa/-/raw/main/Postman%20Collection/00_CICD_Setup_Environment/aws.OSDU%20R3%20PROD%20v2.4.postman_environment.json
  AWS_COLLECTION_TEST_PATH: https://community.opengroup.org/osdu/qa/-/raw/main/Postman%20Collection/41_CICD_Setup_WITSMLIngestion/Trajectory_witsml.xml
Elsi Abraham's avatar
Elsi Abraham committed
  AWS_TEST_FILE_INPUT: aws_WitSML.xml
  AWS_TEST_FILE_KEY: witsmlTrajectoryFile
  AWS_POSTMAN_TESTS_TO_SKIP: "08 Get Storage Records|16 Get Storage Records|23 Check workflow status|24 Get Storage Records|25 Get Storage Instructions - Dataset|27 Create Dataset Registry|29 Create Workflow|30 Trigger Workflow AWS and Google Cloud|31 Check workflow status|32 Get Storage Records|33 Get Storage Instructions - Dataset|35 Create Dataset Registry|37 Create Workflow|38 Trigger Workflow AWS and Google Cloud|39 Check workflow status|40 Get Storage Records|00 create TubularAssembly WPC Till DAG gets fixed|41 Get Storage Instructions - Dataset|43 Create Dataset Registry|45 Create Workflow|46 Trigger Workflow AWS and Google Cloud|47 Check workflow status|48 Get Storage Records|48a Get All tubular components using dataset--File record"
Elsi Abraham's avatar
Elsi Abraham committed

  - project: "osdu/platform/ci-cd-pipelines"
    file: "cloud-providers/azure_dag.yml"
  - project: "osdu/platform/ci-cd-pipelines"
    file: "cloud-providers/gc-containerize.yml"

  - project: "osdu/platform/ci-cd-pipelines"
    file: "cloud-providers/gc-dag.yaml"
  - local: "devops/gc/pipeline/override-stages.yml"
Walter D's avatar
Walter D committed
  - local: "/devops/osdu-ibm/ibm-stages.yml"

Elsi Abraham's avatar
Elsi Abraham committed
  - project: "osdu/platform/ci-cd-pipelines"
    file: "cloud-providers/aws-global-variables.yml"

  - project: "osdu/platform/ci-cd-pipelines"
    file: "cloud-providers/aws-dag.yml"

  - containerize
  - bootstrap_dag
  - publish_dag
  - end_to_end_postman_tests
  - deploy_preship
.azure_prepare_dag_env:
  variables:
    AZURE_K8S_NAMESPACE: airflow
    AZURE_DAG_DIRECTORY: src/energistics
    DATA_PARTITION: "{{ var.value.core__config__data_partition }}"
    OSDU_AZURE_BASE_URL: "{{ var.value.core__config__osdu_azure_base_url }}"

test_parser:
  stage: test
  tags: ["osdu-small"]
  script:
    - docker build --target test -t witsml:test --rm -f ./build/Dockerfile .
    - docker run witsml:test || exit $?
    - docker rmi -f witsml:test
build_parser_azure:
  stage: build
  environment: $DEPLOY_ENVIRONMENT_AZURE
  needs: []
  tags: ["osdu-medium"]
  image: danielscholl/azure-build-image
  variables:
    SHA_IMAGE: ${CI_PROJECT_NAME}:${CI_COMMIT_SHA}
    LATEST_IMAGE: ${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}:latest
    RELEASE_IMAGE: ${CI_PROJECT_NAME}:${CI_COMMIT_SHA}
  before_script:
    - az login --service-principal -u $AZURE_PRINCIPAL_ID -p $AZURE_PRINCIPAL_SECRET --tenant $AZURE_TENANT_ID
    - az acr login -n $AZURE_REGISTRY
  script:
    - |
      if [ "$CI_COMMIT_REF_NAME" = "master" ]; then
        docker build --target azure -t ${AZURE_REGISTRY}.azurecr.io/$LATEST_IMAGE --rm -f ./build/providers/azure/Dockerfile .
        docker push ${AZURE_REGISTRY}.azurecr.io/$LATEST_IMAGE

    - docker build --target azure -t ${AZURE_REGISTRY}.azurecr.io/$SHA_IMAGE --rm -f ./build/providers/azure/Dockerfile .
    - docker push ${AZURE_REGISTRY}.azurecr.io/$SHA_IMAGE
  only:
    variables:
azure_standalone_tests:
  stage: test
    AZURE_SKIP_STANDALONE_TESTS: "true"
  needs: ["build_parser_azure"]
  extends:
    - .azure_variables
    - .azure_prepare_dag_env
  before_script:
    - |
      if [ ! -z "$CI_COMMIT_TAG" ]; then
        AZURE_WITSML_IMAGE_NAME="${CI_PROJECT_NAME}:${CI_COMMIT_SHA}"
      fi
    - |
      # Generating environment file to be passed while running the docker container
      cat > .env << EOF
      CI_REGISTRY_USER=${CI_REGISTRY_USER}
      CI_REGISTRY_PASSWORD=${CI_REGISTRY_PASSWORD}
      CI_REGISTRY=${CI_REGISTRY}
      AZURE_REGISTRY=${AZURE_REGISTRY}
      AZURE_PRINCIPAL_ID=${AZURE_PRINCIPAL_ID}
      AZURE_PRINCIPAL_SECRET=${AZURE_PRINCIPAL_SECRET}
      AZURE_TENANT_ID=${AZURE_TENANT_ID}
      CI_PROJECT_NAME=${CI_PROJECT_NAME}
      CI_COMMIT_REF_SLUG=${CI_COMMIT_REF_SLUG}
      CI_COMMIT_SHA=${CI_COMMIT_SHA}
      CI_REGISTRY_IMAGE=${CI_REGISTRY_IMAGE}
      AZURE_DEPLOYMENTS_SCRIPTS_SUBDIR=${AZURE_DEPLOYMENTS_SCRIPTS_SUBDIR}
      PARSER_IMAGE=${PARSER_IMAGE}
      DAG_IMAGE=${DAG_IMAGE}
      SHARED_TENANT=$SHARED_TENANT
      AZURE_DNS_NAME=$AZURE_DNS_NAME
      WITSML_IMAGE_NAME=${AZURE_WITSML_IMAGE_NAME}
      K8S_NAMESPACE=$AZURE_K8S_NAMESPACE
      AZURE_DAG_DIRECTORY=$AZURE_DAG_DIRECTORY
      DATA_PARTITION=$DATA_PARTITION
      OSDU_AZURE_BASE_URL=$OSDU_AZURE_BASE_URL