.gitlab-ci.yml

#  Copyright 2020 Google LLC
#  Copyright 2020 EPAM Systems
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

variables:
  OSDU_GCP_ENABLE_BOOTSTRAP: "true"
  OSDU_GCP_SERVICE: "ingestion-dags"

  #AWS variables
  AWS_DEPLOY_TARGET: EKS
  # AWS skip stages
  SKIP_BOOTSTRAP: 'true'
  SKIP_PUBLISH: 'true'
  SKIP_END_TO_END: 'false'
  # AWS collection vars
  AWS_COLLECTION_PATH: https://community.opengroup.org/osdu/platform/testing/-/raw/master/Postman%20Collection/29_CICD_Setup_Ingestion/R3%20Full%20manifest-based%20ingestion.postman_collection.json
  AWS_COLLECTION_ENV_PATH: https://community.opengroup.org/osdu/platform/testing/-/raw/master/Postman%20Collection/00_CICD_Setup_Environment/aws.OSDU%20R3%20PROD%20v2.4.postman_environment.json
  AWS_COLLECTION_TEST_PATH: ''
  AWS_TEST_FILE_INPUT: ''
  AWS_TEST_FILE_KEY: ''
  AWS_POSTMAN_TESTS_TO_SKIP: "01a - Reference data ingestion|01b - Workflow - Get Work Status|01c - Search Records|02a - Master data ingestion|02b - Workflow - Get Work Status|02c - Search Records|03a - Master data ingestion|03b - Workflow - Get Work Status|03c - Search Records|05a - WP ingestion, pre-defined ids crazy format|05b - Workflow - Get Work Status|05c - Search Records WP|05d - Search Records WPC|05e - Search Records Dataset|06a - WP ingestion surrogate keys|06b - Workflow - Get Work Status|07a - WP ingestion, pre-defined ids short format|07b - Workflow - Get Work Status|07c - Search Records WP|07d - Search Records WPC|07e - Search Records Dataset|08a - WP ingestion File Collection|08b - Workflow - Get Work Status|08c - Search Records WP|08d - Search Records WPC|08e - Search Records Dataset|09a - WP ingestion Artefact|09b - Workflow - Get Work Status|10a - Multiple manifests|10b - Workflow - Get Work Status|10c - Search Records MasterData|09d - Search Records Reference Data|11a - 3 types|11b - Workflow - Get Work Status|11c - Search Records WP|11d - Search Records WPC|11e - Search Records Dataset|11f- Search Records MasterData|11g - Search Records Reference Data|12a - NS parent data doesnt exist|12b - Workflow - Get Work Status|13a - NS extra WPC in WP block|13b - Workflow - Get Work Status|14a - NS extra Dataset in WPC block|14b - Workflow - Get Work Status|15a - NS File Collection no FileSource|15b - Workflow - Get Work Status|16a - NS File no File Source|16b - Workflow - Get Work Status"

image: google/cloud-sdk:alpine

stages:
  - linters
  - unit_tests
  - test_dags
  - containerize
  - bootstrap_dag
  - publish_dag
  - deploy
  - bootstrap
  - end_to_end_postman_tests

include:
  - project: "osdu/platform/ci-cd-pipelines"
    file: "cloud-providers/azure_dag.yml"

  - project: "osdu/platform/ci-cd-pipelines"
    file: "build/python-linters.yml"

  - project: "osdu/platform/ci-cd-pipelines"
    file: "cloud-providers/osdu-gcp-dag.yaml"

  - project: "osdu/platform/ci-cd-pipelines"
    file: "cloud-providers/osdu-gcp-bootstrap.yml"

  - local: "/devops/ibm/ibm-stages.yml"

  - project: "osdu/platform/ci-cd-pipelines"
    file: "cloud-providers/aws-global-variables.yml"

  - project: "osdu/platform/ci-cd-pipelines"
    file: "cloud-providers/aws-dag.yml"

osdu-gcp-containerize-bootstrap-gitlab:
  variables:
    IMAGE_NAME: osdu-gcp-bootstrap-dags
    BUILD_PATH: devops/gcp/Dockerfile

osdu-gcp-containerize-bootstrap-gcr:
  variables:
    IMAGE_NAME: osdu-gcp-bootstrap-dags
    BUILD_PATH: devops/gcp/Dockerfile

pylint:
  variables:
    SRC_FILES: "src/*/*.py"
  before_script:
    - pip install -r src/osdu_dags/requirements.txt

isort:
  variables:
    SRC_FILES: "src/*/*.py"

test_dags:
  tags: ["osdu-small"]
  stage: test_dags
  needs: ["pylint", "isort"]
  image: eu.gcr.io/osdu-cicd-epam/airflow-python-dags/airflow-python-dags:latest
  script:
    - chmod +x tests/test_dags.sh
    - tests/./test_dags.sh  || EXIT_CODE=$?
    - exit ${EXIT_CODE}
  artifacts:
    paths:
      - logs/
    when: on_failure
    expire_in: 1 week

unit_tests:
  tags: ["osdu-small"]
  stage: unit_tests
  needs: ["pylint", "isort"]
  image: eu.gcr.io/osdu-cicd-epam/airflow-python-dags/airflow-python-dags:latest
  script:
    - chmod +x tests/unit_tests.sh
    # - tests/./unit_tests.sh  || EXIT_CODE=$? #TODO: unit tests moved to airflow-osdu-lib, need to remove `unit_tests` CI step later
    - exit ${EXIT_CODE}

osdu_gcp_publish_dag:
  stage: publish_dag
  variables:
    BOOTSTRAPPED_DAG_PATH: src/osdu_dags/osdu-ingest-r3.py
    DAG_FILE: osdu-ingest-r3.py

osdu_anthos_publish_dag:
  stage: publish_dag
  variables:
    BOOTSTRAPPED_DAG_PATH: src/osdu_dags/osdu-ingest-r3.py
    DAG_FILE: osdu-ingest-r3.py

osdu_gcp_deploy_composer_community:
  stage: deploy
  needs: ["unit_tests", "test_dags"]
  variables:
    DAG_FOLDER: "src/osdu_dags"

osdu_gcp_deploy_composer_pre_ship:
  stage: deploy
  needs: ["unit_tests", "test_dags"]
  variables:
    DAG_FOLDER: "src/osdu_dags"

osdu_gcp_postman_tests_community:
  stage: end_to_end_postman_tests
  variables:
    TEST_SOURCE_FILES_DIR: "Postman Collection/29_CICD_Setup_Ingestion"
    COLLECTION_PATH: "Postman Collection/29_CICD_Setup_Ingestion/R3 Full manifest-based ingestion.postman_collection.json"

osdu_gcp_postman_tests_pre_ship:
  stage: end_to_end_postman_tests
  variables:
    TEST_SOURCE_FILES_DIR: "Postman Collection/29_CICD_Setup_Ingestion"
    COLLECTION_PATH: "Postman Collection/29_CICD_Setup_Ingestion/R3 Full manifest-based ingestion.postman_collection.json"

osdu_gcp_bootstrap_dag_community:
  stage: deploy
  variables:
    OSDU_GCP_SKIP_BOOTSTRAP: "true"

osdu_gcp_bootstrap_dag_pre_ship:
  stage: deploy
  variables:
    OSDU_GCP_SKIP_BOOTSTRAP: "true"

osdu_anthos_bootstrap_dag:
  stage: deploy
  variables:
    OSDU_ANTHOS_SKIP_BOOTSTRAP: "true"

# We are skipping this stage using AZURE_SKIP_STANDALONE_TESTS variable as there is already a common stage for Executing Unit
# tests, but we need to override here because of stage name mismatch
azure_standalone_tests:
  stage: unit_tests
  needs: ["pylint"]
  variables:
    AZURE_SKIP_STANDALONE_TESTS: "true"

azure_build_dag:
  needs: ["unit_tests"]