.gitlab-ci.yml

#  Copyright 2022 Google LLC
#  Copyright 2022 EPAM Systems
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

variables:
  IBM_HELM_DAG_PATH: devops/ibm/ibm-manifest-dag
  #AWS variables
  AWS_DEPLOY_TARGET: EKS
  # AWS skip stages
  SKIP_BOOTSTRAP: "true"
  SKIP_PUBLISH: "true"
  SKIP_END_TO_END: "false"
  # AWS collection vars
  AWS_COLLECTION_PATH: https://community.opengroup.org/osdu/platform/testing/-/raw/master/Postman%20Collection/29_CICD_Setup_Ingestion/R3%20Full%20manifest-based%20ingestion.postman_collection.json
  AWS_COLLECTION_ENV_PATH: https://community.opengroup.org/osdu/platform/testing/-/raw/master/Postman%20Collection/00_CICD_Setup_Environment/aws.OSDU%20R3%20PROD%20v2.4.postman_environment.json
  AWS_COLLECTION_TEST_PATH: ""
  AWS_TEST_FILE_INPUT: ""
  AWS_TEST_FILE_KEY: ""
  AWS_POSTMAN_TESTS_TO_SKIP: "01a - Reference data ingestion|01b - Workflow - Get Work Status|01c - Search Records|02a - Master data ingestion|02b - Workflow - Get Work Status|02c - Search Records|03a - Master data ingestion|03b - Workflow - Get Work Status|03c - Search Records|05a - WP ingestion, pre-defined ids crazy format|05b - Workflow - Get Work Status|05c - Search Records WP|05d - Search Records WPC|05e - Search Records Dataset|06a - WP ingestion surrogate keys|06b - Workflow - Get Work Status|07a - WP ingestion, pre-defined ids short format|07b - Workflow - Get Work Status|07c - Search Records WP|07d - Search Records WPC|07e - Search Records Dataset|08a - WP ingestion File Collection|08b - Workflow - Get Work Status|08c - Search Records WP|08d - Search Records WPC|08e - Search Records Dataset|09a - WP ingestion Artefact|09b - Workflow - Get Work Status|10a - Multiple manifests|10b - Workflow - Get Work Status|10c - Search Records MasterData|09d - Search Records Reference Data|11a - 3 types|11b - Workflow - Get Work Status|11c - Search Records WP|11d - Search Records WPC|11e - Search Records Dataset|11f- Search Records MasterData|11g - Search Records Reference Data|12a - NS parent data doesnt exist|12b - Workflow - Get Work Status|13a - NS extra WPC in WP block|13b - Workflow - Get Work Status|14a - NS extra Dataset in WPC block|14b - Workflow - Get Work Status|15a - NS File Collection no FileSource|15b - Workflow - Get Work Status|16a - NS File no File Source|16b - Workflow - Get Work Status"

image: google/cloud-sdk:alpine

stages:
  - linters
  - unit_tests
  - test_dags
  - build
  - containerize
  - bootstrap_dag
  - publish_dag
  - deploy
  - bootstrap
  - end_to_end_postman_tests
  - deploy_preship

include:
  - project: "osdu/platform/ci-cd-pipelines"
    file: "cloud-providers/azure_dag.yml"

  - project: "osdu/platform/ci-cd-pipelines"
    file: "build/python-linters.yml"

  - project: "osdu/platform/ci-cd-pipelines"
    file: "cloud-providers/osdu-gcp-dag.yaml"

  - local: "/devops/gcp/pipeline/override-stages.yml"

  - local: "/devops/ibm/ibm-stages.yml"

  - project: "osdu/platform/ci-cd-pipelines"
    file: "cloud-providers/aws-global-variables.yml"

  - project: "osdu/platform/ci-cd-pipelines"
    file: "cloud-providers/aws-dag.yml"

pylint:
  variables:
    SRC_FILES: "src/*/*.py"
  before_script:
    - pip install -r src/osdu_dags/requirements.txt

isort:
  variables:
    SRC_FILES: "src/*/*.py"

test_dags:
  tags: ["osdu-small"]
  stage: test_dags
  needs: ["pylint", "isort"]
  image:
    name: apache/airflow:2.1.2-python3.8
    entrypoint: [""]
  script:
    - sh -c "tests/./test_dags.sh"  || EXIT_CODE=$?
    - exit ${EXIT_CODE}
  artifacts:
    paths:
      - logs/
    when: on_failure
    expire_in: 1 week

unit_tests:
  tags: ["osdu-small"]
  stage: unit_tests
  needs: ["pylint", "isort"]
  image:
    name: apache/airflow:2.1.2-python3.8
    entrypoint: [""]
  script:
    # - tests/./unit_tests.sh  || EXIT_CODE=$? #TODO: unit tests moved to airflow-osdu-lib, need to remove `unit_tests` CI step later
    - exit 0

# We are skipping this stage using AZURE_SKIP_STANDALONE_TESTS variable as there is already a common stage for Executing Unit
# tests, but we need to override here because of stage name mismatch
azure_standalone_tests:
  stage: unit_tests
  needs: ["pylint"]
  variables:
    AZURE_SKIP_STANDALONE_TESTS: "true"

azure_build_dag:
  needs: ["unit_tests"]