diff --git a/.gitignore b/.gitignore index d4eaab722280ec3d8a06f510ffd5d16d5ef2309c..40378b17bdeeb3624e9411c3d7ab24653b1a0ddd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ **/.idea **/.DS_Store +**/.vscode # Byte-compiled / optimized / DLL files **/__pycache__/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 162c225bfc3f59a7b32608828d8f3eaae0048056..dab232ca732ded7839a684320fae57de08c441e5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -27,12 +27,24 @@ isort: - isort -c -v osdu_airflow/*/*.py || EXIT_CODE=$? - exit ${EXIT_CODE} -unit_tests: +unit_tests_airflow_v1: stage: unit_tests - image: eu.gcr.io/osdu-cicd-epam/airflow-python-dags/airflow-python-dags:latest + image: + name: apache/airflow:1.10.14-python3.6 + entrypoint: [""] script: - - chmod +x ./osdu_airflow/tests/unit_tests.sh - - ./osdu_airflow/tests/./unit_tests.sh || EXIT_CODE=$? + - pip install -r requirements-dev.txt + - sh -c "./osdu_airflow/tests/./unit_tests.sh" || EXIT_CODE=$? + - exit ${EXIT_CODE} + +unit_tests_airflow_v2: + stage: unit_tests + image: + name: apache/airflow:2.1.2-python3.8 + entrypoint: [""] + script: + - pip install -r requirements-dev.txt + - sh -c "./osdu_airflow/tests/./unit_tests.sh" || EXIT_CODE=$? - exit ${EXIT_CODE} deploy: diff --git a/VERSION b/VERSION index ac454c6a1fc3f05f60d3772b45f0b1a5db4b9f87..54d1a4f2a4a7f6afc19897c88a7b73c17ccc54fb 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.12.0 +0.13.0 diff --git a/osdu_airflow/tests/airflow_var.json b/osdu_airflow/tests/airflow_var.json new file mode 100644 index 0000000000000000000000000000000000000000..c069a25fd4f2908e84292932eafbe5ce746d60ec --- /dev/null +++ b/osdu_airflow/tests/airflow_var.json @@ -0,0 +1,11 @@ +{ + "core__service__storage__url": "https://test", + "core__service__workflow__host": "https://test", + "core__service__file__host": "htpps://test", + "core__service__workflow__url": "https://test", + "core__service__search__url": "https://test", + "core__service__schema__url": "https://test", + "core__config__dataload_config_path": "https://test", + "core__auth__access_token": "test", + "core__ingestion__batch_count": 3 +} diff --git a/osdu_airflow/tests/plugin-unit-tests/test_operators_r3.py b/osdu_airflow/tests/plugin-unit-tests/test_operators_r3.py index 18ff82e6dd8c375ca4a4f023bf4b9adc011b047d..c87823939562835068672660f445a62d1ebfdd88 100644 --- a/osdu_airflow/tests/plugin-unit-tests/test_operators_r3.py +++ b/osdu_airflow/tests/plugin-unit-tests/test_operators_r3.py @@ -16,31 +16,27 @@ import http import json -import os -import sys from datetime import datetime -from typing import TypeVar, ClassVar -from airflow import DAG -from airflow.models import TaskInstance - -sys.path.append(f"{os.getenv('AIRFLOW_SRC_DIR')}/plugins") -sys.path.append(f"{os.getenv('AIRFLOW_SRC_DIR')}/dags") -sys.path.insert(0, './') +from typing import ClassVar, TypeVar -from osdu_ingestion.libs.exceptions import PipelineFailedError import pytest -import requests -import mock_providers -from functools import lru_cache - -from file_paths import ( - MANIFEST_WELLBORE_VALID_PATH, - SEARCH_VALID_RESPONSE_PATH, MANIFEST_GENERIC_SCHEMA_PATH, MANIFEST_BATCH_WELLBORE_VALID_PATH) -from osdu_airflow.operators.process_manifest_r3 import ProcessManifestOperatorR3, SchemaValidator, \ - ManifestProcessor -from osdu_airflow.operators.update_status import UpdateStatusOperator +from osdu_ingestion.libs.exceptions import PipelineFailedError from osdu_ingestion.libs.handle_file import FileHandler + +import mock_providers +import requests +from airflow import DAG +from airflow.models import TaskInstance +from file_paths import (MANIFEST_BATCH_WELLBORE_VALID_PATH, MANIFEST_GENERIC_SCHEMA_PATH, + MANIFEST_WELLBORE_VALID_PATH) from mock_responses import MockWorkflowResponse +from osdu_airflow.operators.ensure_manifest_integrity import EnsureManifestIntegrityOperator +from osdu_airflow.operators.mixins.ReceivingContextMixin import ReceivingContextMixin +from osdu_airflow.operators.process_manifest_r3 import (ManifestProcessor, + ProcessManifestOperatorR3, SchemaValidator) +from osdu_airflow.operators.update_status import UpdateStatusOperator +from osdu_airflow.operators.validate_manifest_schema import ValidateManifestSchemaOperator + CustomOperator = TypeVar("CustomOperator") @@ -82,7 +78,6 @@ class TestOperators(object): def test_process_manifest_r3_operator(self, monkeypatch): - @lru_cache() def _get_common_schema(*args, **kwargs): with open(MANIFEST_GENERIC_SCHEMA_PATH) as f: manifest_schema = json.load(f) @@ -157,3 +152,26 @@ class TestOperators(object): """ with pytest.raises(PipelineFailedError): self._test_update_status_operator(monkeypatch, status) + + def test_validate_schema_operator(self, monkeypatch): + + def _get_common_schema(*args, **kwargs): + with open(MANIFEST_GENERIC_SCHEMA_PATH) as f: + manifest_schema = json.load(f) + return manifest_schema + + monkeypatch.setattr(SchemaValidator, "get_schema", _get_common_schema) + monkeypatch.setattr(SchemaValidator, "_validate_against_schema", lambda *args, **kwargs: None) + monkeypatch.setattr(SchemaValidator, "validate_manifest", lambda obj, entities: entities) + task, context = self._create_task(ValidateManifestSchemaOperator) + task.pre_execute(context) + task.execute(context) + + def test_ensure_manifest_integrity(self, monkeypatch): + monkeypatch.setattr(FileHandler, "upload_file", + lambda *args, **kwargs: "test") + monkeypatch.setattr(ReceivingContextMixin, "_get_previously_skipped_entities", + lambda *args, **kwargs: []) + task, context = self._create_task(EnsureManifestIntegrityOperator) + task.pre_execute(context) + task.execute(context) diff --git a/osdu_airflow/tests/set_airflow_env.sh b/osdu_airflow/tests/set_airflow_env.sh deleted file mode 100755 index 07c618a5c62ee22f6bebb5bc512674df23411328..0000000000000000000000000000000000000000 --- a/osdu_airflow/tests/set_airflow_env.sh +++ /dev/null @@ -1,84 +0,0 @@ -# Copyright 2020 Google LLC -# Copyright 2020 EPAM Systems -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -pip uninstall osdu-api -y -pip install --upgrade google-api-python-client -pip install dataclasses -pip install jsonschema -pip install google -pip install toposort -pip install google-cloud-storage -pip install deepdiff -pip install azure-identity -pip install azure-keyvault-secrets -pip install msal -pip install python-keycloak -pip install osdu-api~=0.12.0 --extra-index-url https://community.opengroup.org/api/v4/projects/148/packages/pypi/simple -pip install osdu-ingestion~=0.12.0 --extra-index-url https://community.opengroup.org/api/v4/projects/823/packages/pypi/simple - -export WORKFLOW_URL="http://127.0.0.1:5000" -export UPDATE_STATUS_URL="http://127.0.0.1:5000/wf/us" -export STORAGE_URL="http://127.0.0.1:5000/st" -export SEARCH_URL="http://127.0.0.1:5000/sr/qr" -export LOCALHOST="http://127.0.0.1:5000" -export SEARCH_CONN_ID="http://127.0.0.1:5000" -export WORKFLOW_CONN_ID="http://127.0.0.1:5000" -export DATALOAD_CONFIG_PATH="/usr/local/airflow/dags/configs/dataload.ini" -export SA_FILE_PATH="test" - -airflow initdb > /dev/null 2>&1 - -# exclude testing DAGS -sed -i 's/load_examples = True/load_examples = False/' /usr/local/airflow/airflow.cfg -# turn on all dags -sed -i 's/dags_are_paused_at_creation = True/dags_are_paused_at_creation = False/' /usr/local/airflow/airflow.cfg - -airflow variables -s core__service__storage__url $STORAGE_URL -airflow variables -s core__provider gcp -airflow variables -s core__service__workflow__host $WORKFLOW_URL -airflow variables -s core__service__file__host $LOCALHOST -airflow variables -s core__service__workflow__url $UPDATE_STATUS_URL -airflow variables -s core__service__search__url $SEARCH_URL -airflow variables -s core__service__schema__url $LOCALHOST -airflow variables -s core__config__dataload_config_path $DATALOAD_CONFIG_PATH -airflow variables -s core__auth__access_token test -airflow variables -s core__ingestion__batch_count 3 - -airflow connections -a --conn_id workflow --conn_uri $WORKFLOW_CONN_ID -airflow connections -a --conn_id google_cloud_storage --conn_uri $LOCALHOST - -mkdir -p /usr/local/airflow/dags/ -mkdir -p /usr/local/airflow/plugins/ - -# Copying folders as the tests expect the dags/operator files to follow existing structure -# cp -r src/osdu_dags/osdu_manifest/hooks /usr/local/airflow/plugins/ -# cp -r src/osdu_dags/osdu_manifest/operators /usr/local/airflow/plugins/ - -# cp -r src/osdu_dags/osdu_manifest/configs /usr/local/airflow/dags/ -# cp -r src/osdu_dags/osdu_manifest/libs /usr/local/airflow/dags/ -# cp -r src/osdu_dags/osdu_manifest/providers /usr/local/airflow/dags/ -# cp -r src/osdu_dags/*.py /usr/local/airflow/dags/ - -# Changing import statements for existing tests to work -# for f in $(find /usr/local/airflow/dags -name '*.py'); do sed -i 's/osdu_manifest.//g' $f; done -# for f in $(find /usr/local/airflow/plugins -name '*.py'); do sed -i 's/osdu_manifest.//g' $f; done - -# cp -r tests/end-to-end-tests/mock-external-apis /mock-server -# cp -r tests/end-to-end-tests/mock-data /mock-server/mock-data - -# cp tests/end-to-end-tests/{test-osdu-ingest-r2-success.sh,test-osdu-ingest-r2-fail.sh} /mock-server/ -# cp tests/end-to-end-tests/osdu_api_config.yaml /mock-server/ -cp tests/*.py /mock-server/ - -chmod +x /mock-server/{test-osdu-ingest-r2-success.sh,test-osdu-ingest-r2-fail.sh} diff --git a/osdu_airflow/tests/unit_tests.sh b/osdu_airflow/tests/unit_tests.sh index 17b5bf03365a089716c42c1f09c1762929200afe..5684d49c559f50a5ae70961961983a5557fb3b58 100755 --- a/osdu_airflow/tests/unit_tests.sh +++ b/osdu_airflow/tests/unit_tests.sh @@ -1,12 +1,10 @@ -pip uninstall enum34 -y -pip install pytest -pip install pytest-mock -pip install responses -pip install strict-rfc3339 -pip install --upgrade google-api-python-client -chmod +x ./osdu_airflow/tests/set_airflow_env.sh -export AIRFLOW_SRC_DIR="/usr/local/airflow/" +#!/bin/sh + export CLOUD_PROVIDER="provider_test" -./osdu_airflow/tests/./set_airflow_env.sh > /dev/null 2>&1 -pytest -s || EXIT_CODE=$? + +airflow db init + +airflow variables import "`dirname $0`/airflow_var.json" + +python -m pytest || EXIT_CODE=$? exit $EXIT_CODE diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000000000000000000000000000000000000..3036c26015152057f56390d6224bdbf5c03d57f8 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,12 @@ +pytest +pytest-mock +responses + +requests==2.25.1 +tenacity==6.2.0 + +--extra-index-url https://community.opengroup.org/api/v4/projects/148/packages/pypi/simple +osdu-api~=0.12.0 + +--extra-index-url https://community.opengroup.org/api/v4/projects/823/packages/pypi/simple +osdu-ingestion~=0.12.0