Commit 1ce6bba5 authored by Siarhei Khaletski (EPAM)'s avatar Siarhei Khaletski (EPAM) 🚩
Browse files

Merge branch 'GONRG-3299_Fix_unit_tests' into 'master'

GONRG-3299: Unit tests for AF1 and AF2

See merge request !8
parents 8cf621e5 306f9531
Pipeline #75299 passed with stages
in 1 minute and 5 seconds
**/.idea
**/.DS_Store
**/.vscode
# Byte-compiled / optimized / DLL files
**/__pycache__/
......
......@@ -27,12 +27,24 @@ isort:
- isort -c -v osdu_airflow/*/*.py || EXIT_CODE=$?
- exit ${EXIT_CODE}
unit_tests:
unit_tests_airflow_v1:
stage: unit_tests
image: eu.gcr.io/osdu-cicd-epam/airflow-python-dags/airflow-python-dags:latest
image:
name: apache/airflow:1.10.14-python3.6
entrypoint: [""]
script:
- chmod +x ./osdu_airflow/tests/unit_tests.sh
- ./osdu_airflow/tests/./unit_tests.sh || EXIT_CODE=$?
- pip install -r requirements-dev.txt
- sh -c "./osdu_airflow/tests/./unit_tests.sh" || EXIT_CODE=$?
- exit ${EXIT_CODE}
unit_tests_airflow_v2:
stage: unit_tests
image:
name: apache/airflow:2.1.2-python3.8
entrypoint: [""]
script:
- pip install -r requirements-dev.txt
- sh -c "./osdu_airflow/tests/./unit_tests.sh" || EXIT_CODE=$?
- exit ${EXIT_CODE}
deploy:
......
{
"core__service__storage__url": "https://test",
"core__service__workflow__host": "https://test",
"core__service__file__host": "htpps://test",
"core__service__workflow__url": "https://test",
"core__service__search__url": "https://test",
"core__service__schema__url": "https://test",
"core__config__dataload_config_path": "https://test",
"core__auth__access_token": "test",
"core__ingestion__batch_count": 3
}
......@@ -16,31 +16,27 @@
import http
import json
import os
import sys
from datetime import datetime
from typing import TypeVar, ClassVar
from airflow import DAG
from airflow.models import TaskInstance
sys.path.append(f"{os.getenv('AIRFLOW_SRC_DIR')}/plugins")
sys.path.append(f"{os.getenv('AIRFLOW_SRC_DIR')}/dags")
sys.path.insert(0, './')
from typing import ClassVar, TypeVar
from osdu_ingestion.libs.exceptions import PipelineFailedError
import pytest
import requests
import mock_providers
from functools import lru_cache
from file_paths import (
MANIFEST_WELLBORE_VALID_PATH,
SEARCH_VALID_RESPONSE_PATH, MANIFEST_GENERIC_SCHEMA_PATH, MANIFEST_BATCH_WELLBORE_VALID_PATH)
from osdu_airflow.operators.process_manifest_r3 import ProcessManifestOperatorR3, SchemaValidator, \
ManifestProcessor
from osdu_airflow.operators.update_status import UpdateStatusOperator
from osdu_ingestion.libs.exceptions import PipelineFailedError
from osdu_ingestion.libs.handle_file import FileHandler
import mock_providers
import requests
from airflow import DAG
from airflow.models import TaskInstance
from file_paths import (MANIFEST_BATCH_WELLBORE_VALID_PATH, MANIFEST_GENERIC_SCHEMA_PATH,
MANIFEST_WELLBORE_VALID_PATH)
from mock_responses import MockWorkflowResponse
from osdu_airflow.operators.ensure_manifest_integrity import EnsureManifestIntegrityOperator
from osdu_airflow.operators.mixins.ReceivingContextMixin import ReceivingContextMixin
from osdu_airflow.operators.process_manifest_r3 import (ManifestProcessor,
ProcessManifestOperatorR3, SchemaValidator)
from osdu_airflow.operators.update_status import UpdateStatusOperator
from osdu_airflow.operators.validate_manifest_schema import ValidateManifestSchemaOperator
CustomOperator = TypeVar("CustomOperator")
......@@ -82,7 +78,6 @@ class TestOperators(object):
def test_process_manifest_r3_operator(self, monkeypatch):
@lru_cache()
def _get_common_schema(*args, **kwargs):
with open(MANIFEST_GENERIC_SCHEMA_PATH) as f:
manifest_schema = json.load(f)
......@@ -157,3 +152,26 @@ class TestOperators(object):
"""
with pytest.raises(PipelineFailedError):
self._test_update_status_operator(monkeypatch, status)
def test_validate_schema_operator(self, monkeypatch):
def _get_common_schema(*args, **kwargs):
with open(MANIFEST_GENERIC_SCHEMA_PATH) as f:
manifest_schema = json.load(f)
return manifest_schema
monkeypatch.setattr(SchemaValidator, "get_schema", _get_common_schema)
monkeypatch.setattr(SchemaValidator, "_validate_against_schema", lambda *args, **kwargs: None)
monkeypatch.setattr(SchemaValidator, "validate_manifest", lambda obj, entities: entities)
task, context = self._create_task(ValidateManifestSchemaOperator)
task.pre_execute(context)
task.execute(context)
def test_ensure_manifest_integrity(self, monkeypatch):
monkeypatch.setattr(FileHandler, "upload_file",
lambda *args, **kwargs: "test")
monkeypatch.setattr(ReceivingContextMixin, "_get_previously_skipped_entities",
lambda *args, **kwargs: [])
task, context = self._create_task(EnsureManifestIntegrityOperator)
task.pre_execute(context)
task.execute(context)
# Copyright 2020 Google LLC
# Copyright 2020 EPAM Systems
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
pip uninstall osdu-api -y
pip install --upgrade google-api-python-client
pip install dataclasses
pip install jsonschema
pip install google
pip install toposort
pip install google-cloud-storage
pip install deepdiff
pip install azure-identity
pip install azure-keyvault-secrets
pip install msal
pip install python-keycloak
pip install osdu-api~=0.12.0 --extra-index-url https://community.opengroup.org/api/v4/projects/148/packages/pypi/simple
pip install osdu-ingestion~=0.12.0 --extra-index-url https://community.opengroup.org/api/v4/projects/823/packages/pypi/simple
export WORKFLOW_URL="http://127.0.0.1:5000"
export UPDATE_STATUS_URL="http://127.0.0.1:5000/wf/us"
export STORAGE_URL="http://127.0.0.1:5000/st"
export SEARCH_URL="http://127.0.0.1:5000/sr/qr"
export LOCALHOST="http://127.0.0.1:5000"
export SEARCH_CONN_ID="http://127.0.0.1:5000"
export WORKFLOW_CONN_ID="http://127.0.0.1:5000"
export DATALOAD_CONFIG_PATH="/usr/local/airflow/dags/configs/dataload.ini"
export SA_FILE_PATH="test"
airflow initdb > /dev/null 2>&1
# exclude testing DAGS
sed -i 's/load_examples = True/load_examples = False/' /usr/local/airflow/airflow.cfg
# turn on all dags
sed -i 's/dags_are_paused_at_creation = True/dags_are_paused_at_creation = False/' /usr/local/airflow/airflow.cfg
airflow variables -s core__service__storage__url $STORAGE_URL
airflow variables -s core__provider gcp
airflow variables -s core__service__workflow__host $WORKFLOW_URL
airflow variables -s core__service__file__host $LOCALHOST
airflow variables -s core__service__workflow__url $UPDATE_STATUS_URL
airflow variables -s core__service__search__url $SEARCH_URL
airflow variables -s core__service__schema__url $LOCALHOST
airflow variables -s core__config__dataload_config_path $DATALOAD_CONFIG_PATH
airflow variables -s core__auth__access_token test
airflow variables -s core__ingestion__batch_count 3
airflow connections -a --conn_id workflow --conn_uri $WORKFLOW_CONN_ID
airflow connections -a --conn_id google_cloud_storage --conn_uri $LOCALHOST
mkdir -p /usr/local/airflow/dags/
mkdir -p /usr/local/airflow/plugins/
# Copying folders as the tests expect the dags/operator files to follow existing structure
# cp -r src/osdu_dags/osdu_manifest/hooks /usr/local/airflow/plugins/
# cp -r src/osdu_dags/osdu_manifest/operators /usr/local/airflow/plugins/
# cp -r src/osdu_dags/osdu_manifest/configs /usr/local/airflow/dags/
# cp -r src/osdu_dags/osdu_manifest/libs /usr/local/airflow/dags/
# cp -r src/osdu_dags/osdu_manifest/providers /usr/local/airflow/dags/
# cp -r src/osdu_dags/*.py /usr/local/airflow/dags/
# Changing import statements for existing tests to work
# for f in $(find /usr/local/airflow/dags -name '*.py'); do sed -i 's/osdu_manifest.//g' $f; done
# for f in $(find /usr/local/airflow/plugins -name '*.py'); do sed -i 's/osdu_manifest.//g' $f; done
# cp -r tests/end-to-end-tests/mock-external-apis /mock-server
# cp -r tests/end-to-end-tests/mock-data /mock-server/mock-data
# cp tests/end-to-end-tests/{test-osdu-ingest-r2-success.sh,test-osdu-ingest-r2-fail.sh} /mock-server/
# cp tests/end-to-end-tests/osdu_api_config.yaml /mock-server/
cp tests/*.py /mock-server/
chmod +x /mock-server/{test-osdu-ingest-r2-success.sh,test-osdu-ingest-r2-fail.sh}
pip uninstall enum34 -y
pip install pytest
pip install pytest-mock
pip install responses
pip install strict-rfc3339
pip install --upgrade google-api-python-client
chmod +x ./osdu_airflow/tests/set_airflow_env.sh
export AIRFLOW_SRC_DIR="/usr/local/airflow/"
#!/bin/sh
export CLOUD_PROVIDER="provider_test"
./osdu_airflow/tests/./set_airflow_env.sh > /dev/null 2>&1
pytest -s || EXIT_CODE=$?
airflow db init
airflow variables import "`dirname $0`/airflow_var.json"
python -m pytest || EXIT_CODE=$?
exit $EXIT_CODE
pytest
pytest-mock
responses
requests==2.25.1
tenacity==6.2.0
--extra-index-url https://community.opengroup.org/api/v4/projects/148/packages/pypi/simple
osdu-api~=0.12.0
--extra-index-url https://community.opengroup.org/api/v4/projects/823/packages/pypi/simple
osdu-ingestion~=0.12.0
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment