Commit 0697288a authored by Yan Sushchynski (EPAM)'s avatar Yan Sushchynski (EPAM)
Browse files

GONRG-3452: Move Ingestion logic from Python SDK

parent 7faff778
Pipeline #71324 passed with stages
in 2 minutes and 52 seconds
......@@ -20,10 +20,10 @@ import enum
import logging
from airflow.models import BaseOperator, Variable
from osdu_api.libs.context import Context
from osdu_api.libs.exceptions import PipelineFailedError
from osdu_api.libs.refresh_token import AirflowTokenRefresher
from osdu_api.libs.update_status import UpdateStatus
from osdu_ingestion.libs.context import Context
from osdu_ingestion.libs.exceptions import PipelineFailedError
from osdu_ingestion.libs.refresh_token import AirflowTokenRefresher
from osdu_ingestion.libs.update_status import UpdateStatus
logger = logging.getLogger()
......
......@@ -17,10 +17,10 @@
import logging
from airflow.models import BaseOperator, Variable
from osdu_api.libs.context import Context
from osdu_api.libs.refresh_token import AirflowTokenRefresher
from osdu_api.libs.validation.validate_file_source import FileSourceValidator
from osdu_api.libs.validation.validate_referential_integrity import ManifestIntegrity
from osdu_ingestion.libs.context import Context
from osdu_ingestion.libs.refresh_token import AirflowTokenRefresher
from osdu_ingestion.libs.validation.validate_file_source import FileSourceValidator
from osdu_ingestion.libs.validation.validate_referential_integrity import ManifestIntegrity
from osdu_airflow.backward_compatibility.airflow_utils import apply_defaults
from osdu_airflow.operators.mixins.ReceivingContextMixin import ReceivingContextMixin
......
......@@ -15,7 +15,7 @@
import logging
from osdu_api.libs.types import ManifestType
from osdu_ingestion.libs.types import ManifestType
class ReceivingContextMixin:
......
......@@ -30,8 +30,8 @@ from urllib.error import HTTPError
import requests
import tenacity
from airflow.models import BaseOperator, Variable
from osdu_api.libs.auth.authorization import authorize
from osdu_api.libs.refresh_token import AirflowTokenRefresher
from osdu_ingestion.libs.auth.authorization import authorize
from osdu_ingestion.libs.refresh_token import AirflowTokenRefresher
config = configparser.RawConfigParser()
config.read(Variable.get("core__config__dataload_config_path"))
......
......@@ -24,20 +24,21 @@ from typing import List, Tuple
from airflow.models import BaseOperator, Variable
from jsonschema import SchemaError
from osdu_api.libs.constants import DATA_TYPES_WITH_SURROGATE_KEYS, SURROGATE_KEYS_PATHS
from osdu_api.libs.context import Context
from osdu_api.libs.exceptions import (EmptyManifestError, GenericManifestSchemaError,
GetSchemaError, NotOSDUSchemaFormatError, UploadFileError)
from osdu_api.libs.handle_file import FileHandler
from osdu_api.libs.process_manifest_r3 import ManifestProcessor
from osdu_api.libs.processors.single_manifest_processor import SingleManifestProcessor
from osdu_api.libs.refresh_token import AirflowTokenRefresher
from osdu_api.libs.search_client import SearchClient
from osdu_api.libs.source_file_check import SourceFileChecker
from osdu_api.libs.types import ManifestType
from osdu_api.libs.validation.validate_file_source import FileSourceValidator
from osdu_api.libs.validation.validate_referential_integrity import ManifestIntegrity
from osdu_api.libs.validation.validate_schema import SchemaValidator
from osdu_ingestion.libs.constants import DATA_TYPES_WITH_SURROGATE_KEYS, SURROGATE_KEYS_PATHS
from osdu_ingestion.libs.context import Context
from osdu_ingestion.libs.exceptions import (EmptyManifestError, GenericManifestSchemaError,
GetSchemaError, NotOSDUSchemaFormatError,
UploadFileError)
from osdu_ingestion.libs.handle_file import FileHandler
from osdu_ingestion.libs.process_manifest_r3 import ManifestProcessor
from osdu_ingestion.libs.processors.single_manifest_processor import SingleManifestProcessor
from osdu_ingestion.libs.refresh_token import AirflowTokenRefresher
from osdu_ingestion.libs.search_client import SearchClient
from osdu_ingestion.libs.source_file_check import SourceFileChecker
from osdu_ingestion.libs.types import ManifestType
from osdu_ingestion.libs.validation.validate_file_source import FileSourceValidator
from osdu_ingestion.libs.validation.validate_referential_integrity import ManifestIntegrity
from osdu_ingestion.libs.validation.validate_schema import SchemaValidator
from requests import HTTPError
from osdu_airflow.backward_compatibility.airflow_utils import apply_defaults
......
......@@ -16,9 +16,9 @@
import logging
from airflow.models import BaseOperator, Variable
from osdu_api.libs.context import Context
from osdu_api.libs.refresh_token import AirflowTokenRefresher
from osdu_api.libs.search_record_ids import SearchId
from osdu_ingestion.libs.context import Context
from osdu_ingestion.libs.refresh_token import AirflowTokenRefresher
from osdu_ingestion.libs.search_record_ids import SearchId
logger = logging.getLogger()
......
......@@ -21,10 +21,10 @@ import logging
from typing import Tuple
from airflow.models import BaseOperator, Variable
from osdu_api.libs.context import Context
from osdu_api.libs.exceptions import PipelineFailedError
from osdu_api.libs.refresh_token import AirflowTokenRefresher
from osdu_api.libs.update_status import UpdateStatus
from osdu_ingestion.libs.context import Context
from osdu_ingestion.libs.exceptions import PipelineFailedError
from osdu_ingestion.libs.refresh_token import AirflowTokenRefresher
from osdu_ingestion.libs.update_status import UpdateStatus
from osdu_airflow.backward_compatibility.airflow_utils import apply_defaults
......
......@@ -20,11 +20,11 @@ Validate Manifest against R3 schemas operator.
import logging
from airflow.models import BaseOperator, Variable
from osdu_api.libs.constants import DATA_TYPES_WITH_SURROGATE_KEYS, SURROGATE_KEYS_PATHS
from osdu_api.libs.context import Context
from osdu_api.libs.exceptions import EmptyManifestError, GenericManifestSchemaError
from osdu_api.libs.refresh_token import AirflowTokenRefresher
from osdu_api.libs.validation.validate_schema import SchemaValidator
from osdu_ingestion.libs.constants import DATA_TYPES_WITH_SURROGATE_KEYS, SURROGATE_KEYS_PATHS
from osdu_ingestion.libs.context import Context
from osdu_ingestion.libs.exceptions import EmptyManifestError, GenericManifestSchemaError
from osdu_ingestion.libs.refresh_token import AirflowTokenRefresher
from osdu_ingestion.libs.validation.validate_schema import SchemaValidator
from osdu_airflow.backward_compatibility.airflow_utils import apply_defaults
from osdu_airflow.operators.mixins.ReceivingContextMixin import ReceivingContextMixin
......
......@@ -27,7 +27,7 @@ sys.path.append(f"{os.getenv('AIRFLOW_SRC_DIR')}/plugins")
sys.path.append(f"{os.getenv('AIRFLOW_SRC_DIR')}/dags")
sys.path.insert(0, './')
from osdu_api.libs.exceptions import PipelineFailedError
from osdu_ingestion.libs.exceptions import PipelineFailedError
import pytest
import requests
import mock_providers
......@@ -39,7 +39,7 @@ from file_paths import (
from osdu_airflow.operators.process_manifest_r3 import ProcessManifestOperatorR3, SchemaValidator, \
ManifestProcessor
from osdu_airflow.operators.update_status import UpdateStatusOperator
from osdu_api.libs.handle_file import FileHandler
from osdu_ingestion.libs.handle_file import FileHandler
from mock_responses import MockWorkflowResponse
CustomOperator = TypeVar("CustomOperator")
......
......@@ -24,7 +24,8 @@ pip install azure-identity
pip install azure-keyvault-secrets
pip install msal
pip install python-keycloak
pip install osdu-api==0.10.1.dev0+92014f64 --extra-index-url https://community.opengroup.org/api/v4/projects/148/packages/pypi/simple
pip install osdu-api==0.12.0.dev213 --extra-index-url https://community.opengroup.org/api/v4/projects/148/packages/pypi/simple
pip install osdu-ingestion==0.12.0.dev20 --extra-index-url https://community.opengroup.org/api/v4/projects/823/packages/pypi/simple
export WORKFLOW_URL="http://127.0.0.1:5000"
export UPDATE_STATUS_URL="http://127.0.0.1:5000/wf/us"
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment