Merge branch 'GONRG-3452_Move_ingestion_logic_from_SD' into 'master'

GONRG-3452: Move Ingestion logic from Python SDK

See merge request !6

VERSION

-0.0.1
+0.12.0

osdu_airflow/operators/deprecated/update_status.py

@@ -20,10 +20,10 @@ import enum
 import logging
 
 from airflow.models import BaseOperator, Variable
-from osdu_api.libs.context import Context
-from osdu_api.libs.exceptions import PipelineFailedError
-from osdu_api.libs.refresh_token import AirflowTokenRefresher
-from osdu_api.libs.update_status import UpdateStatus
+from osdu_ingestion.libs.context import Context
+from osdu_ingestion.libs.exceptions import PipelineFailedError
+from osdu_ingestion.libs.refresh_token import AirflowTokenRefresher
+from osdu_ingestion.libs.update_status import UpdateStatus
 
 logger = logging.getLogger()
 

osdu_airflow/operators/ensure_manifest_integrity.py

@@ -17,10 +17,10 @@
 import logging
 
 from airflow.models import BaseOperator, Variable
-from osdu_api.libs.context import Context
-from osdu_api.libs.refresh_token import AirflowTokenRefresher
-from osdu_api.libs.validation.validate_file_source import FileSourceValidator
-from osdu_api.libs.validation.validate_referential_integrity import ManifestIntegrity
+from osdu_ingestion.libs.context import Context
+from osdu_ingestion.libs.refresh_token import AirflowTokenRefresher
+from osdu_ingestion.libs.validation.validate_file_source import FileSourceValidator
+from osdu_ingestion.libs.validation.validate_referential_integrity import ManifestIntegrity
 
 from osdu_airflow.backward_compatibility.airflow_utils import apply_defaults
 from osdu_airflow.operators.mixins.ReceivingContextMixin import ReceivingContextMixin

osdu_airflow/operators/mixins/ReceivingContextMixin.py

@@ -15,7 +15,7 @@
 
 import logging
 
-from osdu_api.libs.types import ManifestType
+from osdu_ingestion.libs.types import ManifestType
 
 
 class ReceivingContextMixin:

osdu_airflow/operators/process_manifest_r2.py

@@ -30,8 +30,8 @@ from urllib.error import HTTPError
 import requests
 import tenacity
 from airflow.models import BaseOperator, Variable
-from osdu_api.libs.auth.authorization import authorize
-from osdu_api.libs.refresh_token import AirflowTokenRefresher
+from osdu_ingestion.libs.auth.authorization import authorize
+from osdu_ingestion.libs.refresh_token import AirflowTokenRefresher
 
 config = configparser.RawConfigParser()
 config.read(Variable.get("core__config__dataload_config_path"))

osdu_airflow/operators/process_manifest_r3.py

@@ -24,20 +24,21 @@ from typing import List, Tuple
 
 from airflow.models import BaseOperator, Variable
 from jsonschema import SchemaError
-from osdu_api.libs.constants import DATA_TYPES_WITH_SURROGATE_KEYS, SURROGATE_KEYS_PATHS
-from osdu_api.libs.context import Context
-from osdu_api.libs.exceptions import (EmptyManifestError, GenericManifestSchemaError,
-                                      GetSchemaError, NotOSDUSchemaFormatError, UploadFileError)
-from osdu_api.libs.handle_file import FileHandler
-from osdu_api.libs.process_manifest_r3 import ManifestProcessor
-from osdu_api.libs.processors.single_manifest_processor import SingleManifestProcessor
-from osdu_api.libs.refresh_token import AirflowTokenRefresher
-from osdu_api.libs.search_client import SearchClient
-from osdu_api.libs.source_file_check import SourceFileChecker
-from osdu_api.libs.types import ManifestType
-from osdu_api.libs.validation.validate_file_source import FileSourceValidator
-from osdu_api.libs.validation.validate_referential_integrity import ManifestIntegrity
-from osdu_api.libs.validation.validate_schema import SchemaValidator
+from osdu_ingestion.libs.constants import DATA_TYPES_WITH_SURROGATE_KEYS, SURROGATE_KEYS_PATHS
+from osdu_ingestion.libs.context import Context
+from osdu_ingestion.libs.exceptions import (EmptyManifestError, GenericManifestSchemaError,
+                                            GetSchemaError, NotOSDUSchemaFormatError,
+                                            UploadFileError)
+from osdu_ingestion.libs.handle_file import FileHandler
+from osdu_ingestion.libs.process_manifest_r3 import ManifestProcessor
+from osdu_ingestion.libs.processors.single_manifest_processor import SingleManifestProcessor
+from osdu_ingestion.libs.refresh_token import AirflowTokenRefresher
+from osdu_ingestion.libs.search_client import SearchClient
+from osdu_ingestion.libs.source_file_check import SourceFileChecker
+from osdu_ingestion.libs.types import ManifestType
+from osdu_ingestion.libs.validation.validate_file_source import FileSourceValidator
+from osdu_ingestion.libs.validation.validate_referential_integrity import ManifestIntegrity
+from osdu_ingestion.libs.validation.validate_schema import SchemaValidator
 from requests import HTTPError
 
 from osdu_airflow.backward_compatibility.airflow_utils import apply_defaults

osdu_airflow/operators/search_record_id.py

@@ -16,9 +16,9 @@
 import logging
 
 from airflow.models import BaseOperator, Variable
-from osdu_api.libs.context import Context
-from osdu_api.libs.refresh_token import AirflowTokenRefresher
-from osdu_api.libs.search_record_ids import SearchId
+from osdu_ingestion.libs.context import Context
+from osdu_ingestion.libs.refresh_token import AirflowTokenRefresher
+from osdu_ingestion.libs.search_record_ids import SearchId
 
 logger = logging.getLogger()
 

osdu_airflow/operators/update_status.py

@@ -21,10 +21,10 @@ import logging
 from typing import Tuple
 
 from airflow.models import BaseOperator, Variable
-from osdu_api.libs.context import Context
-from osdu_api.libs.exceptions import PipelineFailedError
-from osdu_api.libs.refresh_token import AirflowTokenRefresher
-from osdu_api.libs.update_status import UpdateStatus
+from osdu_ingestion.libs.context import Context
+from osdu_ingestion.libs.exceptions import PipelineFailedError
+from osdu_ingestion.libs.refresh_token import AirflowTokenRefresher
+from osdu_ingestion.libs.update_status import UpdateStatus
 
 from osdu_airflow.backward_compatibility.airflow_utils import apply_defaults
 

osdu_airflow/operators/validate_manifest_schema.py

@@ -20,11 +20,11 @@ Validate Manifest against R3 schemas operator.
 import logging
 
 from airflow.models import BaseOperator, Variable
-from osdu_api.libs.constants import DATA_TYPES_WITH_SURROGATE_KEYS, SURROGATE_KEYS_PATHS
-from osdu_api.libs.context import Context
-from osdu_api.libs.exceptions import EmptyManifestError, GenericManifestSchemaError
-from osdu_api.libs.refresh_token import AirflowTokenRefresher
-from osdu_api.libs.validation.validate_schema import SchemaValidator
+from osdu_ingestion.libs.constants import DATA_TYPES_WITH_SURROGATE_KEYS, SURROGATE_KEYS_PATHS
+from osdu_ingestion.libs.context import Context
+from osdu_ingestion.libs.exceptions import EmptyManifestError, GenericManifestSchemaError
+from osdu_ingestion.libs.refresh_token import AirflowTokenRefresher
+from osdu_ingestion.libs.validation.validate_schema import SchemaValidator
 
 from osdu_airflow.backward_compatibility.airflow_utils import apply_defaults
 from osdu_airflow.operators.mixins.ReceivingContextMixin import ReceivingContextMixin

osdu_airflow/tests/plugin-unit-tests/test_operators_r3.py

@@ -27,7 +27,7 @@ sys.path.append(f"{os.getenv('AIRFLOW_SRC_DIR')}/plugins")
 sys.path.append(f"{os.getenv('AIRFLOW_SRC_DIR')}/dags")
 sys.path.insert(0, './')
 
-from osdu_api.libs.exceptions import PipelineFailedError
+from osdu_ingestion.libs.exceptions import PipelineFailedError
 import pytest
 import requests
 import mock_providers
@@ -39,7 +39,7 @@ from file_paths import (
 from osdu_airflow.operators.process_manifest_r3 import ProcessManifestOperatorR3, SchemaValidator, \
     ManifestProcessor
 from osdu_airflow.operators.update_status import UpdateStatusOperator
-from osdu_api.libs.handle_file import FileHandler
+from osdu_ingestion.libs.handle_file import FileHandler
 from mock_responses import MockWorkflowResponse
 
 CustomOperator = TypeVar("CustomOperator")

osdu_airflow/tests/set_airflow_env.sh

@@ -24,7 +24,8 @@ pip install azure-identity
 pip install azure-keyvault-secrets
 pip install msal
 pip install python-keycloak
-pip install osdu-api==0.10.1.dev0+92014f64 --extra-index-url https://community.opengroup.org/api/v4/projects/148/packages/pypi/simple
+pip install osdu-api==0.12.0.dev213 --extra-index-url https://community.opengroup.org/api/v4/projects/148/packages/pypi/simple
+pip install osdu-ingestion==0.12.0.dev20 --extra-index-url https://community.opengroup.org/api/v4/projects/823/packages/pypi/simple
 
 export WORKFLOW_URL="http://127.0.0.1:5000"
 export UPDATE_STATUS_URL="http://127.0.0.1:5000/wf/us"