From 6b7a099076f4cf18edab4ca28af1f06f8db266a0 Mon Sep 17 00:00:00 2001
From: Dmitrii <dmitrii_valuiskii@epam.com>
Date: Tue, 19 Jan 2021 11:35:27 +0300
Subject: [PATCH] GONRG-1684: add support new R3 manifest structure
---
src/dags/libs/handle_file.py | 20 +-
src/dags/libs/process_manifest_r3.py | 139 ++---
src/dags/libs/traverse_manifest.py | 90 +++
src/dags/libs/validate_schema.py | 161 +++--
src/plugins/operators/process_manifest_r3.py | 23 +-
.../data/invalid/NotOsduFormat.json | 6 +-
.../data/invalid/TraversalEmptyManifest.json | 2 +
.../TraversalNotOSDUFormatManifest.json | 8 +
.../data/manifests/Manifest.1.0.0.json | 559 ++++++++++++++++++
.../data/manifests/schema_Manifest.1.0.0.json | 55 ++
.../data/master/Wellbore.0.3.0.json | 107 +---
.../schema_GenericMasterData.1.0.0.json | 96 +++
.../data/master/schema_TestMaster.json | 13 +
.../data/master/schema_Wellbore.3.0.0.json | 6 +-
.../data/master/traversal_Wellbore.0.3.0.json | 41 ++
.../data/workProduct/SeismicTraceData.json | 19 +-
.../workProduct/record_SeismicTraceData.json | 8 +-
.../data/workProduct/schema_File.1.0.0.json | 6 +-
.../schema_SeismicTraceData.1.0.0.json | 6 +-
.../workProduct/schema_WorkProduct.1.0.0.json | 6 +-
.../traversal_SeismicTraceData.1.0.0.json | 420 +++++++++++++
tests/plugin-unit-tests/file_paths.py | 8 +
tests/plugin-unit-tests/test_file_handler.py | 2 +-
.../test_manifest_processor_r3.py | 106 ++--
.../test_manifest_traversal.py | 82 +++
tests/plugin-unit-tests/test_operators_r3.py | 14 +-
.../test_schema_validator_r3.py | 152 +++--
27 files changed, 1763 insertions(+), 392 deletions(-)
create mode 100644 src/dags/libs/traverse_manifest.py
create mode 100644 tests/plugin-unit-tests/data/invalid/TraversalEmptyManifest.json
create mode 100644 tests/plugin-unit-tests/data/invalid/TraversalNotOSDUFormatManifest.json
create mode 100644 tests/plugin-unit-tests/data/manifests/Manifest.1.0.0.json
create mode 100644 tests/plugin-unit-tests/data/manifests/schema_Manifest.1.0.0.json
create mode 100644 tests/plugin-unit-tests/data/master/schema_GenericMasterData.1.0.0.json
create mode 100644 tests/plugin-unit-tests/data/master/schema_TestMaster.json
create mode 100644 tests/plugin-unit-tests/data/master/traversal_Wellbore.0.3.0.json
create mode 100644 tests/plugin-unit-tests/data/workProduct/traversal_SeismicTraceData.1.0.0.json
create mode 100644 tests/plugin-unit-tests/test_manifest_traversal.py
diff --git a/src/dags/libs/handle_file.py b/src/dags/libs/handle_file.py
index f1d81c0..c03ac40 100644
--- a/src/dags/libs/handle_file.py
+++ b/src/dags/libs/handle_file.py
@@ -19,6 +19,7 @@ import dataclasses
import io
import json
import logging
+import uuid
from abc import ABC, abstractmethod
from typing import List, Tuple, TypeVar
from urllib.parse import urlparse
@@ -68,7 +69,7 @@ class FileHandler(HeadersMixin, ABC):
:type file_service_url: str
:param token_refresher: Object to refresh tokens
:type token_refresher: TokenRefresher
- :param context: The tenant context data
+ :param context: The tenant context data
:type context: Context
"""
super().__init__(context)
@@ -96,11 +97,10 @@ class FileHandler(HeadersMixin, ABC):
:raises InvalidFileRecordData: When some of the mandatory fields is
missing or empty
"""
- name = file_record_data.get("Name")
endian = file_record_data.get("Endian")
- file_source = file_record_data.get("FileSource")
- if not (name and endian and file_source):
- raise InvalidFileRecordData(f"Mandatory fields: Name-{name} Endian-{endian}"
+ file_source = file_record_data["DatasetProperties"]["FileSourceInfo"].get("FileSource")
+ if not (endian and file_source):
+ raise InvalidFileRecordData(f"Mandatory fields: Endian-{endian}"
f"FileSource-{file_source}")
@tenacity.retry(**RETRY_SETTINGS)
@@ -208,7 +208,7 @@ class FileHandler(HeadersMixin, ABC):
def get_file_staging_location(self, file_source: str) -> str:
"""Retrieve location (full URI) of the file in staging area.
- :param file_source: The FileSource (relative URI) of the file of the form
+ :param file_source: The FileSource (relative URI) of the file of the form
/{folder}/{file_id}
:type file_source: str
:return: Full URI of the location of the file in staging area
@@ -239,6 +239,12 @@ class FileHandler(HeadersMixin, ABC):
:rtype: str
"""
self._verify_file_record_data(file_record["data"])
+ # TODO fix 'name' field processing
+ # Generate file entity name as workaround because file API required this field.
+ if not file_record["data"].get("Name"):
+ file_record["data"]["Name"] = \
+ f"surrogate_name_{file_record['data']['DatasetProperties']['FileSourceInfo']['PreloadFilePath'].split('/')[-1]}"
+ logger.info(f"Generated name: {file_record['data']['Name']}")
logger.info("Sending file record metadata to File service")
endpoint = f"{self._file_service_url}/v1/files/metadata"
response = self._send_post_request(self.request_headers, endpoint, json.dumps(file_record))
@@ -271,7 +277,7 @@ class GCSFileHandler(FileHandler):
:type file_service_url: str
:param token_refresher: Object to refresh tokens
:type token_refresher: TokenRefresher
- :param context: The tenant context data
+ :param context: The tenant context data
:type context: Context
"""
super().__init__(file_service_url, token_refresher, context)
diff --git a/src/dags/libs/process_manifest_r3.py b/src/dags/libs/process_manifest_r3.py
index 227fe80..6b509ab 100644
--- a/src/dags/libs/process_manifest_r3.py
+++ b/src/dags/libs/process_manifest_r3.py
@@ -19,7 +19,7 @@ import copy
import json
import logging
import uuid
-from typing import List
+from typing import List, Tuple
import requests
import tenacity
@@ -56,7 +56,7 @@ class ManifestProcessor(HeadersMixin):
def __init__(
self,
storage_url: str,
- dagrun_conf: dict,
+ manifest_records: List[dict],
file_handler: FileHandler,
source_file_checker: SourceFileChecker,
token_refresher: TokenRefresher,
@@ -81,7 +81,7 @@ class ManifestProcessor(HeadersMixin):
self.file_handler = file_handler
self.source_file_checker = source_file_checker
self.storage_url = storage_url
- self.data_object = copy.deepcopy(dagrun_conf)
+ self.manifest_records = manifest_records
self.context = context
self.token_refresher = token_refresher
@@ -99,10 +99,10 @@ class ManifestProcessor(HeadersMixin):
:return: file record updated if file was properly uploaded
:rtype: dict
"""
- file_path = file_record["data"]["PreLoadFilePath"]
+ file_path = file_record["data"]["DatasetProperties"]["FileSourceInfo"]["PreloadFilePath"]
try:
file_source = self.file_handler.upload_file(file_path)
- file_record["data"]["FileSource"] = file_source
+ file_record["data"]["DatasetProperties"]["FileSourceInfo"]["FileSource"] = file_source
except Exception as e:
logger.error(f"Unhandled exception while uploading {file_path}: {e}")
return file_record
@@ -134,7 +134,7 @@ class ManifestProcessor(HeadersMixin):
record["kind"] = manifest.pop("kind")
record["legal"] = manifest.pop("legal")
record["acl"] = manifest.pop("acl")
- record["data"] = manifest
+ record["data"] = manifest.pop("data")
return record
def _populate_file_storage_record(self, manifest: dict) -> dict:
@@ -166,15 +166,9 @@ class ManifestProcessor(HeadersMixin):
@tenacity.retry(**RETRY_SETTINGS)
@refresh_token()
- def save_record(self, headers: dict, request_data: List[dict]) -> requests.Response:
- """Send request to record storage API.
-
- :param headers: The request headers
- :type headers: dict
- :param request_data: The list of records to save
- :type request_data: List[dict]
- :return: Storage service response
- :rtype: requests.Response
+ def save_record_to_storage(self, headers: dict, request_data: List[dict]) -> requests.Response:
+ """
+ Send request to record storage API.
"""
request_data = json.dumps(request_data)
logger.info("Sending records to Storage service")
@@ -193,46 +187,30 @@ class ManifestProcessor(HeadersMixin):
f"Response content: {reason}.")
return response
- def process_work_product(self, manifest: dict) -> List[dict]:
- """Process WorkProduct manifest.
-
- :param manifest: WorkProduct manifest
- :type manifest: dict
- :return: The list of populated records
- :rtype: List[dict]
+ def save_record_to_file_service(self, file_records: List[dict]) -> List[str]:
"""
- wp = manifest["WorkProduct"]
- records = [self.populate_manifest_storage_record(wp)]
- return records
-
- def process_work_product_components(self, manifest: dict) -> List[dict]:
- """Process WorkProductComponents manifest.
-
- :param manifest: WorkProductComponents manifest
- :type manifest: dict
- :return: The list of populated records
- :rtype: List[dict]
+ Send request to file service API
"""
- records = []
- for wpc in manifest["WorkProductComponents"]:
- record = self.populate_manifest_storage_record(wpc)
- records.append(record)
- return records
-
- def process_work_product_files(self, manifest: dict) -> List[dict]:
- """Process Files manifest.
+ file_record_ids = []
+ for file_record in file_records:
+ # TODO(python-team) implement concurrent request in File Handler service.
+ record_id = self.file_handler.save_file_record(file_record)
+ file_location = self.file_handler.get_file_permanent_location(record_id)
+ # TODO(python-team) implement rollback strategy in case file validation fails.
+ self.source_file_checker.does_file_exist(file_location)
+ file_record_ids.append(record_id)
+ return file_record_ids
- :param manifest: Files manifest
- :type manifest: dict
- :return: The likst of populated records
- :rtype: List[dict]
+ def process_work_product_files(self, file_records: List[dict]) -> List[dict]:
+ """
+ Process list of file records.
"""
records = []
- for file_record in manifest["Files"]:
- if not file_record["data"]["FileSource"]:
+ for file_record in file_records:
+ if not file_record["data"]["DatasetProperties"]["FileSourceInfo"]["FileSource"]:
file_record = self.upload_source_file(file_record)
else:
- file_source = file_record["data"]["FileSource"]
+ file_source = file_record["data"]["DatasetProperties"]["FileSourceInfo"]["FileSource"]
file_location = self.file_handler.get_file_staging_location(file_source)
self.source_file_checker.does_file_exist(file_location)
@@ -240,41 +218,6 @@ class ManifestProcessor(HeadersMixin):
records.append(record)
return records
- def process_work_product_manifest(self, manifest: dict) -> dict:
- """Process WorkProduct, WorkProduct and Files manifest.
-
- :param manifest: WorkProduct, WorkProduct and Files manifest
- :type manifest: dict
- :return: List of populated records
- :rtype: dict
- """
- wp_records = {}
- wp_records["Files"] = self.process_work_product_files(manifest)
- wp_records["WorkProduct"] = self.process_work_product(manifest)
- wp_records["WorkProductComponents"] = self.process_work_product_components(manifest)
- return wp_records
-
- def create_manifest_records(self,
- manifest_records: List[dict],
- file_records: List[dict]) -> None:
- """Process and create a list with every record in manifest list.
-
- :param manifest_records: The list of manifest records
- :type manifest_records: List[dict]
- :param file_records: The list of file records
- :type file_records: List[dict]
- """
- manifests = self.data_object["manifest"]
- for manifest in manifests:
- if "WorkProduct" in manifest:
- wp_records = self.process_work_product_manifest(manifest)
- manifest_records.extend(wp_records["WorkProduct"])
- manifest_records.extend(wp_records["WorkProductComponents"])
- file_records.extend(wp_records["Files"])
- else:
- record = self.populate_manifest_storage_record(manifest)
- manifest_records.append(record)
-
def process_manifest(self) -> List[str]:
"""Process manifests and save them into Storage service.
@@ -282,29 +225,15 @@ class ManifestProcessor(HeadersMixin):
:return: List of ids of saved records
:rtype: List[str]
"""
- manifest_records = []
- file_records = []
-
- if "manifest" in self.data_object:
- self.create_manifest_records(manifest_records, file_records)
- else:
- raise EmptyManifestError
-
record_ids = []
- if manifest_records:
- save_manifests_response = self.save_record(
- self.request_headers, request_data=manifest_records)
- record_ids.extend(save_manifests_response.json()["recordIds"])
-
- file_record_ids = []
- for file_record in file_records:
- # TODO(python-team) implement concurrent request in File Handler service.
- record_id = self.file_handler.save_file_record(file_record)
- file_location = self.file_handler.get_file_permanent_location(record_id)
- # TODO(python-team) implement rollback strategy in case file validation fails.
- self.source_file_checker.does_file_exist(file_location)
- file_record_ids.append(record_id)
+ populated_manifest_records = []
+ if not self.manifest_records:
+ raise EmptyManifestError
+ for manifest_record in self.manifest_records:
+ populated_manifest_records.append(self.populate_manifest_storage_record(manifest_record.get("entity")))
+ save_manifests_response = self.save_record_to_storage(
+ self.request_headers, request_data=populated_manifest_records)
+ record_ids.extend(save_manifests_response.json()["recordIds"])
- record_ids.extend(file_record_ids)
logger.warning(record_ids)
return record_ids
diff --git a/src/dags/libs/traverse_manifest.py b/src/dags/libs/traverse_manifest.py
new file mode 100644
index 0000000..510cb5f
--- /dev/null
+++ b/src/dags/libs/traverse_manifest.py
@@ -0,0 +1,90 @@
+# Copyright 2021 Google LLC
+# Copyright 2021 EPAM Systems
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import copy
+import logging
+from typing import List
+
+from libs.exceptions import EmptyManifestError
+
+
+logger = logging.getLogger()
+
+
+class ManifestTraversal(object):
+ """Class to traverse manifest and extract all manifest records"""
+
+ def __init__(self, dagrun_conf: dict, manifest_schema: dict):
+ self.data_object = copy.deepcopy(dagrun_conf)
+ self.manifest_schema = manifest_schema
+ self.manifest_info = []
+
+ def _populate_manifest_entity(self, entity: dict, schema: str):
+ """
+ Populate manifest entity for future processing
+
+ :param entity: manifest entity instance (for future processing)
+ :param schema: corresponding generic schema (for future schema validation)
+ :return:
+ """
+ return {
+ "schema": schema,
+ "entity": entity
+ }
+
+ def _traverse_list(self, manifest_entities: List[dict], property_name: str, manifest_schema_part: dict):
+ """
+ Traverse list of entities and returned populated list of entities
+ """
+ entities = []
+ for manifest_entity in manifest_entities:
+ entities.append(
+ self._populate_manifest_entity(manifest_entity,
+ manifest_schema_part[property_name]["items"]["$ref"]))
+ return entities
+
+ def traverse_manifest(self) -> List[dict]:
+ """
+ Traverse manifest structure and return the list of manifest records.
+
+ :return: list of records
+ """
+ if "manifest" not in self.data_object:
+ raise EmptyManifestError
+
+ manifest_entities = []
+ manifest_file = self.data_object["manifest"]
+ if manifest_file.get("ReferenceData"):
+ manifest_entities.extend(self._traverse_list(manifest_file["ReferenceData"],
+ "ReferenceData",
+ self.manifest_schema["properties"]))
+ if manifest_file.get("MasterData"):
+ manifest_entities.extend(self._traverse_list(manifest_file["MasterData"],
+ "MasterData",
+ self.manifest_schema["properties"]))
+ if manifest_file.get("Data"):
+ if manifest_file["Data"].get("WorkProduct"):
+ manifest_entities.append(self._populate_manifest_entity(
+ manifest_file["Data"]["WorkProduct"],
+ self.manifest_schema["properties"]["Data"]["properties"]["WorkProduct"]["$ref"]))
+ if manifest_file["Data"].get("WorkProductComponents"):
+ manifest_entities.extend(self._traverse_list(manifest_file["Data"]["WorkProductComponents"],
+ "WorkProductComponents",
+ self.manifest_schema["properties"]["Data"]["properties"]))
+ if manifest_file["Data"].get("Datasets"):
+ manifest_entities.extend(self._traverse_list(manifest_file["Data"]["Datasets"],
+ "Datasets",
+ self.manifest_schema["properties"]["Data"]["properties"]))
+ return manifest_entities
diff --git a/src/dags/libs/validate_schema.py b/src/dags/libs/validate_schema.py
index e66a9e4..b72647b 100644
--- a/src/dags/libs/validate_schema.py
+++ b/src/dags/libs/validate_schema.py
@@ -16,11 +16,14 @@
"""Provides SchemaValidator."""
import copy
+import json
import logging
+from typing import Union, Any, List
import jsonschema
import requests
import tenacity
+from jsonschema import exceptions
from libs.context import Context
from libs.exceptions import EmptyManifestError, NotOSDUSchemaFormatError
from libs.mixins import HeadersMixin
@@ -72,7 +75,6 @@ class SchemaValidator(HeadersMixin):
def __init__(
self, schema_service: str,
- dagrun_conf: dict,
token_refresher: TokenRefresher,
context: Context
):
@@ -89,7 +91,6 @@ class SchemaValidator(HeadersMixin):
"""
super().__init__(context)
self.schema_service = schema_service
- self.data_object = copy.deepcopy(dagrun_conf)
self.context = context
self.token_refresher = token_refresher
self.resolver_handlers = {
@@ -109,6 +110,23 @@ class SchemaValidator(HeadersMixin):
response = requests.get(uri, headers=headers, timeout=60)
return response
+ def __delete_refs(self, schema_part: Union[dict, list]):
+ """
+ Recursively clear a schema's object parts containing "$ref".
+ This method is used by generic manifest validation, deleting these fields make such a
+ validation.more generic.
+ :param schema_part:
+ """
+ if isinstance(schema_part, dict):
+ if "$ref" in schema_part:
+ schema_part.clear()
+ else:
+ for k in schema_part:
+ self.__delete_refs(schema_part[k])
+ elif isinstance(schema_part, list):
+ for i in schema_part:
+ self.__delete_refs(i)
+
def get_schema_request(self, uri: str) -> dict:
"""Get schema from Schema service. Change $id field to url.
@@ -140,51 +158,126 @@ class SchemaValidator(HeadersMixin):
raise e
return response
- def _validate_schema(self, manifest: dict, schema: dict = None):
- """Validate schema. If argument schema is not defined, then use
- schema service to retrieve corresponding schema.
+ def _delete_id_pattern(self, schema: dict) -> dict:
+ if schema.get("properties") and schema["properties"].get("id"):
+ schema["properties"]["id"].pop("pattern", None)
+ return schema
- :param manifest: the manifest to validate
- :type manifest: dict
- :param schema: the schema to validate against, defaults to None
- :type schema: dict, optional
+ def _validate_entity(self, entity: dict, schema: dict = None):
+ """
+ Validate the 'data' field of any entity against a schema got by entity's kind.
"""
if not schema:
- schema = self.get_schema(manifest["kind"])
- if schema["properties"].get("id"):
- schema["properties"]["id"].pop("pattern", None)
- logger.debug(f"Validating kind {manifest['kind']}")
- resolver = OSDURefResolver(schema_service=self.schema_service,
- base_uri=schema.get("$id", ""), referrer=schema,
- handlers=self.resolver_handlers, cache_remote=True)
- validator = jsonschema.Draft7Validator(schema=schema, resolver=resolver)
- validator.validate(manifest)
-
- def validate_work_product(self, work_product: dict):
- """Validate WP manifest. Raise error if manifest is not valid
+ schema = self.get_schema(entity["kind"])
+ schema = self._delete_id_pattern(schema)
+ data = entity["data"]
+ try:
+ self._validate_against_schema(schema, data)
+ logger.debug(f"Record successfully validated")
+ return True
+ except exceptions.ValidationError as exc:
+ logger.error("Schema validation error. Data field.")
+ logger.error(f"Manifest kind: {entity['kind']}")
+ logger.error(f"Error: {exc}")
+ return False
- :param work_product: the workproduct manifest
- :type work_product: dict
+ def _validate_work_product(self, work_product: dict):
+ """
+ Validate WP manifest. Raise error if manifest is not valid.
"""
for key, value in work_product.items():
if key != "WorkProduct":
for component in value:
- self._validate_schema(component)
+ self._validate_entity(component)
else:
- self._validate_schema(value)
+ self._validate_entity(value)
- def validate_manifest(self):
- """Validate manifest. Raise error if manifest is not valid.
+ def _validate_against_schema(self, schema: dict, data: Any):
+ """
+ Validate any data against schema.
+ :param schema:
+ :param data:
+ :return:
+ """
+ resolver = OSDURefResolver(
+ schema_service=self.schema_service,
+ base_uri=schema.get("$id", ""),
+ referrer=schema,
+ handlers=self.resolver_handlers,
+ cache_remote=True
+ )
+ validator = jsonschema.Draft7Validator(schema=schema, resolver=resolver)
+ validator.validate(data)
- :raises EmptyManifestError: When manifest is empty
- :raises NotOSDUSchemaFormatError: When validation fails
+ def _validate_data_group(self, entities: list):
+ """
+ Validate each entity from a list of entities.
+ :param entities:
+ :return:
+ """
+ if isinstance(entities, list):
+ for entity in entities:
+ self._validate_entity(entity)
+
+ def _validate_whole_manifest(self, manifest: dict):
+ """
+ Validate any manifest in general.
+ Also at this step verify that MasterData, ReferenceData, WorkProduct, WorkProductComponents,
+ Files entities correspond their generic schemas, because references to those schemas are in
+ a Manifest schema.
+ """
+ schema = self.get_schema(manifest["kind"])
+ logger.debug(f"Validating kind {manifest['kind']}")
+ self._validate_against_schema(schema, manifest)
+
+ def validate_common_schema(self, manifest: dict) -> dict:
+ """
+ This is a preliminary validation of a manifest that verifies that a manifest corresponds
+ the OSDU schemes at the highest level.
+ This validation skips verifying each concrete entity by removing references to their schemas.
+ :param manifest:
+ :return: Manifest schema
+ """
+ if "manifest" not in manifest:
+ raise EmptyManifestError
+ schema = self.get_schema(manifest["manifest"]["kind"])
+ schema_without_refs = copy.deepcopy(schema)
+ if schema_without_refs.get("properties"):
+ self.__delete_refs(schema_without_refs["properties"])
+ else:
+ self.__delete_refs(schema_without_refs)
+ logger.debug("Schema without refs")
+ logger.debug(f"{schema_without_refs}")
+ self._validate_against_schema(schema, manifest)
+ return schema
+
+ def _validate_against_generic_schema(self, schema: str, entity: Any) -> bool:
+ try:
+ self._validate_against_schema(schema, entity)
+ logger.debug(f"Record successfully validated against generic schema.")
+ return True
+ except exceptions.ValidationError as exc:
+ logger.error("Schema validation error.")
+ logger.error(f"Manifest kind: {schema['kind']}")
+ logger.error(f"Manifest: {entity}")
+ logger.error(f"Error: {exc}")
+ return False
+
+ def validate_manifest(self, manifest_records: List[dict]) -> List[dict]:
+ """
+ Validate manifest. Raise error if manifest is not valid.
"""
- if "manifest" not in self.data_object:
+ validated_records = []
+ if not manifest_records:
raise EmptyManifestError
- for manifest in self.data_object["manifest"]:
+ for manifest_record in manifest_records:
+ manifest = manifest_record.get("entity")
if isinstance(manifest, dict) and manifest.get("kind"):
- self._validate_schema(manifest)
- elif manifest.get("WorkProductComponents"):
- self.validate_work_product(manifest)
+ generic_schema = self.get_schema(manifest_record.get("schema"))
+ validation_result = self._validate_against_generic_schema(generic_schema, manifest) \
+ and self._validate_entity(manifest)
+ if validation_result:
+ validated_records.append(manifest_record)
else:
raise NotOSDUSchemaFormatError(f"Not valid schema {manifest}")
+ return validated_records
diff --git a/src/plugins/operators/process_manifest_r3.py b/src/plugins/operators/process_manifest_r3.py
index 62b163a..df16b8c 100644
--- a/src/plugins/operators/process_manifest_r3.py
+++ b/src/plugins/operators/process_manifest_r3.py
@@ -15,6 +15,7 @@
"""R2 Process Manifest operator."""
+import logging
from airflow.utils import apply_defaults
from airflow.models import BaseOperator, Variable
from libs.context import Context
@@ -22,8 +23,11 @@ from libs.source_file_check import GCSSourceFileChecker
from libs.handle_file import GCSFileHandler
from libs.refresh_token import AirflowTokenRefresher
from libs.process_manifest_r3 import ManifestProcessor
+from libs.traverse_manifest import ManifestTraversal
from libs.validate_schema import SchemaValidator
+logger = logging.getLogger()
+
class ProcessManifestOperatorR3(BaseOperator):
"""Operator to process manifest R3."""
@@ -42,6 +46,13 @@ class ProcessManifestOperatorR3(BaseOperator):
def execute(self, context: dict):
"""Execute manifest validation then process it.
+ Execution steps:
+ 1) initialize schema validator
+ 2) validate manifest file against common schema
+ 3) traverse manifest file and extract manifest entities
+ 4) validate extracted manifest entities
+ 5) process valid manifest entities
+
:param context: Airflow context
:type context: dict
"""
@@ -49,22 +60,24 @@ class ProcessManifestOperatorR3(BaseOperator):
token_refresher = AirflowTokenRefresher()
file_handler = GCSFileHandler(self.file_service_url, token_refresher, payload_context)
source_file_checker = GCSSourceFileChecker()
-
validator = SchemaValidator(
self.schema_service_url,
- context["dag_run"].conf,
token_refresher,
payload_context
)
+ manifest_schema = validator.validate_common_schema(context["dag_run"].conf)
+ traversal = ManifestTraversal(context["dag_run"].conf, manifest_schema)
+ manifest_entities = traversal.traverse_manifest()
+ logger.debug(f"entities count: {len(manifest_entities)}")
+ valid_manifest_entities = validator.validate_manifest(manifest_entities)
+ logger.debug(f"valid entities count: {len(valid_manifest_entities)}")
manifest_processor = ManifestProcessor(
self.storage_url,
- context["dag_run"].conf,
+ valid_manifest_entities,
file_handler,
source_file_checker,
token_refresher,
payload_context,
)
-
- validator.validate_manifest()
record_ids = manifest_processor.process_manifest()
context["ti"].xcom_push(key="record_ids", value=record_ids)
diff --git a/tests/plugin-unit-tests/data/invalid/NotOsduFormat.json b/tests/plugin-unit-tests/data/invalid/NotOsduFormat.json
index 62b4615..88b8815 100644
--- a/tests/plugin-unit-tests/data/invalid/NotOsduFormat.json
+++ b/tests/plugin-unit-tests/data/invalid/NotOsduFormat.json
@@ -7,10 +7,6 @@
},
"$schema": "https://schema.osdu.opengroup.org/json/master-data/Wellbore.1.0.0.json",
"$filename": "load_Wellbore.1.0.0_350112350400.json",
- "manifest": [
- {
-
- }
- ],
+ "manifest": {"test": "test"},
"WorkflowID": "foo"
}
diff --git a/tests/plugin-unit-tests/data/invalid/TraversalEmptyManifest.json b/tests/plugin-unit-tests/data/invalid/TraversalEmptyManifest.json
new file mode 100644
index 0000000..0d4f101
--- /dev/null
+++ b/tests/plugin-unit-tests/data/invalid/TraversalEmptyManifest.json
@@ -0,0 +1,2 @@
+[
+]
diff --git a/tests/plugin-unit-tests/data/invalid/TraversalNotOSDUFormatManifest.json b/tests/plugin-unit-tests/data/invalid/TraversalNotOSDUFormatManifest.json
new file mode 100644
index 0000000..5c1b830
--- /dev/null
+++ b/tests/plugin-unit-tests/data/invalid/TraversalNotOSDUFormatManifest.json
@@ -0,0 +1,8 @@
+[
+ {
+ "entity": {"test": "test"},
+ "group_type": "",
+ "schema": "GenericMasterData"
+ }
+]
+
diff --git a/tests/plugin-unit-tests/data/manifests/Manifest.1.0.0.json b/tests/plugin-unit-tests/data/manifests/Manifest.1.0.0.json
new file mode 100644
index 0000000..3a1833c
--- /dev/null
+++ b/tests/plugin-unit-tests/data/manifests/Manifest.1.0.0.json
@@ -0,0 +1,559 @@
+{
+ "kind": "osdu:wks:Manifest:1.0.0",
+ "ReferenceData": [
+ {
+ "id": "namespace:reference-data--GenericReferenceData:63ca0ed3-d6fb-53f0-8549-0916ef144266",
+ "kind": "osdu:wks:reference-data--GenericReferenceData:1.0.0",
+ "version": 1562066009929332,
+ "acl": {
+ "owners": [
+ "someone@company.com"
+ ],
+ "viewers": [
+ "someone@company.com"
+ ]
+ },
+ "legal": {
+ "legaltags": [
+ "Example legaltags"
+ ],
+ "otherRelevantDataCountries": [
+ "US"
+ ],
+ "status": "compliant"
+ },
+ "resourceHomeRegionID": "namespace:reference-data--OSDURegion:SomeUniqueOSDURegionID:",
+ "resourceHostRegionIDs": [
+ "namespace:reference-data--OSDURegion:SomeUniqueOSDURegionID:"
+ ],
+ "createTime": "2020-12-16T11:46:20.163Z",
+ "createUser": "some-user@some-company-cloud.com",
+ "modifyTime": "2020-12-16T11:52:24.477Z",
+ "modifyUser": "some-user@some-company-cloud.com",
+ "resourceCurationStatus": "namespace:reference-data--ResourceCurationStatus:CREATED:",
+ "resourceLifecycleStatus": "namespace:reference-data--ResourceLifecycleStatus:LOADING:",
+ "resourceSecurityClassification": "namespace:reference-data--ResourceSecurityClassification:SomeUniqueResourceSecurityClassificationID:",
+ "ancestry": {
+ "parents": []
+ },
+ "meta": [
+ {
+ "kind": "CRS",
+ "name": "NAD27 * OGP-Usa Conus / North Dakota South [32021,15851]",
+ "persistableReference": "{\"authCode\":{\"auth\":\"EPSG\",\"code\":\"32615\"},\"type\":\"LBC\",\"ver\":\"PE_10_3_1\",\"name\":\"WGS_1984_UTM_Zone_15N\",\"wkt\":\"PROJCS[\\\"WGS_1984_UTM_Zone_15N\\\",GEOGCS[\\\"GCS_WGS_1984\\\",DATUM[\\\"D_WGS_1984\\\",SPHEROID[\\\"WGS_1984\\\",6378137.0,298.257223563]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],PROJECTION[\\\"Transverse_Mercator\\\"],PARAMETER[\\\"False_Easting\\\",500000.0],PARAMETER[\\\"False_Northing\\\",0.0],PARAMETER[\\\"Central_Meridian\\\",-93.0],PARAMETER[\\\"Scale_Factor\\\",0.9996],PARAMETER[\\\"Latitude_Of_Origin\\\",0.0],UNIT[\\\"Meter\\\",1.0],AUTHORITY[\\\"EPSG\\\",32615]]\"}",
+ "coordinateReferenceSystemID": "namespace:reference-data--CoordinateReferenceSystem:EPSG.32615:",
+ "propertyNames": [
+ "KickOffPosition.X",
+ "KickOffPosition.Y"
+ ]
+ }
+ ],
+ "source": "Example Data Source",
+ "existenceKind": "namespace:reference-data--ExistenceKind:Prototype:",
+ "data": {}
+ }
+ ],
+ "MasterData": [
+ {
+ "id": "namespace:master-data--GenericMasterData:9ca8054c-bce6-5a3a-b51d-f216fb1085a5",
+ "kind": "osdu:wks:master-data--GenericMasterData:1.0.0",
+ "version": 1562066009929332,
+ "acl": {
+ "owners": [
+ "someone@company.com"
+ ],
+ "viewers": [
+ "someone@company.com"
+ ]
+ },
+ "legal": {
+ "legaltags": [
+ "Example legaltags"
+ ],
+ "otherRelevantDataCountries": [
+ "US"
+ ],
+ "status": "compliant"
+ },
+ "resourceHomeRegionID": "namespace:reference-data--OSDURegion:SomeUniqueOSDURegionID:",
+ "resourceHostRegionIDs": [
+ "namespace:reference-data--OSDURegion:SomeUniqueOSDURegionID:"
+ ],
+ "createTime": "2020-12-16T11:46:20.163Z",
+ "createUser": "some-user@some-company-cloud.com",
+ "modifyTime": "2020-12-16T11:52:24.477Z",
+ "modifyUser": "some-user@some-company-cloud.com",
+ "resourceCurationStatus": "namespace:reference-data--ResourceCurationStatus:CREATED:",
+ "resourceLifecycleStatus": "namespace:reference-data--ResourceLifecycleStatus:LOADING:",
+ "resourceSecurityClassification": "namespace:reference-data--ResourceSecurityClassification:SomeUniqueResourceSecurityClassificationID:",
+ "ancestry": {
+ "parents": []
+ },
+ "meta": [
+ {
+ "kind": "CRS",
+ "name": "NAD27 * OGP-Usa Conus / North Dakota South [32021,15851]",
+ "persistableReference": "{\"authCode\":{\"auth\":\"EPSG\",\"code\":\"32615\"},\"type\":\"LBC\",\"ver\":\"PE_10_3_1\",\"name\":\"WGS_1984_UTM_Zone_15N\",\"wkt\":\"PROJCS[\\\"WGS_1984_UTM_Zone_15N\\\",GEOGCS[\\\"GCS_WGS_1984\\\",DATUM[\\\"D_WGS_1984\\\",SPHEROID[\\\"WGS_1984\\\",6378137.0,298.257223563]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],PROJECTION[\\\"Transverse_Mercator\\\"],PARAMETER[\\\"False_Easting\\\",500000.0],PARAMETER[\\\"False_Northing\\\",0.0],PARAMETER[\\\"Central_Meridian\\\",-93.0],PARAMETER[\\\"Scale_Factor\\\",0.9996],PARAMETER[\\\"Latitude_Of_Origin\\\",0.0],UNIT[\\\"Meter\\\",1.0],AUTHORITY[\\\"EPSG\\\",32615]]\"}",
+ "coordinateReferenceSystemID": "namespace:reference-data--CoordinateReferenceSystem:EPSG.32615:",
+ "propertyNames": [
+ "KickOffPosition.X",
+ "KickOffPosition.Y"
+ ]
+ }
+ ],
+ "source": "Example Data Source",
+ "existenceKind": "namespace:reference-data--ExistenceKind:Prototype:",
+ "data": {}
+ }
+ ],
+ "Data": {
+ "WorkProduct": {
+ "id": "surrogate-key:wp-id-1",
+ "kind": "osdu:wks:work-product--GenericWorkProduct:1.0.0",
+ "version": 1562066009929332,
+ "acl": {
+ "owners": [
+ "someone@company.com"
+ ],
+ "viewers": [
+ "someone@company.com"
+ ]
+ },
+ "legal": {
+ "legaltags": [
+ "Example legaltags"
+ ],
+ "otherRelevantDataCountries": [
+ "US"
+ ],
+ "status": "compliant"
+ },
+ "resourceHomeRegionID": "namespace:reference-data--OSDURegion:SomeUniqueOSDURegionID:",
+ "resourceHostRegionIDs": [
+ "namespace:reference-data--OSDURegion:SomeUniqueOSDURegionID:"
+ ],
+ "createTime": "2020-12-16T11:46:20.163Z",
+ "createUser": "some-user@some-company-cloud.com",
+ "modifyTime": "2020-12-16T11:52:24.477Z",
+ "modifyUser": "some-user@some-company-cloud.com",
+ "resourceCurationStatus": "namespace:reference-data--ResourceCurationStatus:CREATED:",
+ "resourceLifecycleStatus": "namespace:reference-data--ResourceLifecycleStatus:LOADING:",
+ "resourceSecurityClassification": "namespace:reference-data--ResourceSecurityClassification:SomeUniqueResourceSecurityClassificationID:",
+ "ancestry": {
+ "parents": []
+ },
+ "meta": [
+ {
+ "kind": "CRS",
+ "name": "NAD27 * OGP-Usa Conus / North Dakota South [32021,15851]",
+ "persistableReference": "{\"authCode\":{\"auth\":\"EPSG\",\"code\":\"32615\"},\"type\":\"LBC\",\"ver\":\"PE_10_3_1\",\"name\":\"WGS_1984_UTM_Zone_15N\",\"wkt\":\"PROJCS[\\\"WGS_1984_UTM_Zone_15N\\\",GEOGCS[\\\"GCS_WGS_1984\\\",DATUM[\\\"D_WGS_1984\\\",SPHEROID[\\\"WGS_1984\\\",6378137.0,298.257223563]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],PROJECTION[\\\"Transverse_Mercator\\\"],PARAMETER[\\\"False_Easting\\\",500000.0],PARAMETER[\\\"False_Northing\\\",0.0],PARAMETER[\\\"Central_Meridian\\\",-93.0],PARAMETER[\\\"Scale_Factor\\\",0.9996],PARAMETER[\\\"Latitude_Of_Origin\\\",0.0],UNIT[\\\"Meter\\\",1.0],AUTHORITY[\\\"EPSG\\\",32615]]\"}",
+ "coordinateReferenceSystemID": "namespace:reference-data--CoordinateReferenceSystem:EPSG.32615:",
+ "propertyNames": [
+ "KickOffPosition.X",
+ "KickOffPosition.Y"
+ ]
+ }
+ ],
+ "source": "Example Data Source",
+ "existenceKind": "namespace:reference-data--ExistenceKind:Prototype:",
+ "data": {
+ "Components": [
+ "surrogate-key:wpc-id-1"
+ ],
+ "IsExtendedLoad": true,
+ "IsDiscoverable": true,
+ "Name": "Example Name",
+ "Description": "Example Description",
+ "CreationDateTime": "2020-02-13T09:13:15.55Z",
+ "Tags": [
+ "Example Tags"
+ ],
+ "SpatialPoint": {
+ "SpatialLocationCoordinatesDate": "2020-02-13T09:13:15.55Z",
+ "QuantitativeAccuracyBandID": "namespace:reference-data--QuantitativeAccuracyBand:SomeUniqueQuantitativeAccuracyBandID:",
+ "QualitativeSpatialAccuracyTypeID": "namespace:reference-data--QualitativeSpatialAccuracyType:SomeUniqueQualitativeSpatialAccuracyTypeID:",
+ "CoordinateQualityCheckPerformedBy": "Example CoordinateQualityCheckPerformedBy",
+ "CoordinateQualityCheckDateTime": "2020-02-13T09:13:15.55Z",
+ "CoordinateQualityCheckRemarks": [
+ "Example CoordinateQualityCheckRemarks"
+ ],
+ "AsIngestedCoordinates": {
+ "type": "AnyCrsFeatureCollection",
+ "CoordinateReferenceSystemID": "namespace:reference-data--CoordinateReferenceSystem:BoundCRS.SLB.32021.15851:",
+ "VerticalCoordinateReferenceSystemID": "namespace:reference-data--CoordinateReferenceSystem:VerticalCRS.EPSG.5773:",
+ "persistableReferenceCRS": "{\"lateBoundCRS\":{\"wkt\":\"PROJCS[\\\"NAD_1927_StatePlane_North_Dakota_South_FIPS_3302\\\",GEOGCS[\\\"GCS_North_American_1927\\\",DATUM[\\\"D_North_American_1927\\\",SPHEROID[\\\"Clarke_1866\\\",6378206.4,294.9786982]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],PROJECTION[\\\"Lambert_Conformal_Conic\\\"],PARAMETER[\\\"False_Easting\\\",2000000.0],PARAMETER[\\\"False_Northing\\\",0.0],PARAMETER[\\\"Central_Meridian\\\",-100.5],PARAMETER[\\\"Standard_Parallel_1\\\",46.1833333333333],PARAMETER[\\\"Standard_Parallel_2\\\",47.4833333333333],PARAMETER[\\\"Latitude_Of_Origin\\\",45.6666666666667],UNIT[\\\"Foot_US\\\",0.304800609601219],AUTHORITY[\\\"EPSG\\\",32021]]\",\"ver\":\"PE_10_3_1\",\"name\":\"NAD_1927_StatePlane_North_Dakota_South_FIPS_3302\",\"authCode\":{\"auth\":\"EPSG\",\"code\":\"32021\"},\"type\":\"LBC\"},\"singleCT\":{\"wkt\":\"GEOGTRAN[\\\"NAD_1927_To_WGS_1984_79_CONUS\\\",GEOGCS[\\\"GCS_North_American_1927\\\",DATUM[\\\"D_North_American_1927\\\",SPHEROID[\\\"Clarke_1866\\\",6378206.4,294.9786982]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],GEOGCS[\\\"GCS_WGS_1984\\\",DATUM[\\\"D_WGS_1984\\\",SPHEROID[\\\"WGS_1984\\\",6378137.0,298.257223563]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],METHOD[\\\"NADCON\\\"],PARAMETER[\\\"Dataset_conus\\\",0.0],AUTHORITY[\\\"EPSG\\\",15851]]\",\"ver\":\"PE_10_3_1\",\"name\":\"NAD_1927_To_WGS_1984_79_CONUS\",\"authCode\":{\"auth\":\"EPSG\",\"code\":\"15851\"},\"type\":\"ST\"},\"ver\":\"PE_10_3_1\",\"name\":\"NAD27 * OGP-Usa Conus / North Dakota South [32021,15851]\",\"authCode\":{\"auth\":\"SLB\",\"code\":\"32021079\"},\"type\":\"EBC\"}",
+ "persistableReferenceVerticalCRS": "{\"authCode\":{\"auth\":\"EPSG\",\"code\":\"5773\"},\"type\":\"LBC\",\"ver\":\"PE_10_3_1\",\"name\":\"EGM96_Geoid\",\"wkt\":\"VERTCS[\\\"EGM96_Geoid\\\",VDATUM[\\\"EGM96_Geoid\\\"],PARAMETER[\\\"Vertical_Shift\\\",0.0],PARAMETER[\\\"Direction\\\",1.0],UNIT[\\\"Meter\\\",1.0],AUTHORITY[\\\"EPSG\\\",5773]]\"}",
+ "persistableReferenceUnitZ": "{\"scaleOffset\":{\"scale\":1.0,\"offset\":0.0},\"symbol\":\"m\",\"baseMeasurement\":{\"ancestry\":\"Length\",\"type\":\"UM\"},\"type\":\"USO\"}",
+ "features": [
+ {
+ "type": "AnyCrsFeature",
+ "properties": {},
+ "geometry": {
+ "type": "AnyCrsPoint",
+ "coordinates": [
+ 12345.6,
+ 12345.6
+ ],
+ "bbox": [
+ 12345.6,
+ 12345.6,
+ 12345.6,
+ 12345.6
+ ]
+ },
+ "bbox": [
+ 12345.6,
+ 12345.6,
+ 12345.6,
+ 12345.6
+ ]
+ }
+ ],
+ "bbox": [
+ 12345.6,
+ 12345.6,
+ 12345.6,
+ 12345.6
+ ]
+ },
+ "Wgs84Coordinates": {
+ "type": "FeatureCollection",
+ "features": [
+ {
+ "type": "Feature",
+ "properties": {},
+ "geometry": {
+ "type": "Point",
+ "coordinates": [
+ 12345.6,
+ 12345.6
+ ],
+ "bbox": [
+ 12345.6,
+ 12345.6,
+ 12345.6,
+ 12345.6
+ ]
+ },
+ "bbox": [
+ 12345.6,
+ 12345.6,
+ 12345.6,
+ 12345.6
+ ]
+ }
+ ],
+ "bbox": [
+ 12345.6,
+ 12345.6,
+ 12345.6,
+ 12345.6
+ ]
+ },
+ "OperationsApplied": [
+ "conversion from ED_1950_UTM_Zone_31N to GCS_European_1950; 1 points converted",
+ "transformation GCS_European_1950 to GCS_WGS_1984 using ED_1950_To_WGS_1984_24; 1 points successfully transformed"
+ ],
+ "SpatialParameterTypeID": "namespace:reference-data--SpatialParameterType:Outline:",
+ "SpatialGeometryTypeID": "namespace:reference-data--SpatialGeometryType:Point:"
+ },
+ "SpatialArea": {
+ "SpatialLocationCoordinatesDate": "2020-02-13T09:13:15.55Z",
+ "QuantitativeAccuracyBandID": "namespace:reference-data--QuantitativeAccuracyBand:SomeUniqueQuantitativeAccuracyBandID:",
+ "QualitativeSpatialAccuracyTypeID": "namespace:reference-data--QualitativeSpatialAccuracyType:SomeUniqueQualitativeSpatialAccuracyTypeID:",
+ "CoordinateQualityCheckPerformedBy": "Example CoordinateQualityCheckPerformedBy",
+ "CoordinateQualityCheckDateTime": "2020-02-13T09:13:15.55Z",
+ "CoordinateQualityCheckRemarks": [
+ "Example CoordinateQualityCheckRemarks"
+ ],
+ "AsIngestedCoordinates": {
+ "type": "AnyCrsFeatureCollection",
+ "CoordinateReferenceSystemID": "namespace:reference-data--CoordinateReferenceSystem:BoundCRS.SLB.32021.15851:",
+ "VerticalCoordinateReferenceSystemID": "namespace:reference-data--CoordinateReferenceSystem:VerticalCRS.EPSG.5773:",
+ "persistableReferenceCRS": "{\"lateBoundCRS\":{\"wkt\":\"PROJCS[\\\"NAD_1927_StatePlane_North_Dakota_South_FIPS_3302\\\",GEOGCS[\\\"GCS_North_American_1927\\\",DATUM[\\\"D_North_American_1927\\\",SPHEROID[\\\"Clarke_1866\\\",6378206.4,294.9786982]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],PROJECTION[\\\"Lambert_Conformal_Conic\\\"],PARAMETER[\\\"False_Easting\\\",2000000.0],PARAMETER[\\\"False_Northing\\\",0.0],PARAMETER[\\\"Central_Meridian\\\",-100.5],PARAMETER[\\\"Standard_Parallel_1\\\",46.1833333333333],PARAMETER[\\\"Standard_Parallel_2\\\",47.4833333333333],PARAMETER[\\\"Latitude_Of_Origin\\\",45.6666666666667],UNIT[\\\"Foot_US\\\",0.304800609601219],AUTHORITY[\\\"EPSG\\\",32021]]\",\"ver\":\"PE_10_3_1\",\"name\":\"NAD_1927_StatePlane_North_Dakota_South_FIPS_3302\",\"authCode\":{\"auth\":\"EPSG\",\"code\":\"32021\"},\"type\":\"LBC\"},\"singleCT\":{\"wkt\":\"GEOGTRAN[\\\"NAD_1927_To_WGS_1984_79_CONUS\\\",GEOGCS[\\\"GCS_North_American_1927\\\",DATUM[\\\"D_North_American_1927\\\",SPHEROID[\\\"Clarke_1866\\\",6378206.4,294.9786982]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],GEOGCS[\\\"GCS_WGS_1984\\\",DATUM[\\\"D_WGS_1984\\\",SPHEROID[\\\"WGS_1984\\\",6378137.0,298.257223563]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],METHOD[\\\"NADCON\\\"],PARAMETER[\\\"Dataset_conus\\\",0.0],AUTHORITY[\\\"EPSG\\\",15851]]\",\"ver\":\"PE_10_3_1\",\"name\":\"NAD_1927_To_WGS_1984_79_CONUS\",\"authCode\":{\"auth\":\"EPSG\",\"code\":\"15851\"},\"type\":\"ST\"},\"ver\":\"PE_10_3_1\",\"name\":\"NAD27 * OGP-Usa Conus / North Dakota South [32021,15851]\",\"authCode\":{\"auth\":\"SLB\",\"code\":\"32021079\"},\"type\":\"EBC\"}",
+ "persistableReferenceVerticalCRS": "{\"authCode\":{\"auth\":\"EPSG\",\"code\":\"5773\"},\"type\":\"LBC\",\"ver\":\"PE_10_3_1\",\"name\":\"EGM96_Geoid\",\"wkt\":\"VERTCS[\\\"EGM96_Geoid\\\",VDATUM[\\\"EGM96_Geoid\\\"],PARAMETER[\\\"Vertical_Shift\\\",0.0],PARAMETER[\\\"Direction\\\",1.0],UNIT[\\\"Meter\\\",1.0],AUTHORITY[\\\"EPSG\\\",5773]]\"}",
+ "persistableReferenceUnitZ": "{\"scaleOffset\":{\"scale\":1.0,\"offset\":0.0},\"symbol\":\"m\",\"baseMeasurement\":{\"ancestry\":\"Length\",\"type\":\"UM\"},\"type\":\"USO\"}",
+ "features": [
+ {
+ "type": "AnyCrsFeature",
+ "properties": {},
+ "geometry": {
+ "type": "AnyCrsPoint",
+ "coordinates": [
+ 12345.6,
+ 12345.6
+ ],
+ "bbox": [
+ 12345.6,
+ 12345.6,
+ 12345.6,
+ 12345.6
+ ]
+ },
+ "bbox": [
+ 12345.6,
+ 12345.6,
+ 12345.6,
+ 12345.6
+ ]
+ }
+ ],
+ "bbox": [
+ 12345.6,
+ 12345.6,
+ 12345.6,
+ 12345.6
+ ]
+ },
+ "Wgs84Coordinates": {
+ "type": "FeatureCollection",
+ "features": [
+ {
+ "type": "Feature",
+ "properties": {},
+ "geometry": {
+ "type": "Point",
+ "coordinates": [
+ 12345.6,
+ 12345.6
+ ],
+ "bbox": [
+ 12345.6,
+ 12345.6,
+ 12345.6,
+ 12345.6
+ ]
+ },
+ "bbox": [
+ 12345.6,
+ 12345.6,
+ 12345.6,
+ 12345.6
+ ]
+ }
+ ],
+ "bbox": [
+ 12345.6,
+ 12345.6,
+ 12345.6,
+ 12345.6
+ ]
+ },
+ "OperationsApplied": [
+ "conversion from ED_1950_UTM_Zone_31N to GCS_European_1950; 1 points converted",
+ "transformation GCS_European_1950 to GCS_WGS_1984 using ED_1950_To_WGS_1984_24; 1 points successfully transformed"
+ ],
+ "SpatialParameterTypeID": "namespace:reference-data--SpatialParameterType:Outline:",
+ "SpatialGeometryTypeID": "namespace:reference-data--SpatialGeometryType:Point:"
+ },
+ "SubmitterName": "Example SubmitterName",
+ "BusinessActivities": [
+ "Example BusinessActivities"
+ ],
+ "AuthorIDs": [
+ "Example AuthorIDs"
+ ],
+ "LineageAssertions": [
+ {
+ "ID": "namespace:any-group-type--AnyIndividualType:SomeUniqueAnyIndividualTypeID:",
+ "LineageRelationshipType": "namespace:reference-data--LineageRelationshipType:Direct:"
+ }
+ ],
+ "Annotations": [
+ "Example Annotations"
+ ]
+ }
+ },
+ "WorkProductComponents": [
+ {
+ "id": "surrogate-key:wpc-id-1",
+ "kind": "osdu:wks:work-product-component--GenericWorkProductComponent:1.0.0",
+ "version": 1562066009929332,
+ "acl": {
+ "owners": [
+ "someone@company.com"
+ ],
+ "viewers": [
+ "someone@company.com"
+ ]
+ },
+ "legal": {
+ "legaltags": [
+ "Example legaltags"
+ ],
+ "otherRelevantDataCountries": [
+ "US"
+ ],
+ "status": "compliant"
+ },
+ "resourceHomeRegionID": "namespace:reference-data--OSDURegion:SomeUniqueOSDURegionID:",
+ "resourceHostRegionIDs": [
+ "namespace:reference-data--OSDURegion:SomeUniqueOSDURegionID:"
+ ],
+ "createTime": "2020-12-16T11:46:20.163Z",
+ "createUser": "some-user@some-company-cloud.com",
+ "modifyTime": "2020-12-16T11:52:24.477Z",
+ "modifyUser": "some-user@some-company-cloud.com",
+ "resourceCurationStatus": "namespace:reference-data--ResourceCurationStatus:CREATED:",
+ "resourceLifecycleStatus": "namespace:reference-data--ResourceLifecycleStatus:LOADING:",
+ "resourceSecurityClassification": "namespace:reference-data--ResourceSecurityClassification:SomeUniqueResourceSecurityClassificationID:",
+ "ancestry": {
+ "parents": []
+ },
+ "meta": [
+ {
+ "kind": "CRS",
+ "name": "NAD27 * OGP-Usa Conus / North Dakota South [32021,15851]",
+ "persistableReference": "{\"authCode\":{\"auth\":\"EPSG\",\"code\":\"32615\"},\"type\":\"LBC\",\"ver\":\"PE_10_3_1\",\"name\":\"WGS_1984_UTM_Zone_15N\",\"wkt\":\"PROJCS[\\\"WGS_1984_UTM_Zone_15N\\\",GEOGCS[\\\"GCS_WGS_1984\\\",DATUM[\\\"D_WGS_1984\\\",SPHEROID[\\\"WGS_1984\\\",6378137.0,298.257223563]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],PROJECTION[\\\"Transverse_Mercator\\\"],PARAMETER[\\\"False_Easting\\\",500000.0],PARAMETER[\\\"False_Northing\\\",0.0],PARAMETER[\\\"Central_Meridian\\\",-93.0],PARAMETER[\\\"Scale_Factor\\\",0.9996],PARAMETER[\\\"Latitude_Of_Origin\\\",0.0],UNIT[\\\"Meter\\\",1.0],AUTHORITY[\\\"EPSG\\\",32615]]\"}",
+ "coordinateReferenceSystemID": "namespace:reference-data--CoordinateReferenceSystem:EPSG.32615:",
+ "propertyNames": [
+ "KickOffPosition.X",
+ "KickOffPosition.Y"
+ ]
+ }
+ ],
+ "source": "Example Data Source",
+ "existenceKind": "namespace:reference-data--ExistenceKind:Prototype:",
+ "data": {
+ "Datasets": [
+ "surrogate-key:.+"
+ ],
+ "Artefacts": [
+ {
+ "RoleID": "namespace:reference-data--ArtefactRole:SomeUniqueArtefactRoleID:",
+ "ResourceKind": "namespace:source_name:group_type/IndividualType:0.0.0",
+ "ResourceID": "surrogate-key:file-2"
+ }
+ ],
+ "IsExtendedLoad": true,
+ "IsDiscoverable": true,
+ "DatasetIDs": [
+ "surrogate-key:file-1"
+ ]
+ }
+ }
+ ],
+ "Datasets": [
+ {
+ "id": "surrogate-key:file-1",
+ "kind": "osdu:wks:dataset--GenericDataset:1.0.0",
+ "version": 1562066009929332,
+ "acl": {
+ "owners": [
+ "someone@company.com"
+ ],
+ "viewers": [
+ "someone@company.com"
+ ]
+ },
+ "legal": {
+ "legaltags": [
+ "Example legaltags"
+ ],
+ "otherRelevantDataCountries": [
+ "US"
+ ],
+ "status": "compliant"
+ },
+ "resourceHomeRegionID": "namespace:reference-data--OSDURegion:SomeUniqueOSDURegionID:",
+ "resourceHostRegionIDs": [
+ "namespace:reference-data--OSDURegion:SomeUniqueOSDURegionID:"
+ ],
+ "createTime": "2020-12-16T11:46:20.163Z",
+ "createUser": "some-user@some-company-cloud.com",
+ "modifyTime": "2020-12-16T11:52:24.477Z",
+ "modifyUser": "some-user@some-company-cloud.com",
+ "resourceCurationStatus": "namespace:reference-data--ResourceCurationStatus:CREATED:",
+ "resourceLifecycleStatus": "namespace:reference-data--ResourceLifecycleStatus:LOADING:",
+ "resourceSecurityClassification": "namespace:reference-data--ResourceSecurityClassification:SomeUniqueResourceSecurityClassificationID:",
+ "ancestry": {
+ "parents": []
+ },
+ "meta": [
+ {
+ "kind": "CRS",
+ "name": "NAD27 * OGP-Usa Conus / North Dakota South [32021,15851]",
+ "persistableReference": "{\"authCode\":{\"auth\":\"EPSG\",\"code\":\"32615\"},\"type\":\"LBC\",\"ver\":\"PE_10_3_1\",\"name\":\"WGS_1984_UTM_Zone_15N\",\"wkt\":\"PROJCS[\\\"WGS_1984_UTM_Zone_15N\\\",GEOGCS[\\\"GCS_WGS_1984\\\",DATUM[\\\"D_WGS_1984\\\",SPHEROID[\\\"WGS_1984\\\",6378137.0,298.257223563]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],PROJECTION[\\\"Transverse_Mercator\\\"],PARAMETER[\\\"False_Easting\\\",500000.0],PARAMETER[\\\"False_Northing\\\",0.0],PARAMETER[\\\"Central_Meridian\\\",-93.0],PARAMETER[\\\"Scale_Factor\\\",0.9996],PARAMETER[\\\"Latitude_Of_Origin\\\",0.0],UNIT[\\\"Meter\\\",1.0],AUTHORITY[\\\"EPSG\\\",32615]]\"}",
+ "coordinateReferenceSystemID": "namespace:reference-data--CoordinateReferenceSystem:EPSG.32615:",
+ "propertyNames": [
+ "KickOffPosition.X",
+ "KickOffPosition.Y"
+ ]
+ }
+ ],
+ "source": "Example Data Source",
+ "existenceKind": "namespace:reference-data--ExistenceKind:Prototype:",
+ "data": {
+ "Name": "Dataset X221/15",
+ "Description": "As originally delivered by ACME.com.",
+ "Source": "ACME.com",
+ "TotalSize": "13245217273",
+ "EncodingFormatTypeID": "namespace:reference-data--EncodingFormatType:UTF-8:",
+ "MimeType": "namespace:reference-data--MimeType:application/geo+json:",
+ "Endian": "BIG",
+ "DatasetProperties": {}
+ }
+ },
+ {
+ "id": "surrogate-key:file-2",
+ "kind": "osdu:wks:dataset--GenericDataset:1.0.0",
+ "version": 1562066009929332,
+ "acl": {
+ "owners": [
+ "someone@company.com"
+ ],
+ "viewers": [
+ "someone@company.com"
+ ]
+ },
+ "legal": {
+ "legaltags": [
+ "Example legaltags"
+ ],
+ "otherRelevantDataCountries": [
+ "US"
+ ],
+ "status": "compliant"
+ },
+ "resourceHomeRegionID": "namespace:reference-data--OSDURegion:SomeUniqueOSDURegionID:",
+ "resourceHostRegionIDs": [
+ "namespace:reference-data--OSDURegion:SomeUniqueOSDURegionID:"
+ ],
+ "createTime": "2020-12-16T11:46:20.163Z",
+ "createUser": "some-user@some-company-cloud.com",
+ "modifyTime": "2020-12-16T11:52:24.477Z",
+ "modifyUser": "some-user@some-company-cloud.com",
+ "resourceCurationStatus": "namespace:reference-data--ResourceCurationStatus:CREATED:",
+ "resourceLifecycleStatus": "namespace:reference-data--ResourceLifecycleStatus:LOADING:",
+ "resourceSecurityClassification": "namespace:reference-data--ResourceSecurityClassification:SomeUniqueResourceSecurityClassificationID:",
+ "ancestry": {
+ "parents": []
+ },
+ "meta": [
+ {
+ "kind": "CRS",
+ "name": "NAD27 * OGP-Usa Conus / North Dakota South [32021,15851]",
+ "persistableReference": "{\"authCode\":{\"auth\":\"EPSG\",\"code\":\"32615\"},\"type\":\"LBC\",\"ver\":\"PE_10_3_1\",\"name\":\"WGS_1984_UTM_Zone_15N\",\"wkt\":\"PROJCS[\\\"WGS_1984_UTM_Zone_15N\\\",GEOGCS[\\\"GCS_WGS_1984\\\",DATUM[\\\"D_WGS_1984\\\",SPHEROID[\\\"WGS_1984\\\",6378137.0,298.257223563]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],PROJECTION[\\\"Transverse_Mercator\\\"],PARAMETER[\\\"False_Easting\\\",500000.0],PARAMETER[\\\"False_Northing\\\",0.0],PARAMETER[\\\"Central_Meridian\\\",-93.0],PARAMETER[\\\"Scale_Factor\\\",0.9996],PARAMETER[\\\"Latitude_Of_Origin\\\",0.0],UNIT[\\\"Meter\\\",1.0],AUTHORITY[\\\"EPSG\\\",32615]]\"}",
+ "coordinateReferenceSystemID": "namespace:reference-data--CoordinateReferenceSystem:EPSG.32615:",
+ "propertyNames": [
+ "KickOffPosition.X",
+ "KickOffPosition.Y"
+ ]
+ }
+ ],
+ "source": "Example Data Source",
+ "existenceKind": "namespace:reference-data--ExistenceKind:Prototype:",
+ "data": {
+ "Name": "Dataset X221/15",
+ "Description": "As originally delivered by ACME.com.",
+ "Source": "ACME.com",
+ "TotalSize": "13245217273",
+ "EncodingFormatTypeID": "namespace:reference-data--EncodingFormatType:UTF-8:",
+ "MimeType": "namespace:reference-data--MimeType:application/geo+json:",
+ "Endian": "BIG",
+ "DatasetProperties": {}
+ }
+ }
+ ]
+ }
+}
diff --git a/tests/plugin-unit-tests/data/manifests/schema_Manifest.1.0.0.json b/tests/plugin-unit-tests/data/manifests/schema_Manifest.1.0.0.json
new file mode 100644
index 0000000..facd766
--- /dev/null
+++ b/tests/plugin-unit-tests/data/manifests/schema_Manifest.1.0.0.json
@@ -0,0 +1,55 @@
+{
+ "x-osdu-license": "Copyright 2021, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$id": "https://schema.osdu.opengroup.org/json/manifest/Manifest.1.0.0.json",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "title": "Load Manifest Schema",
+ "description": "Load manifest applicable for all types defined as 'kind', i.e. registered as schemas with the Schema Service. It supports loading of individual 'records' of any group-type or combinations. The load sequence follows a well-defined sequence. The 'ReferenceData' array is processed first (if populated). The 'MasterData' array is processed second (if populated) second. The 'Data' structure is processed last (if populated). Inside the 'Data' property the 'Files' array is processed first, followed by the 'WorkProductComponents' array, the 'WorkProduct' is processed last. Any arrays are ordered. should there be interdependencies, the dependent items must be placed behind their relationship targets, e.g. a master-data Well record must placed in the 'MasterData' array before its Wellbores.",
+ "type": "object",
+ "properties": {
+ "kind": {
+ "description": "The schema identification for the manifest record following the pattern {Namespace}:{Source}:{Type}:{VersionMajor}.{VersionMinor}.{VersionPatch}. The versioning scheme follows the semantic versioning, https://semver.org/.",
+ "title": "Manifest Kind",
+ "type": "string",
+ "pattern": "^[\\w\\-\\.]+:[\\w\\-\\.]+:[\\w\\-\\.\\/]+:[0-9]+.[0-9]+.[0-9]+$",
+ "example": "osdu:wks:Manifest:1.0.0"
+ },
+ "ReferenceData": {
+ "description": "Reference-data are submitted as an array of records.",
+ "type": "array",
+ "items": {
+ "$ref": "GenericReferenceData.1.0.0.json"
+ }
+ },
+ "MasterData": {
+ "description": "Master-data are submitted as an array of records.",
+ "type": "array",
+ "items": {
+ "$ref": "GenericMasterData.1.0.0.json"
+ }
+ },
+ "Data": {
+ "description": "Manifest schema for work-product, work-product-component, dataset ensembles. The items in 'Datasets' are processed first since they are referenced by 'WorkProductComponents' ('data.DatasetIDs[]' and 'data.Artefacts[].ResourceID'). The WorkProduct is processed last collecting the WorkProductComponents.",
+ "type": "object",
+ "properties": {
+ "WorkProduct": {
+ "description": "The work-product component capturing the work-product-component records belonging to this loading/ingestion transaction.",
+ "$ref": "GenericWorkProduct.1.0.0.json"
+ },
+ "WorkProductComponents": {
+ "description": "The list of work-product-components records. The record ids are internal surrogate keys enabling the association of work-product-component records with the work-product records.",
+ "type": "array",
+ "items": {
+ "$ref": "GenericWorkProductComponent.1.0.0.json"
+ }
+ },
+ "Datasets": {
+ "description": "The list of 'Datasets' or data containers holding the actual data. The record ids are usually internal surrogate keys enabling the association of dataset records with work-product-component records, namely via 'DatasetIDs' and 'Artefacts.ResourceID' (both referring to 'dataset' group-type entity types).",
+ "type": "array",
+ "items": {
+ "$ref": "GenericDataset.1.0.0.json"
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/tests/plugin-unit-tests/data/master/Wellbore.0.3.0.json b/tests/plugin-unit-tests/data/master/Wellbore.0.3.0.json
index b38869d..9f1bbd6 100644
--- a/tests/plugin-unit-tests/data/master/Wellbore.0.3.0.json
+++ b/tests/plugin-unit-tests/data/master/Wellbore.0.3.0.json
@@ -7,10 +7,13 @@
},
"$schema": "https://schema.osdu.opengroup.org/json/master-data/Wellbore.1.0.0.json",
"$filename": "load_Wellbore.1.0.0_350112350400.json",
- "manifest": [
- {
+ "manifest": {
+ "kind": "test:test:Manifest:1.0.0",
+ "ReferenceData": [],
+ "MasterData": [
+ {
"id": "opendes:master-data/Wellbore:350112350400",
- "kind": "opendes:osdu:Wellbore:0.3.0",
+ "kind": "opendes:osdu:TestMaster:0.3.0",
"groupType": "master-data",
"version": 1,
"acl": {
@@ -40,98 +43,12 @@
"existenceKind": "srn:opendes:reference-data/ExistenceKind:Active:",
"licenseState": "srn:opendes:reference-data/LicenseState:Unlicensed:",
"data": {
- "FacilityTypeID": "srn:opendes:reference-data/FacilityType:Wellbore:",
- "FacilityOperator": [
- {
- "FacilityOperatorOrganisationID": "srn:opendes:master-data/Organisation:CONTINENTAL RESOURCES INC:"
- }
- ],
- "DataSourceOrganisationID": "srn:opendes:master-data/Organisation:Oklahoma Corporation Commission:",
- "SpatialLocation": [
- {
- "Coordinates": [
- {
- "x": -98.580887,
- "y": 35.6381829999999
- }
- ],
- "SpatialGeometryTypeID": "srn:opendes:reference-data/SpatialGeometryType:Point:",
- "VerticalCRSID": "srn:opendes:reference-data/VerticalCRS:MSL:",
- "HorizontalCRSID": "srn:opendes:reference-data/HorizontalCRS:NAD27:",
- "HeightAboveGroundLevelUOMID": "srn:opendes:reference-data/UnitOfMeasure:ft[US]:"
- }
- ],
- "OperatingEnvironmentID": "srn:opendes:reference-data/OperatingEnvironment:onshore:",
- "FacilityName": "IRETA 1-4-9XH",
- "FacilityNameAlias": [
- {
- "AliasName": " IRETA 1-4-9XH",
- "AliasNameTypeID": "srn:opendes:reference-data/AliasNameType:Name:"
- },
- {
- "AliasName": "350112350400",
- "AliasNameTypeID": "srn:opendes:reference-data/AliasNameType:UWBI:"
- }
- ],
- "FacilityEvent": [
- {
- "FacilityEventTypeID": "srn:opendes:reference-data/FacilityEventType:SPUD:",
- "EffectiveDateTime": "2015-03-11T00:00:00-05:00"
- },
- {
- "FacilityEventTypeID": "srn:opendes:reference-data/FacilityEventType:DRILLING FINISH:",
- "EffectiveDateTime": "2015-05-18T00:00:00-06:00"
- }
- ],
- "WellID": "srn:opendes:master-data/Well:3501123504:",
- "SequenceNumber": 1,
- "VerticalMeasurements": [
- {
- "VerticalMeasurementID": "TD_1",
- "VerticalMeasurement": 0,
- "VerticalMeasurementTypeID": "srn:opendes:reference-data/VerticalMeasurementType:Total Depth:",
- "VerticalMeasurementPathID": "srn:opendes:reference-data/VerticalMeasurementPath:Measured Depth:",
- "VerticalMeasurementUnitOfMeasureID": "srn:opendes:reference-data/UnitOfMeasure:ft[US]:",
- "VerticalReferenceID": "Drill Floor"
- },
- {
- "VerticalMeasurementID": "TD_2",
- "VerticalMeasurement": 0,
- "VerticalMeasurementTypeID": "srn:opendes:reference-data/VerticalMeasurementType:Total Depth:",
- "VerticalMeasurementPathID": "srn:opendes:reference-data/VerticalMeasurementPath:True Vertical Depth:",
- "VerticalMeasurementUnitOfMeasureID": "srn:opendes:reference-data/UnitOfMeasure:ft[US]:",
- "VerticalReferenceID": "Drill Floor"
- },
- {
- "VerticalMeasurementID": "Elev_1",
- "VerticalMeasurement": 1636,
- "VerticalMeasurementTypeID": "srn:opendes:reference-data/VerticalMeasurementType:Drill Floor:",
- "VerticalMeasurementPathID": "srn:opendes:reference-data/VerticalMeasurementPath:Elevation:",
- "VerticalMeasurementUnitOfMeasureID": "srn:opendes:reference-data/UnitOfMeasure:ft[US]:",
- "VerticalCRSID": "srn:opendes:reference-data/VerticalCRS:MSL:"
- },
- {
- "VerticalMeasurementID": "Elev_2",
- "VerticalMeasurement": 1606,
- "VerticalMeasurementTypeID": "srn:opendes:reference-data/VerticalMeasurementType:Ground Level:",
- "VerticalMeasurementPathID": "srn:opendes:reference-data/VerticalMeasurementPath:Elevation:",
- "VerticalMeasurementUnitOfMeasureID": "srn:opendes:reference-data/UnitOfMeasure:ft[US]:",
- "VerticalCRSID": "srn:opendes:reference-data/VerticalCRS:MSL:"
- }
- ],
- "TrajectoryTypeID": "srn:opendes:reference-data/WellboreTrajectoryType:Horizontal:",
- "DefaultVerticalMeasurementID": "",
- "GeographicBottomHoleLocation": {
- "Coordinates": [
- {
- "x": -98.580887,
- "y": 35.6381829999999
- }
- ]
- }
- }
+ "SequenceNumber": 1
+ },
+ "schema": "test:test:GenericMasterData:1.0.0"
}
-
- ],
+ ],
+ "Data": {}
+ },
"WorkflowID": "foo"
}
diff --git a/tests/plugin-unit-tests/data/master/schema_GenericMasterData.1.0.0.json b/tests/plugin-unit-tests/data/master/schema_GenericMasterData.1.0.0.json
new file mode 100644
index 0000000..57e53f5
--- /dev/null
+++ b/tests/plugin-unit-tests/data/master/schema_GenericMasterData.1.0.0.json
@@ -0,0 +1,96 @@
+{
+ "$id": "https://schema.osdu.opengroup.org/json/master-data/GenericMasterData.1.0.0.json",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:master-data--GenericMasterData:1.0.0",
+ "title": "GenericMasterData",
+ "description": "An auto-generated placeholder schema representing master-data group-type records in data loading/ingestion/creation manifests. Do not use this kind for actual records.",
+ "type": "object",
+ "properties": {
+ "id": {
+ "description": "Previously called ResourceID or SRN which identifies this OSDU resource object without version.",
+ "title": "Entity ID",
+ "type": "string",
+ "example": "namespace:master-data--GenericMasterData:9ca8054c-bce6-5a3a-b51d-f216fb1085a5"
+ },
+ "kind": {
+ "description": "The schema identification for the OSDU resource object following the pattern {Namespace}:{Source}:{Type}:{VersionMajor}.{VersionMinor}.{VersionPatch}. The versioning scheme follows the semantic versioning, https://semver.org/.",
+ "title": "Entity Kind",
+ "type": "string",
+ "pattern": "^[\\w\\-\\.]+:[\\w\\-\\.]+:[\\w\\-\\.\\/]+:[0-9]+.[0-9]+.[0-9]+$",
+ "example": "osdu:wks:master-data--GenericMasterData:1.0.0"
+ },
+ "version": {
+ "description": "The version number of this OSDU resource; set by the framework.",
+ "title": "Version Number",
+ "type": "integer",
+ "format": "int64",
+ "example": 1562066009929332
+ },
+ "acl": {
+ "description": "The access control tags associated with this entity.",
+ "title": "Access Control List"
+ },
+ "legal": {
+ "description": "The entity's legal tags and compliance status. The actual contents associated with the legal tags is managed by the Compliance Service.",
+ "title": "Legal Tags"
+ },
+ "tags": {
+ "title": "Tag Dictionary",
+ "description": "A generic dictionary of string keys mapping to string value. Only strings are permitted as keys and values.",
+ "type": "object",
+ "additionalProperties": {
+ "type": "string"
+ },
+ "example": {
+ "NameOfKey": "String value"
+ }
+ },
+ "createTime": {
+ "description": "Timestamp of the time at which initial version of this OSDU resource object was created. Set by the System. The value is a combined date-time string in ISO-8601 given in UTC.",
+ "title": "Resource Object Creation DateTime",
+ "type": "string",
+ "format": "date-time",
+ "example": "2020-12-16T11:46:20.163Z"
+ },
+ "createUser": {
+ "title": "Resource Object Creation User Reference",
+ "description": "The user reference, which created the first version of this resource object. Set by the System.",
+ "type": "string",
+ "example": "some-user@some-company-cloud.com"
+ },
+ "modifyTime": {
+ "description": "Timestamp of the time at which this version of the OSDU resource object was created. Set by the System. The value is a combined date-time string in ISO-8601 given in UTC.",
+ "title": "Resource Object Version Creation DateTime",
+ "type": "string",
+ "format": "date-time",
+ "example": "2020-12-16T11:52:24.477Z"
+ },
+ "modifyUser": {
+ "title": "Resource Object Version Creation User Reference",
+ "description": "The user reference, which created this version of this resource object. Set by the System.",
+ "type": "string",
+ "example": "some-user@some-company-cloud.com"
+ },
+ "ancestry": {
+ "description": "The links to data, which constitute the inputs.",
+ "title": "Ancestry"
+ },
+ "meta": {
+ "description": "The Frame of Reference meta data section linking the named properties to self-contained definitions.",
+ "title": "Frame of Reference Meta Data",
+ "type": "array",
+ "items": {
+ }
+ },
+ "data": {
+ "type": "object",
+ "description": "The data block containing the kind-specific payload. The actual schema is defined by the 'kind', resolvable via the Schema Service."
+ }
+ },
+ "required": [
+ "id",
+ "kind",
+ "acl",
+ "legal"
+ ]
+}
diff --git a/tests/plugin-unit-tests/data/master/schema_TestMaster.json b/tests/plugin-unit-tests/data/master/schema_TestMaster.json
new file mode 100644
index 0000000..8e41586
--- /dev/null
+++ b/tests/plugin-unit-tests/data/master/schema_TestMaster.json
@@ -0,0 +1,13 @@
+{
+ "$id": "https://schema.osdu.opengroup.org/json/master-data/Wellbore.1.0.0.json",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "title": "Master",
+ "description": "A hole in the ground extending from a point at the earth's surface to the maximum point of penetration.",
+ "type": "object",
+ "properties": {
+ "SequenceNumber": {
+ "description": "A number that indicates the order in which wellbores were drilled.",
+ "type": "integer"
+ }
+ }
+}
diff --git a/tests/plugin-unit-tests/data/master/schema_Wellbore.3.0.0.json b/tests/plugin-unit-tests/data/master/schema_Wellbore.3.0.0.json
index 43f7e6e..3df9b5c 100644
--- a/tests/plugin-unit-tests/data/master/schema_Wellbore.3.0.0.json
+++ b/tests/plugin-unit-tests/data/master/schema_Wellbore.3.0.0.json
@@ -2,7 +2,7 @@
"x-osdu-license": "Copyright 2020, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
"$schema": "http://json-schema.org/draft-07/schema#",
"description": "A hole in the ground extending from a point at the earth's surface to the maximum point of penetration.",
- "additionalProperties": false,
+ "additionalProperties": true,
"title": "Wellbore",
"type": "object",
"definitions": {
@@ -82,7 +82,7 @@
"x-osdu-license": "Copyright 2020, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
"$schema": "http://json-schema.org/draft-07/schema#",
"description": "Legal meta data like legal tags, relevant other countries, legal status.",
- "additionalProperties": false,
+ "additionalProperties": true,
"title": "Legal Meta Data",
"type": "object",
"properties": {
@@ -118,7 +118,7 @@
"x-osdu-license": "Copyright 2020, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
"$schema": "http://json-schema.org/draft-07/schema#",
"description": "The access control tags associated with this entity.",
- "additionalProperties": false,
+ "additionalProperties": true,
"title": "Access Control List",
"type": "object",
"properties": {
diff --git a/tests/plugin-unit-tests/data/master/traversal_Wellbore.0.3.0.json b/tests/plugin-unit-tests/data/master/traversal_Wellbore.0.3.0.json
new file mode 100644
index 0000000..9a4dd21
--- /dev/null
+++ b/tests/plugin-unit-tests/data/master/traversal_Wellbore.0.3.0.json
@@ -0,0 +1,41 @@
+[
+ {
+ "entity": {
+ "id": "opendes:master-data/Wellbore:350112350400",
+ "kind": "opendes:osdu:TestMaster:0.3.0",
+ "groupType": "master-data",
+ "version": 1,
+ "acl": {
+ "owners": [
+ "data.default.viewers@opendes.osdu-gcp.go3-nrg.projects.epam.com"
+ ],
+ "viewers": [
+ "data.default.owners@opendes.osdu-gcp.go3-nrg.projects.epam.com"
+ ]
+ },
+ "legal": {
+ "legaltags": [
+ "opendes-demo-legaltag"
+ ],
+ "otherRelevantDataCountries": [
+ "srn:opendes:master-data/GeoPoliticalEntity:USA:"
+ ],
+ "status": "srn:opendes:reference-data/LegalStatus:public:1111"
+ },
+ "resourceHostRegionIDs": [
+ "srn:opendes:reference-data/OSDURegion:US-EAST:"
+ ],
+ "resourceObjectCreationDateTime": "2020-10-16T11:14:45-05:00",
+ "resourceVersionCreationDateTime": "2020-10-16T11:14:45-05:00",
+ "resourceSecurityClassification": "srn:opendes:reference-data/ResourceSecurityClassification:public:",
+ "source": "srn:opendes:master-data/Organisation:Oklahoma Corporation Commission:",
+ "existenceKind": "srn:opendes:reference-data/ExistenceKind:Active:",
+ "licenseState": "srn:opendes:reference-data/LicenseState:Unlicensed:",
+ "data": {
+ "SequenceNumber": 1
+ },
+ "schema": "test:test:GenericMasterData:1.0.0"
+ },
+ "schema": "GenericMasterData.1.0.0.json"
+ }
+]
diff --git a/tests/plugin-unit-tests/data/workProduct/SeismicTraceData.json b/tests/plugin-unit-tests/data/workProduct/SeismicTraceData.json
index 9d58f2f..4a7da06 100644
--- a/tests/plugin-unit-tests/data/workProduct/SeismicTraceData.json
+++ b/tests/plugin-unit-tests/data/workProduct/SeismicTraceData.json
@@ -7,8 +7,11 @@
},
"$schema": "https://schema.osdu.opengroup.org/json/master-data/Wellbore.1.0.0.json",
"$filename": "load_Wellbore.1.0.0_350112350400.json",
- "manifest": [
- {
+ "manifest":
+ {
+ "ReferenceData": [],
+ "MasterData": [],
+ "Data": {
"WorkProduct": {
"kind": "opendes:osdu:WorkProduct:1.0.0",
"groupType": "work-product",
@@ -351,7 +354,7 @@
]
}
],
- "Files": [
+ "Datasets": [
{
"kind": "opendes:osdu:File:1.0.0",
"groupType": "file",
@@ -376,8 +379,12 @@
"data": {
"Name": "RequiredName",
"SchemaFormatTypeID": "srn:opendes:reference-data/SchemaFormatType:SEG-Y Seismic Trace Data:",
- "PreLoadFilePath": "test",
- "FileSource": "",
+ "DatasetProperties": {
+ "FileSourceInfo": {
+ "PreloadFilePath": "test",
+ "FileSource": ""
+ }
+ },
"FileSize": 277427976,
"EncodingFormatTypeID": "srn:opendes:reference-data/EncodingFormatType:segy:",
"Endian": "BIG",
@@ -419,6 +426,6 @@
}
]
}
- ],
+ },
"WorkflowID": "foo"
}
diff --git a/tests/plugin-unit-tests/data/workProduct/record_SeismicTraceData.json b/tests/plugin-unit-tests/data/workProduct/record_SeismicTraceData.json
index 4d47dbc..1eb8d59 100644
--- a/tests/plugin-unit-tests/data/workProduct/record_SeismicTraceData.json
+++ b/tests/plugin-unit-tests/data/workProduct/record_SeismicTraceData.json
@@ -22,8 +22,12 @@
"data": {
"Name": "RequiredName",
"SchemaFormatTypeID": "srn:opendes:reference-data/SchemaFormatType:SEG-Y Seismic Trace Data:",
- "PreLoadFilePath": "test",
- "FileSource": "/test/source_file",
+ "DatasetProperties": {
+ "FileSourceInfo": {
+ "FileSource": "/test/source_file",
+ "PreloadFilePath": "test"
+ }
+ },
"FileSize": 277427976,
"EncodingFormatTypeID": "srn:opendes:reference-data/EncodingFormatType:segy:",
"Endian": "BIG",
diff --git a/tests/plugin-unit-tests/data/workProduct/schema_File.1.0.0.json b/tests/plugin-unit-tests/data/workProduct/schema_File.1.0.0.json
index ba5e926..ee12e41 100644
--- a/tests/plugin-unit-tests/data/workProduct/schema_File.1.0.0.json
+++ b/tests/plugin-unit-tests/data/workProduct/schema_File.1.0.0.json
@@ -2,7 +2,7 @@
"x-osdu-license": "Copyright 2020, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
"$schema": "http://json-schema.org/draft-07/schema#",
"description": "The generic file entity.",
- "additionalProperties": false,
+ "additionalProperties": true,
"title": "File",
"type": "object",
"definitions": {
@@ -82,7 +82,7 @@
"x-osdu-license": "Copyright 2020, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
"$schema": "http://json-schema.org/draft-07/schema#",
"description": "Legal meta data like legal tags, relevant other countries, legal status.",
- "additionalProperties": false,
+ "additionalProperties": true,
"title": "Legal Meta Data",
"type": "object",
"properties": {
@@ -118,7 +118,7 @@
"x-osdu-license": "Copyright 2020, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
"$schema": "http://json-schema.org/draft-07/schema#",
"description": "The access control tags associated with this entity.",
- "additionalProperties": false,
+ "additionalProperties": true,
"title": "Access Control List",
"type": "object",
"properties": {
diff --git a/tests/plugin-unit-tests/data/workProduct/schema_SeismicTraceData.1.0.0.json b/tests/plugin-unit-tests/data/workProduct/schema_SeismicTraceData.1.0.0.json
index 7a7e2bd..718e048 100644
--- a/tests/plugin-unit-tests/data/workProduct/schema_SeismicTraceData.1.0.0.json
+++ b/tests/plugin-unit-tests/data/workProduct/schema_SeismicTraceData.1.0.0.json
@@ -2,7 +2,7 @@
"x-osdu-license": "Copyright 2020, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
"$schema": "http://json-schema.org/draft-07/schema#",
"description": "A single logical dataset containing seismic samples.",
- "additionalProperties": false,
+ "additionalProperties": true,
"title": "SeismicTraceData",
"type": "object",
"definitions": {
@@ -82,7 +82,7 @@
"x-osdu-license": "Copyright 2020, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
"$schema": "http://json-schema.org/draft-07/schema#",
"description": "Legal meta data like legal tags, relevant other countries, legal status.",
- "additionalProperties": false,
+ "additionalProperties": true,
"title": "Legal Meta Data",
"type": "object",
"properties": {
@@ -118,7 +118,7 @@
"x-osdu-license": "Copyright 2020, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
"$schema": "http://json-schema.org/draft-07/schema#",
"description": "The access control tags associated with this entity.",
- "additionalProperties": false,
+ "additionalProperties": true,
"title": "Access Control List",
"type": "object",
"properties": {
diff --git a/tests/plugin-unit-tests/data/workProduct/schema_WorkProduct.1.0.0.json b/tests/plugin-unit-tests/data/workProduct/schema_WorkProduct.1.0.0.json
index f2c69e4..ab49b4e 100644
--- a/tests/plugin-unit-tests/data/workProduct/schema_WorkProduct.1.0.0.json
+++ b/tests/plugin-unit-tests/data/workProduct/schema_WorkProduct.1.0.0.json
@@ -2,7 +2,7 @@
"x-osdu-license": "Copyright 2020, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
"$schema": "http://json-schema.org/draft-07/schema#",
"description": "A collection of work product components such as might be produced by a business activity and which is delivered to the data platform for loading.",
- "additionalProperties": false,
+ "additionalProperties": true,
"title": "WorkProduct",
"type": "object",
"definitions": {
@@ -82,7 +82,7 @@
"x-osdu-license": "Copyright 2020, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
"$schema": "http://json-schema.org/draft-07/schema#",
"description": "Legal meta data like legal tags, relevant other countries, legal status.",
- "additionalProperties": false,
+ "additionalProperties": true,
"title": "Legal Meta Data",
"type": "object",
"properties": {
@@ -118,7 +118,7 @@
"x-osdu-license": "Copyright 2020, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
"$schema": "http://json-schema.org/draft-07/schema#",
"description": "The access control tags associated with this entity.",
- "additionalProperties": false,
+ "additionalProperties": true,
"title": "Access Control List",
"type": "object",
"properties": {
diff --git a/tests/plugin-unit-tests/data/workProduct/traversal_SeismicTraceData.1.0.0.json b/tests/plugin-unit-tests/data/workProduct/traversal_SeismicTraceData.1.0.0.json
new file mode 100644
index 0000000..a8ba5d6
--- /dev/null
+++ b/tests/plugin-unit-tests/data/workProduct/traversal_SeismicTraceData.1.0.0.json
@@ -0,0 +1,420 @@
+[
+ {
+ "entity": {
+ "kind": "opendes:osdu:WorkProduct:1.0.0",
+ "groupType": "work-product",
+ "acl": {
+ "owners": [
+ "ownergroup@testcompany.com"
+ ],
+ "viewers": [
+ "viewgroup@testcompany.com"
+ ]
+ },
+ "legal": {
+ "legaltags": [
+ "legaltag"
+ ],
+ "otherRelevantDataCountries": [
+ "NO",
+ "US"
+ ]
+ },
+ "resourceSecurityClassification": "srn:opendes:reference-data/ResourceSecurityClassification:RESTRICTED:",
+ "data": {
+ "Name": "ST0202R08_PS_PSDM_RAW_PP_TIME.MIG_RAW",
+ "Description": "Seismic Trace Data"
+ },
+ "ComponentsAssociativeIDs": [
+ "wpc-1"
+ ]
+ },
+ "schema": "GenericWorkProduct.1.0.0.json"
+ },
+ {
+ "entity": {
+ "kind": "opendes:osdu:SeismicTraceData:1.0.0",
+ "groupType": "work-product-component",
+ "acl": {
+ "owners": [
+ "ownergroup@testcompany.com"
+ ],
+ "viewers": [
+ "viewgroup@testcompany.com"
+ ]
+ },
+ "legal": {
+ "legaltags": [
+ "legaltag"
+ ],
+ "otherRelevantDataCountries": [
+ "NO",
+ "US"
+ ]
+ },
+ "resourceSecurityClassification": "srn:opendes:reference-data/ResourceSecurityClassification:RESTRICTED:",
+ "meta": [
+ {
+ "kind": "Unit",
+ "name": "ms",
+ "persistableReference": "{\"abcd\":{\"a\":0.0,\"b\":0.001,\"c\":1.0,\"d\":0.0},\"symbol\":\"ms\",\"baseMeasurement\":{\"ancestry\":\"T\",\"type\":\"UM\"},\"type\":\"UAD\"}",
+ "unitOfMeasureID": "srn:opendes:reference-data/UnitOfMeasure:Energistics_UoM_ms:",
+ "propertyNames": [
+ "StartTime",
+ "EndTime"
+ ]
+ },
+ {
+ "kind": "Unit",
+ "name": "Amplitude",
+ "persistableReference": "{\"abcd\":{\"a\":0.0,\"b\":1.0,\"c\":1.0,\"d\":0.0},\"symbol\":\"Euc\",\"baseMeasurement\":{\"ancestry\":\"1\",\"type\":\"UM\"},\"type\":\"UAD\"}",
+ "unitOfMeasureID": "srn:opendes:reference-data/UnitOfMeasure:Energistics_UoM_Euc:",
+ "propertyNames": [
+ "RangeAmplitudeMax",
+ "RangeAmplitudeMin"
+ ]
+ }
+ ],
+ "data": {
+ "Name": "ST0202R08_PS_PSDM_RAW_PP_TIME.MIG_RAW",
+ "Description": "Seismic Trace Data",
+ "SpatialArea": {
+ "AsIngestedCoordinates": {
+ "type": "AnyCrsFeatureCollection",
+ "CoordinateReferenceSystemID": "srn:opendes:reference-data/CoordinateReferenceSystem:BoundCRS.SLB.23031.1613:",
+ "persistableReferenceCRS": "{\"lateBoundCRS\":{\"wkt\":\"PROJCS[\\\"ED_1950_UTM_Zone_31N\\\",GEOGCS[\\\"GCS_European_1950\\\",DATUM[\\\"D_European_1950\\\",SPHEROID[\\\"International_1924\\\",6378388.0,297.0]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],PROJECTION[\\\"Transverse_Mercator\\\"],PARAMETER[\\\"False_Easting\\\",500000.0],PARAMETER[\\\"False_Northing\\\",0.0],PARAMETER[\\\"Central_Meridian\\\",3.0],PARAMETER[\\\"Scale_Factor\\\",0.9996],PARAMETER[\\\"Latitude_Of_Origin\\\",0.0],UNIT[\\\"Meter\\\",1.0],AUTHORITY[\\\"EPSG\\\",23031]]\",\"ver\":\"PE_10_3_1\",\"name\":\"ED_1950_UTM_Zone_31N\",\"authCode\":{\"auth\":\"EPSG\",\"code\":\"23031\"},\"type\":\"LBC\"},\"singleCT\":{\"wkt\":\"GEOGTRAN[\\\"ED_1950_To_WGS_1984_24\\\",GEOGCS[\\\"GCS_European_1950\\\",DATUM[\\\"D_European_1950\\\",SPHEROID[\\\"International_1924\\\",6378388.0,297.0]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],GEOGCS[\\\"GCS_WGS_1984\\\",DATUM[\\\"D_WGS_1984\\\",SPHEROID[\\\"WGS_1984\\\",6378137.0,298.257223563]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],METHOD[\\\"Position_Vector\\\"],PARAMETER[\\\"X_Axis_Translation\\\",-90.365],PARAMETER[\\\"Y_Axis_Translation\\\",-101.13],PARAMETER[\\\"Z_Axis_Translation\\\",-123.384],PARAMETER[\\\"X_Axis_Rotation\\\",0.333],PARAMETER[\\\"Y_Axis_Rotation\\\",0.077],PARAMETER[\\\"Z_Axis_Rotation\\\",0.894],PARAMETER[\\\"Scale_Difference\\\",1.994],AUTHORITY[\\\"EPSG\\\",1613]]\",\"ver\":\"PE_10_3_1\",\"name\":\"ED_1950_To_WGS_1984_24\",\"authCode\":{\"auth\":\"EPSG\",\"code\":\"1613\"},\"type\":\"ST\"},\"ver\":\"PE_10_3_1\",\"name\":\"ED50 * EPSG-Nor S62 2001 / UTM zone 31N [23031,1613]\",\"authCode\":{\"auth\":\"SLB\",\"code\":\"23031024\"},\"type\":\"EBC\"}",
+ "features": [
+ {
+ "type": "AnyCrsFeature",
+ "properties": null,
+ "geometry": {
+ "type": "AnyCrsPolygon",
+ "coordinates": [
+ [
+ [
+ 438727.125,
+ 6475514.5
+ ],
+ [
+ 439888.34375,
+ 6480172.0
+ ],
+ [
+ 432562.59375,
+ 6481998.5
+ ],
+ [
+ 431401.375,
+ 6477341.0
+ ],
+ [
+ 438727.125,
+ 6475514.5
+ ]
+ ]
+ ]
+ }
+ }
+ ]
+ },
+ "Wgs84Coordinates": {
+ "type": "FeatureCollection",
+ "features": [
+ {
+ "type": "Feature",
+ "properties": null,
+ "geometry": {
+ "type": "Polygon",
+ "coordinates": [
+ [
+ [
+ 1.9496878,
+ 58.4141503
+ ],
+ [
+ 1.9683366,
+ 58.4561357
+ ],
+ [
+ 1.8422866,
+ 58.4714655
+ ],
+ [
+ 1.8237804,
+ 58.4294624
+ ],
+ [
+ 1.9496878,
+ 58.4141503
+ ]
+ ]
+ ]
+ }
+ }
+ ]
+ },
+ "OperationsApplied": [
+ "AsIngestedCoordinates converted to Wgs84Coordinates: Input CRS EPSG 23031 (ED50 / UTM zone 31N) to Target CRS EPSG 4326 (WGS84) using CT EPSG 1613 (ED50 to WGS 84 (24) - Norway - offshore south of 62°N - North Sea.)"
+ ],
+ "SpatialParameterTypeID": "srn:opendes:reference-data/SpatialParameterType:Outline:",
+ "SpatialGeometryTypeID": "srn:opendes:reference-data/SpatialGeometryType:Polygon:"
+ },
+ "LiveTraceOutline": {
+ "AsIngestedCoordinates": {
+ "type": "AnyCrsFeatureCollection",
+ "CoordinateReferenceSystemID": "srn:opendes:reference-data/CoordinateReferenceSystem:BoundCRS.SLB.23031.1613:",
+ "persistableReferenceCRS": "{\"lateBoundCRS\":{\"wkt\":\"PROJCS[\\\"ED_1950_UTM_Zone_31N\\\",GEOGCS[\\\"GCS_European_1950\\\",DATUM[\\\"D_European_1950\\\",SPHEROID[\\\"International_1924\\\",6378388.0,297.0]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],PROJECTION[\\\"Transverse_Mercator\\\"],PARAMETER[\\\"False_Easting\\\",500000.0],PARAMETER[\\\"False_Northing\\\",0.0],PARAMETER[\\\"Central_Meridian\\\",3.0],PARAMETER[\\\"Scale_Factor\\\",0.9996],PARAMETER[\\\"Latitude_Of_Origin\\\",0.0],UNIT[\\\"Meter\\\",1.0],AUTHORITY[\\\"EPSG\\\",23031]]\",\"ver\":\"PE_10_3_1\",\"name\":\"ED_1950_UTM_Zone_31N\",\"authCode\":{\"auth\":\"EPSG\",\"code\":\"23031\"},\"type\":\"LBC\"},\"singleCT\":{\"wkt\":\"GEOGTRAN[\\\"ED_1950_To_WGS_1984_24\\\",GEOGCS[\\\"GCS_European_1950\\\",DATUM[\\\"D_European_1950\\\",SPHEROID[\\\"International_1924\\\",6378388.0,297.0]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],GEOGCS[\\\"GCS_WGS_1984\\\",DATUM[\\\"D_WGS_1984\\\",SPHEROID[\\\"WGS_1984\\\",6378137.0,298.257223563]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],METHOD[\\\"Position_Vector\\\"],PARAMETER[\\\"X_Axis_Translation\\\",-90.365],PARAMETER[\\\"Y_Axis_Translation\\\",-101.13],PARAMETER[\\\"Z_Axis_Translation\\\",-123.384],PARAMETER[\\\"X_Axis_Rotation\\\",0.333],PARAMETER[\\\"Y_Axis_Rotation\\\",0.077],PARAMETER[\\\"Z_Axis_Rotation\\\",0.894],PARAMETER[\\\"Scale_Difference\\\",1.994],AUTHORITY[\\\"EPSG\\\",1613]]\",\"ver\":\"PE_10_3_1\",\"name\":\"ED_1950_To_WGS_1984_24\",\"authCode\":{\"auth\":\"EPSG\",\"code\":\"1613\"},\"type\":\"ST\"},\"ver\":\"PE_10_3_1\",\"name\":\"ED50 * EPSG-Nor S62 2001 / UTM zone 31N [23031,1613]\",\"authCode\":{\"auth\":\"SLB\",\"code\":\"23031024\"},\"type\":\"EBC\"}",
+ "features": [
+ {
+ "type": "AnyCrsFeature",
+ "properties": null,
+ "geometry": {
+ "type": "AnyCrsPolygon",
+ "coordinates": [
+ [
+ [
+ 438727.125,
+ 6475514.5
+ ],
+ [
+ 439888.34375,
+ 6480172.0
+ ],
+ [
+ 432562.59375,
+ 6481998.5
+ ],
+ [
+ 431401.375,
+ 6477341.0
+ ],
+ [
+ 438727.125,
+ 6475514.5
+ ]
+ ]
+ ]
+ }
+ }
+ ]
+ },
+ "Wgs84Coordinates": {
+ "type": "FeatureCollection",
+ "features": [
+ {
+ "type": "Feature",
+ "properties": null,
+ "geometry": {
+ "type": "Polygon",
+ "coordinates": [
+ [
+ [
+ 1.9496878,
+ 58.4141503
+ ],
+ [
+ 1.9683366,
+ 58.4561357
+ ],
+ [
+ 1.8422866,
+ 58.4714655
+ ],
+ [
+ 1.8237804,
+ 58.4294624
+ ],
+ [
+ 1.9496878,
+ 58.4141503
+ ]
+ ]
+ ]
+ }
+ }
+ ]
+ },
+ "OperationsApplied": [
+ "AsIngestedCoordinates converted to Wgs84Coordinates: Input CRS EPSG 23031 (ED50 / UTM zone 31N) to Target CRS EPSG 4326 (WGS84) using CT EPSG 1613 (ED50 to WGS 84 (24) - Norway - offshore south of 62°N - North Sea.)"
+ ],
+ "SpatialParameterTypeID": "srn:opendes:reference-data/SpatialParameterType:Outline:",
+ "SpatialGeometryTypeID": "srn:opendes:reference-data/SpatialGeometryType:Polygon:"
+ },
+ "PrincipalAcquisitionProjectID": "srn:opendes:master-data/SeismicAcquisitionProject:ST0202R08:",
+ "ProcessingProjectID": "srn:opendes:master-data/SeismicProcessingProject:ST0202R08:",
+ "SeismicTraceDataDimensionalityTypeID": "srn:opendes:reference-data/SeismicTraceDataDimensionalityType:3D:",
+ "SeismicDomainTypeID": "srn:opendes:reference-data/SeismicDomainType:Time:",
+ "SeismicMigrationTypeID": "srn:opendes:reference-data/SeismicMigrationType:Prestack Depth - Kirchhoff:",
+ "SeismicStackingTypeID": "srn:opendes:reference-data/SeismicStackingType:Full:",
+ "SeismicFilteringTypeID": "srn:opendes:reference-data/SeismicFilteringType:Tau-P:",
+ "Phase": "0",
+ "Polarity": "Normal",
+ "SampleInterval": 4.0,
+ "SampleCount": 1126,
+ "Difference": false,
+ "StartTime": 0.0,
+ "EndTime": 4500.0,
+ "TraceCount": 58479,
+ "TraceLength": 4500.0,
+ "TraceDomainUOM": "srn:opendes:reference-data/UnitOfMeasure:ms:",
+ "InlineMin": 9985,
+ "InlineMax": 10369,
+ "CrosslineMin": 1932,
+ "CrosslineMax": 2536,
+ "InlineIncrement": 2,
+ "CrosslineIncrement": 2,
+ "Precision": {
+ "WordFormat": "srn:opendes:reference-data/WordFormatType:IBM_FLOAT:",
+ "WordWidth": 4
+ },
+ "ProcessingParameters": [
+ {
+ "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:REFORMAT:"
+ },
+ {
+ "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:NAVIGATION MERGE:"
+ },
+ {
+ "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:TRACE EDITING:"
+ },
+ {
+ "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:DESIGNATURE/ZEROPHASE:"
+ },
+ {
+ "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:TIDAL STATICS:"
+ },
+ {
+ "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:RESAMPLE:",
+ "ProcessingParameterValue": "4 MS"
+ },
+ {
+ "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:GEOPHONE MATCHING CORRECTION:"
+ },
+ {
+ "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:GEOPHONE ORIENTATION CORRECTION:"
+ },
+ {
+ "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:ROTATION TO RADIAL:"
+ },
+ {
+ "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:NOISE BAND EDITING:"
+ },
+ {
+ "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:RECEIVER SHEAR STATIC CORRECTION:"
+ },
+ {
+ "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:TAUP DECONVOLUTION:"
+ },
+ {
+ "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:3D KIRCHHOFF DEPTH MIGRATION:"
+ },
+ {
+ "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:VEL ANALYSIS AND RMO CORRECTION:",
+ "ProcessingParameterValue": "200M X 200M"
+ },
+ {
+ "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:CONVERSION TO PP TWT:"
+ },
+ {
+ "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:PRE-RADON MILD DIP FILTER:"
+ },
+ {
+ "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:PARABOLIC RADON TRANSFORM DEMULTIPLE:"
+ },
+ {
+ "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:ANGLE MUTE:",
+ "ProcessingParameterValue": "3-37 DEGREES"
+ },
+ {
+ "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:STACK:"
+ },
+ {
+ "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:OUTPUT SEGY:"
+ }
+ ],
+ "TextualFileHeader": [
+ "C 1 CLIENT=STATOIL, PROCESSED BY=WESTERNGECO C 2 LINE ST0202D04-9985 C 3 SURVEY:ST0202 , AREA: VOLVE 15/9 C 4 DATAFORMAT: SEG-Y, DATE:02012008 C 5 DATATYPE: RAW PS PSDM STACK IN PP TIME C 6 DATA SHOT BY/VESSEL:GECO ANGLER, CABLE LENGTH:6000 M NO OF GROUPS: 240x4 C 7 NO OF CABLES 2, SOURCE:2, POP INTERVAL 25.0 M (FLIP-FLOP) C 8 RCV LINE SPACING: 400 M, SRC LINE SPACING: 100 M, RECORD LENGTH: 10.2 S C 9 GEODETIC DATUM: ED-50, SPHEROID: INTERNAT., PROJECTION: UTM C10 CENTR. MERID.: 03, UTM ZONE: 31 N, FALSE EASTING: 500000 C11 FIRST SAMPLE= 0, LAST SAMPLE= 4500, SAMPLE INTERVAL= 4 MS C12 DIST. BETWEEN INLINES=25.0M, XLINES=25.0M BIN SIZE (I x X):12.5M x 12.5M C13 INLINE DIRECTION (GRID): 284.0000000 DEGREES (CLOCKWISE FROM NORTH); C14 X-LINE DIRECTION (GRID): 014.0000000 DEGREES (CLOCKWISE FROM NORTH); C15 ORIGO(1.1) UTMX:431955.70E; UTMY:6348582.15N ; C16 DATA RANGE INLINES=9985-10369 (EVERY 2ND), X-LINES=1932-2536 (EVERY 2ND) C17 CORNER1:3D INLINE 9985, 3D XLINE 1932, UTM-X 438727.0, UTM-Y 6475514.4 C18 CORNER2:3D INLINE 9985, 3D XLINE 2536, UTM-X 431401.3, UTM-Y 6477341.0 C19 CORNER3:3D INLINE 10369, 3D XLINE 2536, UTM-X 432562.5, UTM-Y 6481998.4 C20 CORNER4:3D INLINE 10369, 3D XLINE 1932, UTM-X 439888.3, UTM-Y 6480171.9 C21 LIVE DATA POLYGON: (9985,1932);(9985,2536);(10369,2536);(10369,1932); C22 NAVIGATION SOURCE: P1/90 UKOOA BIN CENTER CELL GRID, C23 PROCESSING SEQUENCE: C24 1) REFORMAT 2) NAVIGATION MERGE 3) TRACE EDITING 4) DESIGNATURE/ZEROPHASE C25 5) TIDAL STATICS 6) RESAMPLE 4MS 7) GEOPHONE MATCHING CORRECTION C26 8) GEOPHONE ORIENTATION CORRECTION 9) ROTATION TO RADIAL C27 10) NOISE BAND EDITING 11) RECEIVER SHEAR STATIC CORRECTION C28 12) TAUP DECONVOLUTION 13) 3D KIRCHHOFF DEPTH MIGRATION C29 14) VEL ANALYSIS AND RMO CORRECTION 200M X 200M 15) CONVERSION TO PP TWT C30 16) PRE-RADON MILD DIP FILTER 17) PARABOLIC RADON TRANSFORM DEMULTIPLE C31 18) 3-37 DEGREE ANGLE MUTE 19) STACK 20) OUTPUT TO SEGY FORMAT C32 C33 C34 C35 HEADER WORD POSITIONS: C36 INLINE: 189-192 ; X-LINE: 193-196; C37 BINX (CDPX): 181-184, BINY (CDPY): 185-188, C38 MERID.: 3.0E, SPHEROID: INT.; ROTATION (AMS): 1245600000, C39 A POSITIVE SAMPLE CORRESPONDS TO A INCREASE IN ACOUSTIC IMPEDANCE. C40 END EBCDIC. "
+ ],
+ "RangeAmplitudeMax": 0.07441109418869019,
+ "RangeAmplitudeMin": -0.10446560382843018
+ },
+ "AssociativeID": "wpc-1",
+ "FileAssociativeIDs": [
+ "f-1"
+ ]
+ },
+ "schema": "GenericWorkProductComponent.1.0.0.json"
+ },
+ {
+ "entity": {
+ "kind": "opendes:osdu:File:1.0.0",
+ "groupType": "file",
+ "acl": {
+ "owners": [
+ "ownergroup@testcompany.com"
+ ],
+ "viewers": [
+ "viewgroup@testcompany.com"
+ ]
+ },
+ "legal": {
+ "legaltags": [
+ "legaltag"
+ ],
+ "otherRelevantDataCountries": [
+ "NO",
+ "US"
+ ]
+ },
+ "resourceSecurityClassification": "srn:opendes:reference-data/ResourceSecurityClassification:RESTRICTED:",
+ "data": {
+ "Name": "RequiredName",
+ "SchemaFormatTypeID": "srn:opendes:reference-data/SchemaFormatType:SEG-Y Seismic Trace Data:",
+ "DatasetProperties": {
+ "FileSourceInfo": {
+ "FileSource": "",
+ "PreloadFilePath": "test"
+ }
+ },
+ "FileSize": 277427976,
+ "EncodingFormatTypeID": "srn:opendes:reference-data/EncodingFormatType:segy:",
+ "Endian": "BIG",
+ "Checksum": "c9df7234d5d0a7a2c2676ee2e2aa48b6",
+ "VectorHeaderMapping": [
+ {
+ "KeyName": "srn:opendes:reference-data/HeaderKeyName:INLINE:",
+ "WordFormat": "srn:opendes:reference-data/WordFormatType:INT:",
+ "WordWidth": 4,
+ "Position": 189
+ },
+ {
+ "KeyName": "srn:opendes:reference-data/HeaderKeyName:CROSSLINE:",
+ "WordFormat": "srn:opendes:reference-data/WordFormatType:INT:",
+ "WordWidth": 4,
+ "Position": 193
+ },
+ {
+ "KeyName": "srn:opendes:reference-data/HeaderKeyName:CMPX:",
+ "WordFormat": "srn:opendes:reference-data/WordFormatType:INT:",
+ "WordWidth": 4,
+ "Position": 181,
+ "UoM": "srn:opendes:reference-data/UnitOfMeasure:M:",
+ "ScalarIndicator": "OVERRIDE",
+ "ScalarOverride": 100.0
+ },
+ {
+ "KeyName": "srn:opendes:reference-data/HeaderKeyName:CMPY:",
+ "WordFormat": "srn:opendes:reference-data/WordFormatType:INT:",
+ "WordWidth": 4,
+ "Position": 185,
+ "UoM": "srn:opendes:reference-data/UnitOfMeasure:M:",
+ "ScalarIndicator": "OVERRIDE",
+ "ScalarOverride": 100.0
+ }
+ ]
+ },
+ "AssociativeID": "f-1"
+ },
+ "schema": "GenericDataset.1.0.0.json"
+ }
+]
diff --git a/tests/plugin-unit-tests/file_paths.py b/tests/plugin-unit-tests/file_paths.py
index 295877c..e236831 100644
--- a/tests/plugin-unit-tests/file_paths.py
+++ b/tests/plugin-unit-tests/file_paths.py
@@ -17,17 +17,25 @@ import os
DATA_PATH_PREFIX = f"{os.path.dirname(__file__)}/data"
+MANIFEST_GENERIC_SCHEMA_PATH = f"{DATA_PATH_PREFIX}/manifests/schema_Manifest.1.0.0.json"
+MANIFEST_GENERIC_PATH = f"{DATA_PATH_PREFIX}/manifests/Manifest.1.0.0.json"
+
MANIFEST_WELLBORE_VALID_PATH = f"{DATA_PATH_PREFIX}/master/Wellbore.0.3.0.json"
SCHEMA_WELLBORE_VALID_PATH = f"{DATA_PATH_PREFIX}/master/schema_Wellbore.3.0.0.json"
RECORD_WELLBORE_VALID_PATH = f"{DATA_PATH_PREFIX}/master/record_Wellbore.0.3.0.json"
+SCHEMA_GENERIC_MASTERDATA_PATH = f"{DATA_PATH_PREFIX}/master/schema_GenericMasterData.1.0.0.json"
+SCHEMA_TEST_MASTERDATA_PATH = f"{DATA_PATH_PREFIX}/master/schema_TestMaster.json"
+TRAVERSAL_WELLBORE_VALID_PATH = f"{DATA_PATH_PREFIX}/master/traversal_Wellbore.0.3.0.json"
MANIFEST_SEISMIC_TRACE_DATA_VALID_PATH = f"{DATA_PATH_PREFIX}/workProduct/SeismicTraceData.json"
+TRAVERSAL_SEISMIC_TRACE_DATA_VALID_PATH = f"{DATA_PATH_PREFIX}/workProduct/traversal_SeismicTraceData.1.0.0.json"
SCHEMA_FILE_VALID_PATH = f"{DATA_PATH_PREFIX}/workProduct/schema_File.1.0.0.json"
SCHEMA_WORK_PRODUCT_VALID_PATH = f"{DATA_PATH_PREFIX}/workProduct/schema_WorkProduct.1.0.0.json"
SCHEMA_SEISMIC_TRACE_DATA_VALID_PATH = f"{DATA_PATH_PREFIX}/workProduct/schema_SeismicTraceData.1.0.0.json"
RECORD_SEISMIC_TRACE_DATA_VALID_PATH = f"{DATA_PATH_PREFIX}/workProduct/record_SeismicTraceData.json"
MANIFEST_EMPTY_PATH = f"{DATA_PATH_PREFIX}/invalid/EmptyManifest.json"
+TRAVERSAL_MANIFEST_EMPTY_PATH = f"{DATA_PATH_PREFIX}/invalid/TraversalEmptyManifest.json"
SEARCH_VALID_RESPONSE_PATH = f"{DATA_PATH_PREFIX}/other/SearchResponseValid.json"
SEARCH_INVALID_RESPONSE_PATH = f"{DATA_PATH_PREFIX}/other/SearchResponseInvalid.json"
diff --git a/tests/plugin-unit-tests/test_file_handler.py b/tests/plugin-unit-tests/test_file_handler.py
index 9b12c45..ebad2d7 100644
--- a/tests/plugin-unit-tests/test_file_handler.py
+++ b/tests/plugin-unit-tests/test_file_handler.py
@@ -157,7 +157,7 @@ class TestFileHandler:
with open(wp_records_file_path) as cf:
file_record, unused_wpc_record, unused_wp_record = json.load(cf)
- file_record["data"].pop("FileSource")
+ file_record["data"]["DatasetProperties"]["FileSourceInfo"].pop("FileSource")
with pytest.raises(InvalidFileRecordData):
file_handler.save_file_record(file_record)
diff --git a/tests/plugin-unit-tests/test_manifest_processor_r3.py b/tests/plugin-unit-tests/test_manifest_processor_r3.py
index be4c270..cc8d1ba 100644
--- a/tests/plugin-unit-tests/test_manifest_processor_r3.py
+++ b/tests/plugin-unit-tests/test_manifest_processor_r3.py
@@ -37,7 +37,7 @@ from file_paths import (
MANIFEST_SEISMIC_TRACE_DATA_VALID_PATH,
RECORD_WELLBORE_VALID_PATH,
RECORD_SEISMIC_TRACE_DATA_VALID_PATH,
-)
+ TRAVERSAL_WELLBORE_VALID_PATH, TRAVERSAL_MANIFEST_EMPTY_PATH, TRAVERSAL_SEISMIC_TRACE_DATA_VALID_PATH)
from libs import process_manifest_r3
TENANT = "opendes"
@@ -77,19 +77,22 @@ class TestManifestProcessor:
monkeypatch.setattr(requests, "put", mockresponse)
@pytest.fixture(autouse=True)
- def manifest_processor(self, monkeypatch, conf_path: str):
+ def manifest_processor(self, monkeypatch, conf_path: str, traversal_manifest_file: str):
with open(conf_path) as f:
conf = json.load(f)
+ with open(traversal_manifest_file) as f:
+ manifest_file = json.load(f)
+ manifest_records = manifest_file
context = Context.populate(conf)
token_refresher = AirflowTokenRefresher()
file_handler = GCSFileHandler("test", token_refresher, context)
source_file_checker = GCSSourceFileChecker()
manifest_processor = process_manifest_r3.ManifestProcessor(
storage_url="",
- dagrun_conf=conf,
+ manifest_records=manifest_records,
token_refresher=token_refresher,
context=context,
- file_handler = file_handler,
+ file_handler=file_handler,
source_file_checker=source_file_checker,
)
@@ -115,38 +118,11 @@ class TestManifestProcessor:
return json.load(f)
@pytest.mark.parametrize(
- "conf_path,records_file_path",
- [
- pytest.param(MANIFEST_SEISMIC_TRACE_DATA_VALID_PATH,
- RECORD_SEISMIC_TRACE_DATA_VALID_PATH,
- id="Valid WorkProduct"),
- pytest.param(MANIFEST_WELLBORE_VALID_PATH,
- RECORD_WELLBORE_VALID_PATH,
- id="Valid Wellbore"),
- ]
- )
- def test_create_manifest_records(
- self,
- manifest_processor: process_manifest_r3.ManifestProcessor,
- mock_records_list: list,
- conf_path: str,
- records_file_path: str
- ):
- manifest_records = []
- file_records = []
- manifest_processor.create_manifest_records(manifest_records, file_records)
- actual_records = manifest_records + file_records
- sort_predicate = lambda x: x["kind"]
- actual_records.sort(key=sort_predicate)
- mock_records_list.sort(key=sort_predicate)
- assert actual_records == mock_records_list, DeepDiff(actual_records, mock_records_list,
- ignore_order=True)
-
- @pytest.mark.parametrize(
- "conf_path,records_file_path",
+ "conf_path,traversal_manifest_file,records_file_path",
[
pytest.param(
MANIFEST_SEISMIC_TRACE_DATA_VALID_PATH,
+ TRAVERSAL_SEISMIC_TRACE_DATA_VALID_PATH,
RECORD_SEISMIC_TRACE_DATA_VALID_PATH,
id="Valid WorkProduct"
),
@@ -157,71 +133,82 @@ class TestManifestProcessor:
monkeypatch,
manifest_processor: process_manifest_r3.ManifestProcessor,
mock_records_list: list,
+ traversal_manifest_file: str,
conf_path: str,
records_file_path: str
):
self.monkeypatch_storage_response(monkeypatch)
- manifest_processor.save_record({}, records_file_path)
+ manifest_processor.save_record_to_storage({}, records_file_path)
@pytest.mark.parametrize(
- "conf_path",
+ "conf_path,traversal_manifest_file",
[
- MANIFEST_SEISMIC_TRACE_DATA_VALID_PATH
+ pytest.param(MANIFEST_SEISMIC_TRACE_DATA_VALID_PATH,
+ TRAVERSAL_SEISMIC_TRACE_DATA_VALID_PATH)
]
)
def test_save_record_invalid_storage_response_value(
self,
monkeypatch,
manifest_processor: process_manifest_r3.ManifestProcessor,
+ traversal_manifest_file: str,
conf_path: str
):
self.monkeypatch_storage_response(monkeypatch, b"{}")
with pytest.raises(ValueError):
- manifest_processor.save_record({}, [{}])
+ manifest_processor.save_record_to_storage({}, [{}])
@pytest.mark.parametrize(
- "conf_path",
+ "conf_path,traversal_manifest_file",
[
- pytest.param(MANIFEST_SEISMIC_TRACE_DATA_VALID_PATH, id="Valid WorkProduct")
+ pytest.param(MANIFEST_SEISMIC_TRACE_DATA_VALID_PATH,
+ TRAVERSAL_SEISMIC_TRACE_DATA_VALID_PATH,
+ id="Valid WorkProduct")
]
)
def test_save_record_storage_response_http_error(
self,
monkeypatch,
manifest_processor: process_manifest_r3.ManifestProcessor,
+ traversal_manifest_file: str,
conf_path: str
):
self.monkeypatch_storage_response_error(monkeypatch, http.HTTPStatus.INTERNAL_SERVER_ERROR)
with pytest.raises(requests.HTTPError):
- manifest_processor.save_record({}, conf_path)
+ manifest_processor.save_record_to_storage({}, conf_path)
@pytest.mark.parametrize(
- "conf_path",
+ "conf_path,traversal_manifest_file",
[
- pytest.param(MANIFEST_SEISMIC_TRACE_DATA_VALID_PATH, id="WorkProduct"),
- pytest.param(MANIFEST_WELLBORE_VALID_PATH, id="Master"),
-
+ pytest.param(MANIFEST_SEISMIC_TRACE_DATA_VALID_PATH,
+ TRAVERSAL_SEISMIC_TRACE_DATA_VALID_PATH,
+ id="WorkProduct"),
+ pytest.param(MANIFEST_WELLBORE_VALID_PATH,
+ TRAVERSAL_WELLBORE_VALID_PATH,
+ id="Master"),
]
)
def test_process_manifest_valid(
self,
monkeypatch,
manifest_processor: process_manifest_r3.ManifestProcessor,
+ traversal_manifest_file: str,
conf_path: str
):
self.monkeypatch_storage_response(monkeypatch)
manifest_processor.process_manifest()
@pytest.mark.parametrize(
- "conf_path",
+ "conf_path,traversal_manifest_file",
[
- pytest.param(MANIFEST_EMPTY_PATH, id="Empty Manifest"),
+ pytest.param(MANIFEST_EMPTY_PATH, TRAVERSAL_MANIFEST_EMPTY_PATH, id="Empty Manifest"),
]
)
def test_process_empty_manifest(
self,
monkeypatch,
manifest_processor: process_manifest_r3.ManifestProcessor,
+ traversal_manifest_file: str,
conf_path: str
):
self.monkeypatch_storage_response(monkeypatch)
@@ -229,9 +216,9 @@ class TestManifestProcessor:
manifest_processor.process_manifest()
@pytest.mark.parametrize(
- "conf_path,expected_kind_name",
+ "conf_path,expected_kind_name,traversal_manifest_file",
[
- pytest.param(MANIFEST_WELLBORE_VALID_PATH, "Wellbore", id="Valid Wellbore"),
+ pytest.param(MANIFEST_WELLBORE_VALID_PATH, "TestMaster", TRAVERSAL_WELLBORE_VALID_PATH, id="Valid Wellbore"),
]
)
def test_get_kind(
@@ -239,37 +226,42 @@ class TestManifestProcessor:
monkeypatch,
manifest_processor: process_manifest_r3.ManifestProcessor,
conf_path: str,
+ traversal_manifest_file: str,
expected_kind_name: str
):
- for manifest_part in manifest_processor.data_object["manifest"]:
- kind = manifest_part["kind"]
+ for manifest_part in manifest_processor.manifest_records:
+ kind = manifest_part["entity"]["kind"]
assert expected_kind_name == manifest_processor._get_kind_name(kind)
@pytest.mark.parametrize(
- "conf_path,expected_kind_name",
+ "conf_path,expected_kind_name,traversal_manifest_file",
[
- pytest.param(MANIFEST_WELLBORE_VALID_PATH, "Wellbore", id="Valid Wellbore"),
+ pytest.param(MANIFEST_WELLBORE_VALID_PATH, "Wellbore", TRAVERSAL_WELLBORE_VALID_PATH, id="Valid Wellbore"),
]
)
def test_generate_id(
self,
monkeypatch,
conf_path: str,
+ traversal_manifest_file: str,
expected_kind_name: str
):
with open(conf_path) as f:
conf = json.load(f)
+ with open(traversal_manifest_file) as f:
+ manifest_file = json.load(f)
+ manifest_records = manifest_file
context = process_manifest_r3.Context.populate(conf)
manifest_processor = process_manifest_r3.ManifestProcessor(
storage_url="",
- dagrun_conf=conf,
+ manifest_records=manifest_records,
token_refresher=AirflowTokenRefresher(),
context=context,
file_handler=GCSFileHandler("test", AirflowTokenRefresher(), context),
source_file_checker=GCSSourceFileChecker()
)
- for manifest_part in manifest_processor.data_object["manifest"]:
- group_type = manifest_part["groupType"]
- kind_name = manifest_processor._get_kind_name(manifest_part["kind"])
- generated_id = manifest_processor.generate_id(manifest_part)
+ for manifest_part in manifest_processor.manifest_records:
+ group_type = manifest_part["entity"]["groupType"]
+ kind_name = manifest_processor._get_kind_name(manifest_part["entity"]["kind"])
+ generated_id = manifest_processor.generate_id(manifest_part["entity"])
assert generated_id.startswith(f"{TENANT}:{group_type}_{kind_name}:")
diff --git a/tests/plugin-unit-tests/test_manifest_traversal.py b/tests/plugin-unit-tests/test_manifest_traversal.py
new file mode 100644
index 0000000..0dd5653
--- /dev/null
+++ b/tests/plugin-unit-tests/test_manifest_traversal.py
@@ -0,0 +1,82 @@
+# Copyright 2020 Google LLC
+# Copyright 2020 EPAM Systems
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+import sys
+
+sys.path.append(f"{os.getenv('AIRFLOW_SRC_DIR')}/plugins")
+sys.path.append(f"{os.getenv('AIRFLOW_SRC_DIR')}/dags")
+
+import pytest
+from file_paths import MANIFEST_WELLBORE_VALID_PATH, TRAVERSAL_WELLBORE_VALID_PATH, \
+ MANIFEST_SEISMIC_TRACE_DATA_VALID_PATH, TRAVERSAL_SEISMIC_TRACE_DATA_VALID_PATH, MANIFEST_EMPTY_PATH, \
+ TRAVERSAL_MANIFEST_EMPTY_PATH, MANIFEST_GENERIC_SCHEMA_PATH
+from libs.exceptions import EmptyManifestError
+from libs.traverse_manifest import ManifestTraversal
+
+
+class TestManifestTraversal:
+ @pytest.fixture
+ def manifest_traversal(self, monkeypatch, manifest_file: str, manifest_schema_file: str):
+ with open(manifest_file) as f:
+ conf_manifest_file = json.load(f)
+ with open(manifest_schema_file) as f:
+ manifest_schema = json.load(f)
+ traversal = ManifestTraversal(
+ conf_manifest_file,
+ manifest_schema
+ )
+ return traversal
+
+ @pytest.mark.parametrize(
+ "manifest_file,manifest_schema_file,traversal_manifest_file",
+ [
+ pytest.param(
+ MANIFEST_WELLBORE_VALID_PATH,
+ MANIFEST_GENERIC_SCHEMA_PATH,
+ TRAVERSAL_WELLBORE_VALID_PATH,
+ id="Valid manifest Wellore"),
+ pytest.param(
+ MANIFEST_SEISMIC_TRACE_DATA_VALID_PATH,
+ MANIFEST_GENERIC_SCHEMA_PATH,
+ TRAVERSAL_SEISMIC_TRACE_DATA_VALID_PATH,
+ id="Valid manifest WPC"),
+ ]
+ )
+ def test_traversal_manifest(self, monkeypatch, manifest_traversal, manifest_file: str,
+ manifest_schema_file: str, traversal_manifest_file: str):
+ with open(traversal_manifest_file) as f:
+ traversal_manifest = json.load(f)
+ manifest_records = manifest_traversal.traverse_manifest()
+ assert manifest_records == traversal_manifest
+
+ @pytest.mark.parametrize(
+ "manifest_file,manifest_schema_file,traversal_manifest_file",
+ [
+ pytest.param(
+ MANIFEST_EMPTY_PATH,
+ MANIFEST_GENERIC_SCHEMA_PATH,
+ TRAVERSAL_MANIFEST_EMPTY_PATH,
+ id="Empty manifest"),
+ ]
+ )
+ def test_traversal_empty_manifest(self, monkeypatch,
+ manifest_traversal,
+ manifest_file: str,
+ manifest_schema_file: str,
+ traversal_manifest_file: str):
+ with pytest.raises(EmptyManifestError):
+ manifest_traversal.traverse_manifest()
diff --git a/tests/plugin-unit-tests/test_operators_r3.py b/tests/plugin-unit-tests/test_operators_r3.py
index 8d720ce..f99441d 100644
--- a/tests/plugin-unit-tests/test_operators_r3.py
+++ b/tests/plugin-unit-tests/test_operators_r3.py
@@ -32,14 +32,12 @@ import requests
from file_paths import (
MANIFEST_WELLBORE_VALID_PATH,
- SEARCH_VALID_RESPONSE_PATH
-)
+ SEARCH_VALID_RESPONSE_PATH, MANIFEST_GENERIC_SCHEMA_PATH)
from operators.process_manifest_r3 import ProcessManifestOperatorR3, SchemaValidator, \
ManifestProcessor
from operators.search_record_id import SearchRecordIdOperator
from operators.update_status import UpdateStatusOperator
from libs.handle_file import GCSFileHandler
-from libs.refresh_token import AirflowTokenRefresher
from mock_responses import MockSearchResponse, MockWorkflowResponse
CustomOperator = TypeVar("CustomOperator")
@@ -70,8 +68,14 @@ class TestOperators(object):
return task, context
def test_process_manifest_r3_operator(self, monkeypatch):
- monkeypatch.setattr(SchemaValidator, "validate_manifest", lambda obj: None)
- monkeypatch.setattr(ManifestProcessor, "save_record",
+
+ def _get_common_schema(*args, **kwargs):
+ with open(MANIFEST_GENERIC_SCHEMA_PATH) as f:
+ manifest_schema = json.load(f)
+ return manifest_schema
+ monkeypatch.setattr(SchemaValidator, "get_schema", _get_common_schema)
+ monkeypatch.setattr(SchemaValidator, "validate_manifest", lambda obj, entities: entities)
+ monkeypatch.setattr(ManifestProcessor, "save_record_to_storage",
lambda obj, headers, request_data: MockStorageResponse())
monkeypatch.setattr(GCSFileHandler, "upload_file",
lambda *args, **kwargs: "test")
diff --git a/tests/plugin-unit-tests/test_schema_validator_r3.py b/tests/plugin-unit-tests/test_schema_validator_r3.py
index ce13c10..1b138b4 100644
--- a/tests/plugin-unit-tests/test_schema_validator_r3.py
+++ b/tests/plugin-unit-tests/test_schema_validator_r3.py
@@ -28,11 +28,18 @@ from file_paths import (
DATA_PATH_PREFIX,
MANIFEST_EMPTY_PATH,
SCHEMA_FILE_VALID_PATH,
+ SCHEMA_GENERIC_MASTERDATA_PATH,
SCHEMA_SEISMIC_TRACE_DATA_VALID_PATH,
SCHEMA_WORK_PRODUCT_VALID_PATH,
+ SCHEMA_TEST_MASTERDATA_PATH,
MANIFEST_SEISMIC_TRACE_DATA_VALID_PATH,
MANIFEST_WELLBORE_VALID_PATH,
- SCHEMA_WELLBORE_VALID_PATH
+ MANIFEST_GENERIC_PATH,
+ MANIFEST_GENERIC_SCHEMA_PATH,
+ SCHEMA_WELLBORE_VALID_PATH,
+ TRAVERSAL_WELLBORE_VALID_PATH,
+ TRAVERSAL_SEISMIC_TRACE_DATA_VALID_PATH,
+ TRAVERSAL_MANIFEST_EMPTY_PATH
)
from mock_responses import MockSchemaResponse
from libs.context import Context
@@ -48,14 +55,14 @@ TENANT = "opendes"
class TestSchemaValidator:
@pytest.fixture
- def schema_validator(self, monkeypatch, manifest_file: str, schema_file: str):
- with open(manifest_file) as f:
- manifest_file = json.load(f)
- conf = copy.deepcopy(manifest_file)
- context = Context.populate(conf)
+ def schema_validator(
+ self,
+ monkeypatch,
+ schema_file: str
+ ):
+ context = Context(app_key="", data_partition_id="")
validator = SchemaValidator(
"",
- conf,
AirflowTokenRefresher(),
context
)
@@ -64,60 +71,53 @@ class TestSchemaValidator:
lambda *args, **kwargs: MockSchemaResponse(schema_file))
return validator
- @pytest.mark.parametrize(
- "manifest_file,schema_file",
- [
- pytest.param(
- MANIFEST_SEISMIC_TRACE_DATA_VALID_PATH, None,
- id="Valid manifest_file")
- ]
- )
- def test_schema_validator_validate_workproduct_manifest(self,
- monkeypatch,
- schema_validator: SchemaValidator,
- manifest_file: str,
- schema_file: str):
- def mock_get_schema(uri: str):
- if ":WorkProduct:" in uri:
- schema_path = SCHEMA_WORK_PRODUCT_VALID_PATH
- elif ":SeismicTraceData:" in uri:
- schema_path = SCHEMA_SEISMIC_TRACE_DATA_VALID_PATH
- elif ":File:" in uri:
- schema_path = SCHEMA_FILE_VALID_PATH
- else:
- print(uri)
- raise
- with open(schema_path) as f:
- schema_content = json.load(f)
- return schema_content
-
- monkeypatch.setattr(schema_validator, "get_schema_request", mock_get_schema)
- schema_validator.validate_manifest()
+ @staticmethod
+ def mock_get_schema(uri: str):
+ if "WorkProduct" in uri:
+ schema_path = SCHEMA_WORK_PRODUCT_VALID_PATH
+ elif "SeismicTraceData" in uri:
+ schema_path = SCHEMA_SEISMIC_TRACE_DATA_VALID_PATH
+ elif "File" in uri:
+ schema_path = SCHEMA_FILE_VALID_PATH
+ elif "GenericMasterData" in uri:
+ schema_path = SCHEMA_GENERIC_MASTERDATA_PATH
+ elif "TestMaster" in uri:
+ schema_path = SCHEMA_TEST_MASTERDATA_PATH
+ else:
+ print(uri)
+ raise Exception(f"Can't get such a schema {uri} in data files of tests")
+ with open(schema_path) as f:
+ schema_content = json.load(f)
+ return schema_content
@pytest.mark.parametrize(
- "manifest_file,schema_file",
+ "traversal_manifest_file_path,schema_file",
[
pytest.param(
- MANIFEST_WELLBORE_VALID_PATH,
- SCHEMA_WELLBORE_VALID_PATH,
- id="Valid manifest Wellore"),
+ TRAVERSAL_WELLBORE_VALID_PATH,
+ None,
+ id="Valid manifest_file"
+ )
]
)
- def test_schema_validator_master_manifest(self, monkeypatch, schema_validator, manifest_file,
- schema_file):
- def mock_get_schema(uri: str):
- with open(schema_file) as f:
- schema_content = json.load(f)
- return schema_content
-
- monkeypatch.setattr(schema_validator, "get_schema_request", mock_get_schema)
- schema_validator.validate_manifest()
+ def test_schema_validator_master_manifest(
+ self,
+ monkeypatch,
+ schema_validator,
+ traversal_manifest_file_path: str,
+ schema_file
+ ):
+ monkeypatch.setattr(schema_validator, "get_schema", self.mock_get_schema)
+ with open(traversal_manifest_file_path) as f:
+ manifest_file = json.load(f)
+ validated_records = schema_validator.validate_manifest(manifest_file)
+ assert len(manifest_file) == len(validated_records)
@pytest.mark.parametrize(
- "manifest_file,schema_file",
+ "manifest_file,traversal_manifest_file,schema_file",
[
pytest.param(
- MANIFEST_EMPTY_PATH, None,
+ MANIFEST_EMPTY_PATH, TRAVERSAL_MANIFEST_EMPTY_PATH, None,
id="Empty Manifest"),
]
)
@@ -125,31 +125,38 @@ class TestSchemaValidator:
monkeypatch,
schema_validator: SchemaValidator,
manifest_file: str,
+ traversal_manifest_file: str,
schema_file: str):
+ with open(traversal_manifest_file) as f:
+ manifest_file = json.load(f)
with pytest.raises(EmptyManifestError):
- schema_validator.validate_manifest()
+ schema_validator.validate_manifest(manifest_file)
@pytest.mark.parametrize(
- "manifest_file,schema_file",
+ "traversal_manifest_file,schema_file",
[
pytest.param(
- f"{DATA_PATH_PREFIX}/invalid/NotOsduFormat.json", None,
+ f"{DATA_PATH_PREFIX}/invalid/TraversalNotOSDUFormatManifest.json",
+ None,
id="Not OSDU FORMAT"),
]
)
def test_schema_validator_not_osdu_format(self,
monkeypatch,
schema_validator: SchemaValidator,
- manifest_file: str,
+ traversal_manifest_file: str,
schema_file: str):
+ with open(traversal_manifest_file) as f:
+ manifest_file = json.load(f)
with pytest.raises(NotOSDUSchemaFormatError):
- schema_validator.validate_manifest()
+ schema_validator.validate_manifest(manifest_file)
@pytest.mark.parametrize(
- "manifest_file,schema_file,kind",
+ "manifest_file,traversal_manifest_file,schema_file,kind",
[
pytest.param(
MANIFEST_WELLBORE_VALID_PATH,
+ TRAVERSAL_WELLBORE_VALID_PATH,
SCHEMA_WELLBORE_VALID_PATH,
"opendes:osdu:Wellbore:0.3.0",
id="Valid manifest Wellore"),
@@ -158,15 +165,17 @@ class TestSchemaValidator:
def test_get_schema_request(self,
schema_validator: SchemaValidator,
manifest_file: str,
+ traversal_manifest_file: str,
schema_file: str,
kind: str):
schema_validator.get_schema_request(kind)
@pytest.mark.parametrize(
- "manifest_file,schema_file,kind",
+ "manifest_file,traversal_manifest_file,schema_file,kind",
[
pytest.param(
MANIFEST_WELLBORE_VALID_PATH,
+ TRAVERSAL_WELLBORE_VALID_PATH,
SCHEMA_WELLBORE_VALID_PATH,
"opendes:osdu:Wellbore:0.3.0",
id="Valid manifest Wellore"),
@@ -176,6 +185,7 @@ class TestSchemaValidator:
monkeypatch,
schema_validator: SchemaValidator,
manifest_file: str,
+ traversal_manifest_file: str,
schema_file: str,
kind: str):
monkeypatch.setattr(requests,
@@ -184,3 +194,29 @@ class TestSchemaValidator:
http.HTTPStatus.INTERNAL_SERVER_ERROR))
with pytest.raises(requests.HTTPError):
schema_validator.get_schema(kind)
+
+ @pytest.mark.parametrize(
+ "manifest_file,schema_file",
+ [
+ pytest.param(
+ MANIFEST_GENERIC_PATH,
+ MANIFEST_GENERIC_SCHEMA_PATH,
+ id="Valid generic manifest"),
+ ]
+ )
+ def test_generic_manifest_validation(
+ self,
+ monkeypatch,
+ schema_validator: SchemaValidator,
+ manifest_file: str,
+ schema_file
+ ):
+ def mock_get_schema(uri: str):
+ with open(schema_file) as f:
+ schema_content = json.load(f)
+ return schema_content
+
+ with open(manifest_file) as f:
+ manifest_content = json.load(f)
+ monkeypatch.setattr(schema_validator, "get_schema", mock_get_schema)
+ schema_validator.validate_common_schema({"manifest": manifest_content})
--
GitLab