Commit a0037ce4 authored by Gregory Harris's avatar Gregory Harris
Browse files

Response to PR comments

parent 9516364e
Pipeline #97176 failed with stages
in 1 minute and 3 seconds
......@@ -66,7 +66,6 @@ The wbdutil requires a configuration file that has the following JSON structure:
{
"base_url": "https://osdu-ship.msft-osdu-test.org",
"data_partition_id": "opendes",
"acl_domain": "contoso.com",
"legal":
{
"legaltags": ["opendes-public-usa-dataset-7643990"],
......@@ -78,23 +77,27 @@ The wbdutil requires a configuration file that has the following JSON structure:
"viewers": [ "data.default.viewers@opendes.contoso.com" ],
"owners": [ "data.default.owners@opendes.contoso.com" ]
}
}
},
"wellbore_mapping": {},
"welllog_mapping": {},
}
```
The `base_url`, `data_partition_id` and `acl_domain` must be correct for the OSDU instance you want to connect to.
The `base_url` and `data_partition_id` must be correct for the OSDU instance that you want to connect to.
`wellbore_mapping` and `welllog_mapping` are optional features described in the [custom mappings](#custom-mappings) section.
#### Custom mappings
The configuration file can be used to define custom mappings from `lasio.LASFile` data objects
The configuration file can be used to define optional custom mappings from `lasio.LASFile` data objects
to OSDU wellbore and welllog objects of a specified kind.
This is an advanced feature of `wbdutil` that requires knowledge of both `lasio.LASFile` and OSDU data object schemas and should be used with care.
It is recommended that a new mapping is thoroughly tested using the `parse` command group, before upload to OSDU.
There are 2 mapping types `wellbore_mapping` and `welllog_mapping` both must contain a `kind` and a `mapping` attribute.
If `wellbore_mapping` or `welllog_mapping` are not defined in the configuration file `wbdutil` will use the default mappings for the well bore and/or well log.
The `kind` attribute defines the target OSDU data type (kind), for example `osdu:wks:work-product-component--WellLog:1.1.0`.
The `mapping` type describes how data in the incoming `lasio.LASFile` object should be transformed into the target OSDU data kind.
Here is an example.
Here is an example mapping for a welllog that could be added to a configuration file.
```
{
......@@ -134,7 +137,7 @@ Here is an example.
}
```
The simple data mappings take the for of a key and string value pair.
The simple data mappings take the form of a key and string value pair.
The key (string to the left of the semi-colon) is the target field within the OSDU data kind and
the value string defines the source of the data. For example:
`"data.ReferenceCurveID": "curves[0].mnemonic"` will set the `data.ReferenceCurveID` field of the output OSDU object
......
{
"base_url": "https://osdu-ship.msft-osdu-test.org",
"data_partition_id": "opendes",
"acl_domain": "contoso.com",
"legal": {
"legaltags": ["opendes-public-usa-dataset-7643990"],
"otherRelevantDataCountries": ["US"],
......
{
"data_partition_id": "opendes",
"acl_domain": "contoso.com",
"legal": {
"legaltags": [ "opendes-public-usa-dataset-7643990" ],
"otherRelevantDataCountries": [
......
{
"acl_domain": "contoso.com",
"legal": {
"legaltags": ["opendes-public-usa-dataset-7643990"],
"status": "compliant"
......
......@@ -14,7 +14,6 @@ class TestConfiguration:
_config = {
"base_url": "https://osdu-ship.msft-osdu-test.org",
"data_partition_id": "opendes",
"acl_domain": "contoso.com",
"legal":
{
"legaltags": ["opendes-public-usa-dataset-7643990"],
......@@ -55,6 +54,5 @@ class TestConfiguration:
conf = Configuration(mock_reader, "some_filename.json")
# Assert
assert conf.get_recursive(["acl_domain"]) == self._config["acl_domain"]
assert conf.get_recursive(["legal", "legaltags"]) == self._config["legal"]["legaltags"]
assert conf.get_recursive(["data", "default", "owners"]) == self._config["data"]["default"]["owners"]
......@@ -25,10 +25,7 @@ class TestPropertyMapper:
"uvw": "ef"
}
mock_reader = Mock(spec=IPropertyMappingLoader)
mock_reader.mapping = mapping
target = PropertyMapper(mock_reader, None, None)
target = self._construct_property_mapper(mapping)
source = {
"abc": "some data",
......@@ -50,10 +47,7 @@ class TestPropertyMapper:
"dest3": "level1a"
}
mock_reader = Mock(spec=IPropertyMappingLoader)
mock_reader.mapping = mapping
target = PropertyMapper(mock_reader, None, None)
target = self._construct_property_mapper(mapping)
source = {
"level1a": {
......@@ -74,7 +68,7 @@ class TestPropertyMapper:
assert result["dest2"] is source["level1b"]["level2"]
assert result["dest3"] is source["level1a"]
def test_GIVEN_source_mapping_wrong_WHEN_remap_data_THEN_raises_ExtendedPropertiesAttributeError(self):
def test_GIVEN_source_mapping_wrong_WHEN_remap_data_THEN_missing_field_is_populated_with_None(self):
# Arrange
mapping = {
"dest1": "level1a.level2.level4",
......@@ -82,10 +76,7 @@ class TestPropertyMapper:
"dest3": "level1a"
}
mock_reader = Mock(spec=IPropertyMappingLoader)
mock_reader.mapping = mapping
target = PropertyMapper(mock_reader, None, None)
target = self._construct_property_mapper(mapping)
source = {
"level1a": {
......@@ -113,10 +104,7 @@ class TestPropertyMapper:
"level1.b": "b"
}
mock_reader = Mock(spec=IPropertyMappingLoader)
mock_reader.mapping = mapping
target = PropertyMapper(mock_reader, None, None)
target = self._construct_property_mapper(mapping)
source = {
"a": "some data",
......@@ -136,9 +124,7 @@ class TestPropertyMapper:
"dest": "src[1]",
}
mock_reader = Mock(spec=IPropertyMappingLoader)
mock_reader.mapping = mapping
target = PropertyMapper(mock_reader, None, None)
target = self._construct_property_mapper(mapping)
source = {
"src": ["element0", "element1"]
......@@ -156,9 +142,7 @@ class TestPropertyMapper:
"dest": "src[0][1]",
}
mock_reader = Mock(spec=IPropertyMappingLoader)
mock_reader.mapping = mapping
target = PropertyMapper(mock_reader, None, None)
target = self._construct_property_mapper(mapping)
source = {
"src": [['element0.0', 'element0.1'], 'element1']
......@@ -176,9 +160,7 @@ class TestPropertyMapper:
"dest": "src[0][1].nest[1].value",
}
mock_reader = Mock(spec=IPropertyMappingLoader)
mock_reader.mapping = mapping
target = PropertyMapper(mock_reader, None, None)
target = self._construct_property_mapper(mapping)
source = {
"src": [["element0.0", {"nest": ["", {"value": 123}]}], "element1"]
......@@ -196,9 +178,7 @@ class TestPropertyMapper:
"dest": "src[a]",
}
mock_reader = Mock(spec=IPropertyMappingLoader)
mock_reader.mapping = mapping
target = PropertyMapper(mock_reader, None, None)
target = self._construct_property_mapper(mapping)
source = {
"src": ["element0", "element1"]
......@@ -216,9 +196,7 @@ class TestPropertyMapper:
"dest": "[1]",
}
mock_reader = Mock(spec=IPropertyMappingLoader)
mock_reader.mapping = mapping
target = PropertyMapper(mock_reader, None, None)
target = self._construct_property_mapper(mapping)
source = {
"src": ["element0", "element1"]
......@@ -239,9 +217,7 @@ class TestPropertyMapper:
mock_config = Mock(spec=Configuration)
mock_config.get_recursive.return_value = "A value"
mock_reader = Mock(spec=IPropertyMappingLoader)
mock_reader.mapping = mapping
target = PropertyMapper(mock_reader, mock_config, None)
target = self._construct_property_mapper(mapping, mock_config)
source = {
"src": ["element0", "element1"]
......@@ -267,9 +243,7 @@ class TestPropertyMapper:
mock_functions = Mock()
mock_functions.get_wellbore_id.return_value = "12345"
mock_reader = Mock(spec=IPropertyMappingLoader)
mock_reader.mapping = mapping
target = PropertyMapper(mock_reader, None, mock_functions)
target = self._construct_property_mapper(mapping, None, mock_functions)
source = {}
......@@ -293,9 +267,7 @@ class TestPropertyMapper:
mock_functions = Mock()
mock_functions.some_function.return_value = "12345"
mock_reader = Mock(spec=IPropertyMappingLoader)
mock_reader.mapping = mapping
target = PropertyMapper(mock_reader, None, mock_functions)
target = self._construct_property_mapper(mapping, None, mock_functions)
source = {
"src": {
......@@ -321,9 +293,7 @@ class TestPropertyMapper:
}
}
mock_reader = Mock(spec=IPropertyMappingLoader)
mock_reader.mapping = mapping
target = PropertyMapper(mock_reader, None, object())
target = self._construct_property_mapper(mapping, None, object())
source = {
"arg1": 123
......@@ -349,9 +319,7 @@ class TestPropertyMapper:
mock_functions = Mock()
mock_functions.some_function.return_value = "12345"
mock_reader = Mock(spec=IPropertyMappingLoader)
mock_reader.mapping = mapping
target = PropertyMapper(mock_reader, None, mock_functions)
target = self._construct_property_mapper(mapping, None, mock_functions)
# Act
# Assert
......@@ -373,9 +341,7 @@ class TestPropertyMapper:
}
}
mock_reader = Mock(spec=IPropertyMappingLoader)
mock_reader.mapping = mapping
target = PropertyMapper(mock_reader, None, None)
target = self._construct_property_mapper(mapping)
source = {
"curves": [{"src1": 1, "src2": "a"}, {"src1": 2, "src2": "b"}, {"src1": 3, "src2": "c"}]
......@@ -419,7 +385,6 @@ class TestPropertyMapper:
config_str = '''{
"base_url": "https://osdu-ship.msft-osdu-test.org",
"data_partition_id": "opendes",
"acl_domain": "contoso.com",
"legal": {
"legaltags": ["opendes-public-usa-dataset-7643990"],
"otherRelevantDataCountries": ["US"],
......@@ -580,3 +545,8 @@ class TestPropertyMapper:
calls = [call('opendes', 'mnemonic_0'), call('opendes', 'mnemonic_1'), call('opendes', 'mnemonic_2')]
mock_functions.las2osdu_curve_uom_converter.assert_has_calls(calls)
def _construct_property_mapper(self, mapping, mock_config = None, mock_functions = None):
mock_reader = Mock(spec=IPropertyMappingLoader)
mock_reader.mapping = mapping
return PropertyMapper(mock_reader, mock_config, mock_functions)
......@@ -61,12 +61,12 @@ class TestReflectionHelper:
assert result == 5
def test_GIVEN_empty_source_dataWHEN_getattr_recursive_THEN_result_is_none(self):
def test_GIVEN_empty_source_data_WHEN_getattr_recursive_THEN_result_is_none(self):
data = {}
result = ReflectionHelper.getattr_recursive(data, ["xyz", "123"])
assert result is None
def test_GIVEN_none_source_dataWHEN_getattr_recursive_THEN_result_is_none(self):
def test_GIVEN_none_source_data_WHEN_getattr_recursive_THEN_result_is_none(self):
data = None
result = ReflectionHelper.getattr_recursive(data, ["xyz", "123"])
assert result is None
......
......@@ -123,7 +123,7 @@ class PropertyMapper:
if mapping_type == "function":
try:
function_name = source_field.get("function")
if mapping_type is None:
if function_name is None:
raise ValueError("The function name must be given in the configuration file")
argument_names = source_field.get("args", [])
......
......@@ -56,7 +56,7 @@ class WellLogRecord(Record):
class WellboreMappingFunctions:
def build_wellbore_name_aliases(self, uwi: str, data_partition_id: str) -> List[Dict[str, str]]:
"""
Determine what the NameAliases attribute should be, depending on the existence of a UWI.
Map data to the wellbore name alias data type.
:param uwi str: The UWI from the LASFile.
:param data_partition_id str: Record config object containing config .
......
......@@ -26,6 +26,7 @@ class ReflectionHelper:
except (KeyError, AttributeError):
if field_name.endswith(']') and '[' in field_name:
# probably a reference to a list item
logger.debug(f"Attempting to parse the mapping field {field_name} as an array")
subfield = ReflectionHelper._getattr_list(source, field_name)
else:
message = f"The attribute '{field_name}' was not found in the source data. An empty value for this field will be returned."
......@@ -62,7 +63,7 @@ class ReflectionHelper:
Set the specified field to the given value, recursively create member objects where needed. Update in place.
:param any value: The data value
:param any destination: The destination object
:param List[str] destination_field_heirachy: List of fields, from lowest to the deepest in the heirachy.
:param List[str] destination_field_heirachy: List of fields in the hierarchy, ordered from top to bottom.
"""
if destination is None:
message = "The destination dictionary must not be null"
......
{
"base_url": "https://osdu-ship.msft-osdu-test.org",
"data_partition_id": "opendes",
"acl_domain": "contoso.com",
"legal": {
"legaltags": [
"opendes-public-usa-dataset-7643990"
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment