Commit c0f051bc authored by Greg Harris's avatar Greg Harris
Browse files

Soft mapping OSDU -> LasFileMappingFunctions

Some test fixes and improvements

Squash
parent eca7513d
Pipeline #99373 passed with stages
in 2 minutes and 27 seconds
......@@ -60,5 +60,20 @@
}
}
}
}
},
"las_file_mapping": {
"mapping": {
"Well.WELL": "WELLBORE.data.FacilityName",
"Well.UWI": {
"type": "function",
"function": "extract_uwi_from_name_aliases",
"args": ["WELLBORE.data.NameAliases"]
},
"Curves": {
"type": "function",
"function": "build_curves_section",
"args": ["WELLLOG.data.Curves", "CURVES"]
}
}
}
}
......@@ -33,7 +33,7 @@ class TestPropertyMapper:
}
# Act
result = target.remap_data(source)
result = target.remap_data(source, {})
# Assert
assert result["xyz"] is source["abc"]
......@@ -61,7 +61,7 @@ class TestPropertyMapper:
}
# Act
result = target.remap_data(source)
result = target.remap_data(source, {})
# Assert
assert result["dest1"] is source["level1a"]["level2"]["level3"]
......@@ -90,7 +90,7 @@ class TestPropertyMapper:
}
# Act
result = target.remap_data(source)
result = target.remap_data(source, {})
# Assert
assert result["dest1"] is None
......@@ -112,7 +112,7 @@ class TestPropertyMapper:
}
# Act
result = target.remap_data(source)
result = target.remap_data(source, {})
# Assert
assert result["level1"]["level2"]["a"] is source["a"]
......@@ -131,7 +131,7 @@ class TestPropertyMapper:
}
# Act
result = target.remap_data(source)
result = target.remap_data(source, {})
# Assert
assert result["dest"] is source["src"][1]
......@@ -149,7 +149,7 @@ class TestPropertyMapper:
}
# Act
result = target.remap_data(source)
result = target.remap_data(source, {})
# Assert
assert result["dest"] is source["src"][0][1]
......@@ -167,7 +167,7 @@ class TestPropertyMapper:
}
# Act
result = target.remap_data(source)
result = target.remap_data(source, {})
# Assert
assert result["dest"] == 123
......@@ -186,7 +186,7 @@ class TestPropertyMapper:
# Act
with pytest.raises(WbdutilAttributeError) as ex:
target.remap_data(source)
target.remap_data(source, {})
assert "src[a]" in ex.value.message
......@@ -204,7 +204,7 @@ class TestPropertyMapper:
# Act
with pytest.raises(WbdutilAttributeError) as ex:
target.remap_data(source)
target.remap_data(source, {})
assert "[1]" in ex.value.message
......@@ -224,7 +224,7 @@ class TestPropertyMapper:
}
# Act
result = target.remap_data(source)
result = target.remap_data(source, {})
# Assert
assert result["dest"]["dest2"] == "A value"
......@@ -248,7 +248,7 @@ class TestPropertyMapper:
source = {}
# Act
result = target.remap_data(source)
result = target.remap_data(source, {})
# Assert
assert result["data"]["WellboreID"] == "12345"
......@@ -277,7 +277,7 @@ class TestPropertyMapper:
}
# Act
result = target.remap_data(source)
result = target.remap_data(source, {})
# Assert
assert result["data"]["function_result"] == "12345"
......@@ -302,7 +302,7 @@ class TestPropertyMapper:
# Act
# Assert
with pytest.raises(WbdutilAttributeError) as execinfo:
target.remap_data(source)
target.remap_data(source, {})
assert execinfo.value.message == "The function 'some_function_that_does_not_exist' was not found in the provided mapping functions"
......@@ -324,7 +324,7 @@ class TestPropertyMapper:
# Act
# Assert
with pytest.raises(WbdutilAttributeError) as execinfo:
target.remap_data({})
target.remap_data({}, {})
assert execinfo.value.message == "All of the mapping function arguments must be strings."
......@@ -348,7 +348,7 @@ class TestPropertyMapper:
}
# Act
result = target.remap_data(source)
result = target.remap_data(source, {})
# Assert
result_array = result["data"]["array_remap_result"]
......
......@@ -312,7 +312,9 @@ class TestMapWellLogToLas:
})
data = DataFrame({"ABC": [1, 2, 3], "XYZ": [9, 8, 7], "IJK": [-1, -2, -3]})
subject = MapWellLogToLas(wellbore, welllog, data)
mockConfig = Mock(spec=Configuration)
mockConfig.las_file_mapping = None
subject = MapWellLogToLas(mockConfig, wellbore, welllog, data)
# ACT
result = subject.build_las_file()
......@@ -353,7 +355,9 @@ class TestMapWellLogToLas:
data = DataFrame({})
subject = MapWellLogToLas(wellbore, welllog, data)
mockConfig = Mock(spec=Configuration)
mockConfig.las_file_mapping = None
subject = MapWellLogToLas(mockConfig, wellbore, welllog, data)
# ACT
result = subject.build_las_file()
......@@ -371,7 +375,10 @@ class TestMapWellLogToLas:
wellbore = Record({"kind": "BoreKind", "acl": {}, "legal": {}, "data": {}})
data = DataFrame({"ABC": [1, 2, 3], "XYZ": [9, 8, 7], "IJK": [-1, -2, -3]})
subject = MapWellLogToLas(wellbore, welllog, data)
mockConfig = Mock(spec=Configuration)
mockConfig.las_file_mapping = None
subject = MapWellLogToLas(mockConfig, wellbore, welllog, data)
# ACT
result = subject.build_las_file()
......
......@@ -20,6 +20,8 @@ LasToRecordMapper = record_mapper.LasToRecordMapper
Record = record_mapper.Record
WellLogRecord = record_mapper.WellLogRecord
Configuration = import_module(f"{TARGET_MODULE_NAME}.configuration").Configuration
class TestWellLogService:
......@@ -213,8 +215,11 @@ class TestWellLogService:
arg_welllog_id = "wellog_id argument"
arg_curves = ["ABC", "XYZ"]
mockConfig = Mock(spec=Configuration)
mockConfig.las_file_mapping = None
# Act
result = subject.download_and_construct_las(arg_welllog_id, arg_curves)
result = subject.download_and_construct_las(mockConfig, arg_welllog_id, arg_curves)
# Assert
client.get_welllog_record.assert_called_once_with(arg_welllog_id)
......@@ -258,8 +263,11 @@ class TestWellLogService:
arg_welllog_id = "wellog_id argument"
arg_curves = ["ABC", "XYZ"]
mockConfig = Mock(spec=Configuration)
mockConfig.las_file_mapping = None
# Act
result = subject.download_and_construct_las(arg_welllog_id, arg_curves)
result = subject.download_and_construct_las(mockConfig, arg_welllog_id, arg_curves)
# Assert
client.get_welllog_record.assert_called_once_with(arg_welllog_id)
......
......@@ -31,7 +31,7 @@ def download_las(welllog_id: str,
service = WellLogService(client)
las_file = service.download_and_construct_las(welllog_id, curves)
las_file = service.download_and_construct_las(config, welllog_id, curves)
logger.warning(f"Writing to file {outfile}")
......
......@@ -62,3 +62,12 @@ class Configuration:
rtype: Dict[str, Any]
"""
return self._config.get("welllog_mapping")
@property
def las_file_mapping(self) -> Union[Dict[str, Any], None]:
"""
Gets the las_file_mapping.
return: the las_file_mapping
rtype: Dict[str, Any]
"""
return self._config.get("las_file_mapping")
......@@ -42,7 +42,7 @@ class DictionaryMappingLoader(IPropertyMappingLoader):
return: the kind string
rtype: str
"""
return self._base_mapping["kind"]
return self._base_mapping.get("kind")
class PropertyMapper:
......@@ -75,22 +75,23 @@ class PropertyMapper:
:return: A new data object that contains the remapped data
:rtype: Dict[str, Any]
"""
output = self.remap_data(source)
output["kind"] = self._kind
return output
destination = self.remap_data(source, {})
destination["kind"] = self._kind
return destination
def remap_data(self, source: any) -> Dict[str, Any]:
def remap_data(self, source: any, destination: Dict[str, Any]) -> Dict[str, Any]:
"""
Remap the data in the source object using the mapping dictionary to a new object
:param any source: The source object
:return: A new data object that contains the remapped data
:param Dict[str, Any] destination: The destination object
:return: The populated destination object
:rtype: Dict[str, Any]
"""
return self._remap_data(source, self._mapping)
return self._remap_data(source, self._mapping, destination)
def _remap_data(self, source: any, mapping: Dict[str, Any]) -> Dict[str, Any]:
extensible = {}
def _remap_data(self, source: any, mapping: Dict[str, Any], extensible: Dict[str, Any]) -> Dict[str, Any]:
# iterate through the mapping dictionary remapping data as we go
for target_field, source_field in mapping.items():
......@@ -156,7 +157,7 @@ class PropertyMapper:
remapped_array = []
for item in source_array:
remapped_array.append(self._remap_data(item, submapping))
remapped_array.append(self._remap_data(item, submapping, {}))
return remapped_array
......
from typing import Dict, List
import urllib
import lasio
from lasio.las_items import CurveItem, SectionItems
from knack.log import get_logger
from lasio.las import LASFile
from pandas.core.frame import DataFrame
......@@ -168,7 +169,6 @@ class LasToRecordMapper:
:returns: A Wellbore record object that can be uploaded to OSDU as JSON payload.
:rtype: Record
"""
mapper = PropertyMapper(
DictionaryMappingLoader(self.config.wellbore_mapping or self._DEFAULT_WELLBORE_MAPPING),
self.config,
......@@ -184,6 +184,7 @@ class LasToRecordMapper:
:returns: A Well Log record object that can be uploaded to OSDU as JSON payload.
:rtype: Record
"""
mapper = PropertyMapper(
DictionaryMappingLoader(self.config.welllog_mapping or self._DEFAULT_WELLLOG_MAPPING),
self.config,
......@@ -203,16 +204,36 @@ class LasToRecordMapper:
class MapWellLogToLas:
def __init__(self, wellbore: Record, welllog: Record, data: DataFrame) -> None:
_DEFAULT_LAS_MAPPING = {
"mapping": {
"Well.WELL": "WELLBORE.data.FacilityName",
"Well.UWI": {
"type": "function",
"function": "extract_uwi_from_name_aliases",
"args": ["WELLBORE.data.NameAliases"]
},
"Curves": {
"type": "function",
"function": "build_curves_section",
"args": ["WELLLOG.data.Curves", "CURVES"]
}
}
}
def __init__(self, configuration: Configuration, wellbore: Record, welllog: Record, data: DataFrame) -> None:
"""
Construct a new instance of MapWellLogToLas
:param configuration Configuration: The application configuration
:param wellbore Record: The wellbore record
:param welllog Record: The well log record
:param data DataFrame: The well log curve data
"""
self._wellbore = wellbore
self._welllog = welllog
self._data = data
self._config = configuration
self._data = {
"CURVES": data,
"WELLLOG": welllog,
"WELLBORE": wellbore
}
def build_las_file(self) -> LASFile:
"""
......@@ -220,18 +241,25 @@ class MapWellLogToLas:
:returns: A new instance of LASFile.
:rtype: LASFile
"""
mapper = PropertyMapper(
DictionaryMappingLoader(self._config.las_file_mapping or self._DEFAULT_LAS_MAPPING),
None,
LasFileMappingFunctions())
las = lasio.LASFile()
las.sections = mapper.remap_data(self._data, las.sections)
return las
# Add wellbore info
nameAlias = self._wellbore.data.get("NameAliases")
las.well.UWI.value = nameAlias[0].get("AliasName") if nameAlias is not None and len(nameAlias) > 0 else None
las.well.WELL.value = self._wellbore.data.get("FacilityName")
# Get wellog curve info
wl_curves = self._welllog.data.get("Curves")
class LasFileMappingFunctions:
def extract_uwi_from_name_aliases(self, name_aliases: list):
return name_aliases[0].get("AliasName") if name_aliases is not None and len(name_aliases) > 0 else None
# Add curves to las file
for curve in self._data:
def build_curves_section(self, wl_curves: list, data: DataFrame) -> SectionItems:
curve_items = []
for curve in data:
# Try to get the curve units from the welllog
if wl_curves is not None:
wl_curve_matches = [c for c in wl_curves if c.get("CurveID") == curve]
......@@ -241,9 +269,9 @@ class MapWellLogToLas:
else:
las_units = None
las.append_curve(curve, self._data[curve], unit=las_units)
curve_items.append(CurveItem(curve, las_units, "", "", data[curve]))
return las
return SectionItems(curve_items)
class MappingUtilities:
......
......@@ -3,6 +3,8 @@ from typing import Dict, List, Tuple
import urllib
from knack.log import get_logger
from lasio.las import LASFile
from wbdutil.configuration import Configuration
from .file_loader import FileValidationError, LasParser, LocalFileLoader
from .osdu_client import OsduClient, DataLoaderWebResponseError
from .record_mapper import LasToRecordMapper, Record, MappingUtilities, MapWellLogToLas
......@@ -148,9 +150,10 @@ class WellLogService:
data = las_data.df().reset_index()
self._client.add_welllog_data(data, welllog_id)
def download_and_construct_las(self, welllog_id: str, curves: List[str] = None) -> LASFile:
def download_and_construct_las(self, config: Configuration, welllog_id: str, curves: List[str] = None) -> LASFile:
"""
Download wellbore and log data and convert to a LAS file.
:param Configuration config: The application configuration
:param str welllog_id: The welllog_id
:param List[str] curves: The Curves to get, or None for all curves
:return: A new instance of the LAS file object
......@@ -175,7 +178,7 @@ class WellLogService:
welllog_data = self._client.get_welllog_data(welllog_id, curves)
mapper = MapWellLogToLas(wellbore, welllog, welllog_data)
mapper = MapWellLogToLas(config, wellbore, welllog, welllog_data)
return mapper.build_las_file()
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment