Soft mapping OSDU -> LasFileMappingFunctions

Some test fixes and improvements

Squash

src/example_opendes_configuration.json

@@ -60,5 +60,20 @@
                 }
             }
         }
-    }
+    },
+    "las_file_mapping": {
+        "mapping": {
+            "Well.WELL": "WELLBORE.data.FacilityName",
+            "Well.UWI": {
+                "type": "function",
+                "function": "extract_uwi_from_name_aliases",
+                "args": ["WELLBORE.data.NameAliases"]
+            },
+            "Curves": {
+                "type": "function",
+                "function": "build_curves_section",
+                "args": ["WELLLOG.data.Curves", "CURVES"]
+            }
+        }
+    }    
 }

src/test/test_property_mapping.py

@@ -33,7 +33,7 @@ class TestPropertyMapper:
         }
 
         # Act
-        result = target.remap_data(source)
+        result = target.remap_data(source, {})
 
         # Assert
         assert result["xyz"] is source["abc"]
@@ -61,7 +61,7 @@ class TestPropertyMapper:
         }
 
         # Act
-        result = target.remap_data(source)
+        result = target.remap_data(source, {})
 
         # Assert
         assert result["dest1"] is source["level1a"]["level2"]["level3"]
@@ -90,7 +90,7 @@ class TestPropertyMapper:
         }
 
         # Act
-        result = target.remap_data(source)
+        result = target.remap_data(source, {})
 
         # Assert
         assert result["dest1"] is None
@@ -112,7 +112,7 @@ class TestPropertyMapper:
         }
 
         # Act
-        result = target.remap_data(source)
+        result = target.remap_data(source, {})
 
         # Assert
         assert result["level1"]["level2"]["a"] is source["a"]
@@ -131,7 +131,7 @@ class TestPropertyMapper:
         }
 
         # Act
-        result = target.remap_data(source)
+        result = target.remap_data(source, {})
 
         # Assert
         assert result["dest"] is source["src"][1]
@@ -149,7 +149,7 @@ class TestPropertyMapper:
         }
 
         # Act
-        result = target.remap_data(source)
+        result = target.remap_data(source, {})
 
         # Assert
         assert result["dest"] is source["src"][0][1]
@@ -167,7 +167,7 @@ class TestPropertyMapper:
         }
 
         # Act
-        result = target.remap_data(source)
+        result = target.remap_data(source, {})
 
         # Assert
         assert result["dest"] == 123
@@ -186,7 +186,7 @@ class TestPropertyMapper:
 
         # Act
         with pytest.raises(WbdutilAttributeError) as ex:
-            target.remap_data(source)
+            target.remap_data(source, {})
 
         assert "src[a]" in ex.value.message
 
@@ -204,7 +204,7 @@ class TestPropertyMapper:
 
         # Act
         with pytest.raises(WbdutilAttributeError) as ex:
-            target.remap_data(source)
+            target.remap_data(source, {})
 
         assert "[1]" in ex.value.message
 
@@ -224,7 +224,7 @@ class TestPropertyMapper:
         }
 
         # Act
-        result = target.remap_data(source)
+        result = target.remap_data(source, {})
 
         # Assert
         assert result["dest"]["dest2"] == "A value"
@@ -248,7 +248,7 @@ class TestPropertyMapper:
         source = {}
 
         # Act
-        result = target.remap_data(source)
+        result = target.remap_data(source, {})
 
         # Assert
         assert result["data"]["WellboreID"] == "12345"
@@ -277,7 +277,7 @@ class TestPropertyMapper:
         }
 
         # Act
-        result = target.remap_data(source)
+        result = target.remap_data(source, {})
 
         # Assert
         assert result["data"]["function_result"] == "12345"
@@ -302,7 +302,7 @@ class TestPropertyMapper:
         # Act
         # Assert
         with pytest.raises(WbdutilAttributeError) as execinfo:
-            target.remap_data(source)
+            target.remap_data(source, {})
 
         assert execinfo.value.message == "The function 'some_function_that_does_not_exist' was not found in the provided mapping functions"
 
@@ -324,7 +324,7 @@ class TestPropertyMapper:
         # Act
         # Assert
         with pytest.raises(WbdutilAttributeError) as execinfo:
-            target.remap_data({})
+            target.remap_data({}, {})
 
         assert execinfo.value.message == "All of the mapping function arguments must be strings."
 
@@ -348,7 +348,7 @@ class TestPropertyMapper:
         }
 
         # Act
-        result = target.remap_data(source)
+        result = target.remap_data(source, {})
 
         # Assert
         result_array = result["data"]["array_remap_result"]

src/test/test_record_mapper.py

@@ -312,7 +312,9 @@ class TestMapWellLogToLas:
         })
         data = DataFrame({"ABC": [1, 2, 3], "XYZ": [9, 8, 7], "IJK": [-1, -2, -3]})
 
-        subject = MapWellLogToLas(wellbore, welllog, data)
+        mockConfig = Mock(spec=Configuration)
+        mockConfig.las_file_mapping = None
+        subject = MapWellLogToLas(mockConfig, wellbore, welllog, data)
 
         # ACT
         result = subject.build_las_file()
@@ -353,7 +355,9 @@ class TestMapWellLogToLas:
 
         data = DataFrame({})
 
-        subject = MapWellLogToLas(wellbore, welllog, data)
+        mockConfig = Mock(spec=Configuration)
+        mockConfig.las_file_mapping = None
+        subject = MapWellLogToLas(mockConfig, wellbore, welllog, data)
 
         # ACT
         result = subject.build_las_file()
@@ -371,7 +375,10 @@ class TestMapWellLogToLas:
         wellbore = Record({"kind": "BoreKind", "acl": {}, "legal": {}, "data": {}})
         data = DataFrame({"ABC": [1, 2, 3], "XYZ": [9, 8, 7], "IJK": [-1, -2, -3]})
 
-        subject = MapWellLogToLas(wellbore, welllog, data)
+        mockConfig = Mock(spec=Configuration)
+        mockConfig.las_file_mapping = None
+
+        subject = MapWellLogToLas(mockConfig, wellbore, welllog, data)
 
         # ACT
         result = subject.build_las_file()

src/test/test_well_service.py

@@ -20,6 +20,8 @@ LasToRecordMapper = record_mapper.LasToRecordMapper
 Record = record_mapper.Record
 WellLogRecord = record_mapper.WellLogRecord
 
+Configuration = import_module(f"{TARGET_MODULE_NAME}.configuration").Configuration
+
 
 class TestWellLogService:
 
@@ -213,8 +215,11 @@ class TestWellLogService:
         arg_welllog_id = "wellog_id argument"
         arg_curves = ["ABC", "XYZ"]
 
+        mockConfig = Mock(spec=Configuration)
+        mockConfig.las_file_mapping = None
+
         # Act
-        result = subject.download_and_construct_las(arg_welllog_id, arg_curves)
+        result = subject.download_and_construct_las(mockConfig, arg_welllog_id, arg_curves)
 
         # Assert
         client.get_welllog_record.assert_called_once_with(arg_welllog_id)
@@ -258,8 +263,11 @@ class TestWellLogService:
         arg_welllog_id = "wellog_id argument"
         arg_curves = ["ABC", "XYZ"]
 
+        mockConfig = Mock(spec=Configuration)
+        mockConfig.las_file_mapping = None
+
         # Act
-        result = subject.download_and_construct_las(arg_welllog_id, arg_curves)
+        result = subject.download_and_construct_las(mockConfig, arg_welllog_id, arg_curves)
 
         # Assert
         client.get_welllog_record.assert_called_once_with(arg_welllog_id)

src/wbdutil/commands/download.py

@@ -31,7 +31,7 @@ def download_las(welllog_id: str,
 
     service = WellLogService(client)
 
-    las_file = service.download_and_construct_las(welllog_id, curves)
+    las_file = service.download_and_construct_las(config, welllog_id, curves)
 
     logger.warning(f"Writing to file {outfile}")
 

src/wbdutil/configuration.py

@@ -62,3 +62,12 @@ class Configuration:
         rtype:  Dict[str, Any]
         """
         return self._config.get("welllog_mapping")
+
+    @property
+    def las_file_mapping(self) -> Union[Dict[str, Any], None]:
+        """
+        Gets the las_file_mapping.
+        return: the las_file_mapping
+        rtype:  Dict[str, Any]
+        """
+        return self._config.get("las_file_mapping")

src/wbdutil/property_mapping.py

@@ -42,7 +42,7 @@ class DictionaryMappingLoader(IPropertyMappingLoader):
         return: the kind string
         rtype: str
         """
-        return self._base_mapping["kind"]
+        return self._base_mapping.get("kind")
 
 
 class PropertyMapper:
@@ -75,22 +75,23 @@ class PropertyMapper:
         :return: A new data object that contains the remapped data
         :rtype: Dict[str, Any]
         """
-        output = self.remap_data(source)
-        output["kind"] = self._kind
-        return output
+        destination = self.remap_data(source, {})
+        destination["kind"] = self._kind
+        return destination
 
-    def remap_data(self, source: any) -> Dict[str, Any]:
+    def remap_data(self, source: any, destination: Dict[str, Any]) -> Dict[str, Any]:
         """
         Remap the data in the source object using the mapping dictionary to a new object
         :param any source: The source object
-        :return: A new data object that contains the remapped data
+        :param Dict[str, Any] destination: The destination object
+
+        :return: The populated destination object
         :rtype: Dict[str, Any]
         """
 
-        return self._remap_data(source, self._mapping)
+        return self._remap_data(source, self._mapping, destination)
 
-    def _remap_data(self, source: any, mapping: Dict[str, Any]) -> Dict[str, Any]:
-        extensible = {}
+    def _remap_data(self, source: any, mapping: Dict[str, Any], extensible: Dict[str, Any]) -> Dict[str, Any]:
 
         # iterate through the mapping dictionary remapping data as we go
         for target_field, source_field in mapping.items():
@@ -156,7 +157,7 @@ class PropertyMapper:
 
             remapped_array = []
             for item in source_array:
-                remapped_array.append(self._remap_data(item, submapping))
+                remapped_array.append(self._remap_data(item, submapping, {}))
 
             return remapped_array
 

src/wbdutil/record_mapper.py

 from typing import Dict, List
 import urllib
 import lasio
+from lasio.las_items import CurveItem, SectionItems
 from knack.log import get_logger
 from lasio.las import LASFile
 from pandas.core.frame import DataFrame
@@ -168,7 +169,6 @@ class LasToRecordMapper:
         :returns: A Wellbore record object that can be uploaded to OSDU as JSON payload.
         :rtype: Record
         """
-
         mapper = PropertyMapper(
             DictionaryMappingLoader(self.config.wellbore_mapping or self._DEFAULT_WELLBORE_MAPPING),
             self.config,
@@ -184,6 +184,7 @@ class LasToRecordMapper:
         :returns: A Well Log record object that can be uploaded to OSDU as JSON payload.
         :rtype: Record
         """
+
         mapper = PropertyMapper(
             DictionaryMappingLoader(self.config.welllog_mapping or self._DEFAULT_WELLLOG_MAPPING),
             self.config,
@@ -203,16 +204,36 @@ class LasToRecordMapper:
 
 
 class MapWellLogToLas:
-    def __init__(self, wellbore: Record, welllog: Record, data: DataFrame) -> None:
+    _DEFAULT_LAS_MAPPING = {
+        "mapping": {
+            "Well.WELL": "WELLBORE.data.FacilityName",
+            "Well.UWI": {
+                "type": "function",
+                "function": "extract_uwi_from_name_aliases",
+                "args": ["WELLBORE.data.NameAliases"]
+            },
+            "Curves": {
+                "type": "function",
+                "function": "build_curves_section",
+                "args": ["WELLLOG.data.Curves", "CURVES"]
+            }
+        }
+    }
+
+    def __init__(self, configuration: Configuration, wellbore: Record, welllog: Record, data: DataFrame) -> None:
         """
         Construct a new instance of MapWellLogToLas
+        :param configuration Configuration: The application configuration
         :param wellbore Record: The wellbore record
         :param welllog Record: The well log record
         :param data DataFrame: The well log curve data
         """
-        self._wellbore = wellbore
-        self._welllog = welllog
-        self._data = data
+        self._config = configuration
+        self._data = {
+            "CURVES": data,
+            "WELLLOG": welllog,
+            "WELLBORE": wellbore
+        }
 
     def build_las_file(self) -> LASFile:
         """
@@ -220,18 +241,25 @@ class MapWellLogToLas:
         :returns: A new instance of LASFile.
         :rtype: LASFile
         """
+
+        mapper = PropertyMapper(
+            DictionaryMappingLoader(self._config.las_file_mapping or self._DEFAULT_LAS_MAPPING),
+            None,
+            LasFileMappingFunctions())
+
         las = lasio.LASFile()
+        las.sections = mapper.remap_data(self._data, las.sections)
+        return las
 
-        # Add wellbore info
-        nameAlias = self._wellbore.data.get("NameAliases")
-        las.well.UWI.value = nameAlias[0].get("AliasName") if nameAlias is not None and len(nameAlias) > 0 else None
-        las.well.WELL.value = self._wellbore.data.get("FacilityName")
 
-        # Get wellog curve info
-        wl_curves = self._welllog.data.get("Curves")
+class LasFileMappingFunctions:
+    def extract_uwi_from_name_aliases(self, name_aliases: list):
+        return name_aliases[0].get("AliasName") if name_aliases is not None and len(name_aliases) > 0 else None
 
-        # Add curves to las file
-        for curve in self._data:
+    def build_curves_section(self, wl_curves: list, data: DataFrame) -> SectionItems:
+        curve_items = []
+
+        for curve in data:
             # Try to get the curve units from the welllog
             if wl_curves is not None:
                 wl_curve_matches = [c for c in wl_curves if c.get("CurveID") == curve]
@@ -241,9 +269,9 @@ class MapWellLogToLas:
             else:
                 las_units = None
 
-            las.append_curve(curve, self._data[curve], unit=las_units)
+            curve_items.append(CurveItem(curve, las_units, "", "", data[curve]))
 
-        return las
+        return SectionItems(curve_items)
 
 
 class MappingUtilities:

src/wbdutil/well_service.py

@@ -3,6 +3,8 @@ from typing import Dict, List, Tuple
 import urllib
 from knack.log import get_logger
 from lasio.las import LASFile
+
+from wbdutil.configuration import Configuration
 from .file_loader import FileValidationError, LasParser, LocalFileLoader
 from .osdu_client import OsduClient, DataLoaderWebResponseError
 from .record_mapper import LasToRecordMapper, Record, MappingUtilities, MapWellLogToLas
@@ -148,9 +150,10 @@ class WellLogService:
         data = las_data.df().reset_index()
         self._client.add_welllog_data(data, welllog_id)
 
-    def download_and_construct_las(self, welllog_id: str, curves: List[str] = None) -> LASFile:
+    def download_and_construct_las(self, config: Configuration, welllog_id: str, curves: List[str] = None) -> LASFile:
         """
         Download wellbore and log data and convert to a LAS file.
+        :param Configuration config: The application configuration
         :param str welllog_id: The welllog_id
         :param List[str] curves: The Curves to get, or None for all curves
         :return: A new instance of the LAS file object
@@ -175,7 +178,7 @@ class WellLogService:
 
         welllog_data = self._client.get_welllog_data(welllog_id, curves)
 
-        mapper = MapWellLogToLas(wellbore, welllog, welllog_data)
+        mapper = MapWellLogToLas(config, wellbore, welllog, welllog_data)
 
         return mapper.build_las_file()