Merge branch '16-las-output-from-bulk-data-ingested-in-wellbore-ddms' into 'main'

Resolve "LAS Output from bulk data ingested in wellbore DDMS"

Closes #16

See merge request osdu/platform/data-flow/data-loading/wellbore-ddms-las-loader!19

src/lascli.py

@@ -49,6 +49,9 @@ class LasCommandLoader(CLICommandsLoader):
             group.command('welllog', 'welllog')
             group.command('curves', 'welllog_data')
 
+        with CommandGroup(self, 'download', 'lasloader.commands.download#{}') as group:
+            group.command('welllog', 'download_las')
+
         return OrderedDict(self.command_table)
 
     def load_arguments(self, command):
@@ -114,6 +117,26 @@ class LasCommandLoader(CLICommandsLoader):
             arg_context.argument('curves', type=str, options_list=('--curves'), nargs='*',
                                  help='The list of curves to retrieve (space separated list), returns all curves if not specified.')
 
+        with ArgumentsContext(self, 'download') as arg_context:
+            arg_context.argument('url', type=str, options_list=('-u', '--url'),
+                                 help='Url for the OSDU instance.')
+
+        with ArgumentsContext(self, 'download') as arg_context:
+            arg_context.argument('token', type=str, options_list=('-t', '--token'),
+                                 help='A valid bearer token used to authenticate with the OSDU instance.')
+
+        with ArgumentsContext(self, 'download') as arg_context:
+            arg_context.argument('welllog_id', type=str, options_list=('--welllog_id'),
+                                 help='The welllog id of the record to retrieve.')
+
+        with ArgumentsContext(self, 'download') as arg_context:
+            arg_context.argument('curves', type=str, options_list=('--curves'), nargs='*',
+                                 help='The list of curves to retrieve (space separated list), returns all curves if not specified.')
+
+        with ArgumentsContext(self, 'download') as arg_context:
+            arg_context.argument('outfile', type=str, options_list=('--out'),
+                                 help='The output file path')
+
         super(LasCommandLoader, self).load_arguments(command)
 
 

src/lasloader/commands/download.py

+
+from knack.log import get_logger
+from lasloader.osdu_client import OsduClient
+from lasloader.configuration import Configuration
+from lasloader.file_loader import LocalFileLoader
+from lasloader.well_service import WellLogService
+
+
+logger = get_logger(__name__)
+
+
+def download_las(url: str,
+                 token: str,
+                 config_path: str,
+                 welllog_id: str,
+                 outfile: str,
+                 curves: list[str] = None):
+    """
+    Retrieve welllog data from an OSDU instance and save to a LAS format file
+    :param str url: The base URL of the OSDU instance
+    :param str token: a valid bearer token that is used to authenticate against the OSDU instance
+    :param str config_path: Path to the las metadata file
+    :param str welllog_id: The well bore id of the record to retrieve
+    :param str outfile: The path of the output file
+    :param list[str] curves: The curves to retrieve, use None to get all curves
+    """
+    config = Configuration(LocalFileLoader(), config_path)
+    client = OsduClient(url, token, config.data_partition_id)
+
+    service = WellLogService(client)
+
+    las_file = service.download_and_construct_las(welllog_id, curves)
+
+    logger.warning(f"Writing to file {outfile}")
+
+    with open(outfile, mode='w') as f:
+        las_file.write(f, version=2)

src/lasloader/record_mapper.py

@@ -2,10 +2,15 @@ from dataclasses import dataclass
 from typing import Union
 import urllib
 import lasio
+from knack.log import get_logger
+from lasio.las import LASFile
 from pandas.core.frame import DataFrame
 from lasloader.configuration import Configuration
 
 
+logger = get_logger(__name__)
+
+
 @dataclass
 class Record:
     kind: str
@@ -289,3 +294,67 @@ class LasToRecordMapper:
         """
 
         return self.las.df().reset_index()
+
+
+class MapWellLogToLas:
+    def __init__(self, wellbore: Record, welllog: Record, data: DataFrame) -> None:
+        """
+        Construct a new instance of MapWellLogToLas
+        :param wellbore Record: The wellbore record
+        :param welllog Record: The well log record
+        :param data DataFrame: The well log curve data
+        """
+        self._wellbore = wellbore
+        self._welllog = welllog
+        self._data = data
+
+    def build_las_file(self) -> LASFile:
+        """
+        Build a new instance of LASFile and populate it with data from the welllog
+        :returns: A new instance of LASFile.
+        :rtype: LASFile
+        """
+        las = lasio.LASFile()
+
+        # Add wellbore info
+        nameAlias = self._wellbore.data.get("NameAliases")
+        las.well.UWI.value = nameAlias[0].get("AliasName") if nameAlias is not None and len(nameAlias) > 0 else None
+        las.well.WELL.value = self._wellbore.data.get("FacilityName")
+
+        # Get wellog curve info
+        wl_curves = self._welllog.data.get("Curves")
+
+        # Add curves to las file
+        for curve in self._data:
+            # Try to get the curve units from the welllog
+            if wl_curves is not None:
+                wl_curve_matches = [c for c in wl_curves if c.get("CurveID") == curve]
+                wl_curve = wl_curve_matches[0] if wl_curve_matches is not None and len(wl_curve_matches) > 0 else {}
+                wl_unit = wl_curve.get("CurveUnit")
+                las_units = MappingUtilities.convert_osdu_unit_to_raw_unit(wl_unit)
+            else:
+                las_units = None
+
+            las.append_curve(curve, self._data[curve], unit=las_units)
+
+        return las
+
+
+class MappingUtilities:
+    @staticmethod
+    def convert_osdu_unit_to_raw_unit(osdu_unit: str) -> str:
+        """
+        Extract and return the unit section from a OSDU qualified unit.
+        :param str osdu_unit: The osdu qualified unit
+        :return: The unit part or None if the unit cannot be extracted.
+        :rtype: str
+        """
+        if osdu_unit is None:
+            return None
+
+        split_units = osdu_unit.split(':')
+        if (split_units is None or len(split_units) < 3):
+            logger.warning(f"Could not extract units from {osdu_unit}")
+            return None
+
+        return urllib.parse.unquote(split_units[2])

src/lasloader/well_service.py

 import json
 from typing import Tuple
 import urllib
-
-from lasio.las import LASFile
 from knack.log import get_logger
+from lasio.las import LASFile
 from lasloader.file_loader import FileValidationError, LasParser, LocalFileLoader
 from lasloader.osdu_client import OsduClient, LasLoaderWebResponseError
-from lasloader.record_mapper import LasToRecordMapper, Record
+from lasloader.record_mapper import LasToRecordMapper, Record, MappingUtilities, MapWellLogToLas
 
 
 logger = get_logger(__name__)
@@ -48,7 +47,7 @@ class WellLogService:
         for curve in curves:
             mnemonic = curve["Mnemonic"]
             osdu_unit = curve["CurveUnit"]
-            unit = self._convert_osdu_unit_to_raw_unit(osdu_unit)
+            unit = MappingUtilities.convert_osdu_unit_to_raw_unit(osdu_unit)
 
             curve["LogCurveFamilyID"] = self.recognize_curve_family(mnemonic, unit, data_partition_id) if unit is not None else None
 
@@ -105,22 +104,36 @@ class WellLogService:
         data = las_data.df().reset_index()
         self._client.add_welllog_data(data, welllog_id)
 
-    def _convert_osdu_unit_to_raw_unit(self, osdu_unit) -> str:
+    def download_and_construct_las(self, welllog_id: str, curves: list[str] = None) -> LASFile:
         """
-        Extract and return the unit section from a OSDU qualified unit.
-        :param str osdu_unit: The osdu qualified unit
-        :return: The unit part or None if the unit cannot be extracted.
-        :rtype: str
+        Download wellbore and log data and convert to a LAS file.
+        :param str welllog_id: The welllog_id
+        :param list[str] curves: The Curves to get, or None for all curves
+        :return: A new instance of the LAS file object
+        :rtype: LASFile
         """
-        if osdu_unit is None:
-            return None
 
-        split_units = osdu_unit.split(':')
-        if (split_units is None or len(split_units) < 3):
-            logger.warning(f"Could not extract units from {osdu_unit}")
-            return None
+        logger.warning(f"Getting welllog ID {welllog_id}")
+        welllog = self._client.get_welllog_record(welllog_id)
+        wellbore_id = welllog.data.get("WellboreID")
+
+        if wellbore_id is None:
+            logger.error("The welllog records contained no wellbore Id, cannot get wellbore")
+            wellbore = Record(None, {}, {}, {})
+        else:
+            logger.warning(f"Getting wellbore ID {wellbore_id}")
+            wellbore = self._client.get_wellbore_record(wellbore_id)
+
+        logger.warning(f"Getting curve data for welllog ID {welllog_id}")
+
+        if curves:
+            logger.warning(f"Curves: {curves}")
+
+        welllog_data = self._client.get_welllog_data(welllog_id, curves)
+
+        mapper = MapWellLogToLas(wellbore, welllog, welllog_data)
 
-        return urllib.parse.unquote(split_units[2])
+        return mapper.build_las_file()
 
     def _validate_welllog_data_ingest_file(self, las_data: LASFile, welllog_id: str) -> None:
         """
@@ -176,7 +189,7 @@ class WellBoreService:
         logger.warning(f"New wellbore IDs: {ids}")
 
         wellbore_id = self._safe_get_first_record(ids)
-        logger.info(json.dumps(self._client.get_wellbore_record(wellbore_id), indent=4, sort_keys=True))
+        logger.info(json.dumps(self._client.get_wellbore_record(wellbore_id).get_raw_data(), indent=4, sort_keys=True))
 
         welllog_record = mapper.map_to_well_log_record(wellbore_id)
 

src/test/test_record_mapper.py

@@ -3,7 +3,8 @@ import urllib
 from pandas import DataFrame
 from lasio import LASFile
 from unittest.mock import Mock
-from lasloader.record_mapper import Record, AttributeBuilder, LasToRecordMapper, WellLogRecord, WellboreRecord
+from lasloader.record_mapper import Record, AttributeBuilder, LasToRecordMapper, WellLogRecord, MappingUtilities
+from lasloader.record_mapper import MapWellLogToLas, WellboreRecord
 from lasloader.file_loader import LasParser, LocalFileLoader
 from lasloader.configuration import Configuration
 
@@ -373,3 +374,116 @@ class TestWellLogRecord:
         assert record.legal == welllog['legal']
         assert record.get_raw_data() == welllog
         assert record.get_curveids() == ['abc', 'xyz']
+
+
+class TestMappingUtilities:
+    @pytest.mark.parametrize("osdu_unit,expected",
+                             [("opendes:reference-data--UnitOfMeasure:M:", "M"), ("opendes:reference-data--UnitOfMeasure:GAPI:", "GAPI"),
+                              ("opendes:reference-data--UnitOfMeasure:US%2FF:", "US/F"),
+                              ("opendes:reference-data--UnitOfMeasure:G%2FC3:", "G/C3")])
+    def test_convert_osdu_unit_to_raw_unit(self, osdu_unit, expected):
+        # Assemble
+        # Act
+        result = MappingUtilities.convert_osdu_unit_to_raw_unit(osdu_unit)
+
+        # Assert
+        assert result == expected
+
+    @pytest.mark.parametrize("osdu_unit", ["", None, "opendes:reference-data--UnitOfMeasure"])
+    def test_convert_osdu_unit_to_raw_unit_bad_data(self, osdu_unit):
+        # Assemble
+        # Act
+        result = MappingUtilities.convert_osdu_unit_to_raw_unit(osdu_unit)
+
+        # Assert
+        assert result is None
+
+
+class TestMapWellLogToLas:
+    def test_build_las_file_happy(self):
+        # Assemble
+        logdata = {
+            "Curves": [
+                {"CurveID": "ABC", "CurveUnit": "opendes:reference-data--UnitOfMeasure:M:"},
+                {"CurveID": "LMN", "CurveUnit": "opendes:reference-data--UnitOfMeasure:Ft:"},
+                {"CurveID": "XYZ", "CurveUnit": "opendes:reference-data--UnitOfMeasure:GAPI:"}]
+            }
+
+        welllog = Record("LogKind", {}, {}, logdata)
+        wellbore = Record("BoreKind", {}, {}, {"FacilityName": "Well name", "NameAliases": [{"AliasName": "Some Name"}]})
+        data = DataFrame({"ABC": [1, 2, 3], "XYZ": [9, 8, 7], "IJK": [-1, -2, -3]})
+
+        subject = MapWellLogToLas(wellbore, welllog, data)
+
+        # ACT
+        result = subject.build_las_file()
+
+        # Assert
+        assert result.well.UWI.value == "Some Name"
+        assert result.well.WELL.value == "Well name"
+
+        assert len(result.curves) == 3
+        assert result.curves[0].mnemonic == "ABC"
+        assert result.curves[0].unit == "M"
+        assert (result.curves[0].data == [1, 2, 3]).all()
+
+        assert result.curves[1].mnemonic == "XYZ"
+        assert result.curves[1].unit == "GAPI"
+        assert (result.curves[1].data == [9, 8, 7]).all()
+
+        assert result.curves[2].mnemonic == "IJK"
+        assert result.curves[2].unit is None
+        assert (result.curves[2].data == [-1, -2, -3]).all()
+
+    def test_build_las_file_no_curve_data(self):
+        # Assemble
+        logdata = {
+            "Curves": [
+                {"CurveID": "ABC", "CurveUnit": "opendes:reference-data--UnitOfMeasure:M:"},
+                {"CurveID": "LMN", "CurveUnit": "opendes:reference-data--UnitOfMeasure:Ft:"},
+                {"CurveID": "XYZ", "CurveUnit": "opendes:reference-data--UnitOfMeasure:GAPI:"}]
+            }
+
+        welllog = Record("LogKind", {}, {}, logdata)
+        wellbore = Record("BoreKind", {}, {}, {"FacilityName": "Well name", "NameAliases": [{"AliasName": "Some Name"}]})
+        data = DataFrame({})
+
+        subject = MapWellLogToLas(wellbore, welllog, data)
+
+        # ACT
+        result = subject.build_las_file()
+
+        # Assert
+        assert result.well.UWI.value == "Some Name"
+        assert result.well.WELL.value == "Well name"
+
+        assert len(result.curves) == 0
+
+    def test_build_las_file_empty_log_and_bore(self):
+        # Assemble
+
+        welllog = Record("LogKind", {}, {}, {})
+        wellbore = Record("BoreKind", {}, {}, {})
+        data = DataFrame({"ABC": [1, 2, 3], "XYZ": [9, 8, 7], "IJK": [-1, -2, -3]})
+
+        subject = MapWellLogToLas(wellbore, welllog, data)
+
+        # ACT
+        result = subject.build_las_file()
+
+        # Assert
+        assert result.well.UWI.value is None
+        assert result.well.WELL.value is None
+
+        assert len(result.curves) == 3
+        assert result.curves[0].mnemonic == "ABC"
+        assert result.curves[0].unit is None
+        assert (result.curves[0].data == [1, 2, 3]).all()
+
+        assert result.curves[1].mnemonic == "XYZ"
+        assert result.curves[1].unit is None
+        assert (result.curves[1].data == [9, 8, 7]).all()
+
+        assert result.curves[2].mnemonic == "IJK"
+        assert result.curves[2].unit is None
+        assert (result.curves[2].data == [-1, -2, -3]).all()

src/test/test_well_service.py

@@ -119,32 +119,97 @@ class TestWellLogService:
         assert welllog_well_name == "Well-ABC"
         assert welllog_curve_ids == ["DEPT", "BWV", "DT"]
 
-    @pytest.mark.parametrize("osdu_unit,expected",
-                             [("opendes:reference-data--UnitOfMeasure:M:", "M"), ("opendes:reference-data--UnitOfMeasure:GAPI:", "GAPI"),
-                              ("opendes:reference-data--UnitOfMeasure:US%2FF:", "US/F"),
-                              ("opendes:reference-data--UnitOfMeasure:G%2FC3:", "G/C3")])
-    def test_convert_osdu_unit_to_raw_unit(self, osdu_unit, expected):
-        # Assemble
+    def test_download_and_construct_las(self):
+        logdata = {
+            "WellboreID": "well bore Id",
+            "Curves": [
+                {"CurveID": "ABC", "CurveUnit": "opendes:reference-data--UnitOfMeasure:M:"},
+                {"CurveID": "LMN", "CurveUnit": "opendes:reference-data--UnitOfMeasure:Ft:"},
+                {"CurveID": "XYZ", "CurveUnit": "opendes:reference-data--UnitOfMeasure:GAPI:"}]
+            }
+
+        welllog = Record("LogKind", {}, {}, logdata)
+        wellbore = WellboreRecord({"data": {"FacilityName": "Well name", "NameAliases": [{"AliasName": "Some Name"}]}})
+        data = DataFrame({"ABC": [1, 2, 3], "XYZ": [9, 8, 7], "IJK": [-1, -2, -3]})
+
         client = Mock(spec=OsduClient)
+
+        client.get_welllog_record.return_value = welllog
+        client.get_wellbore_record.return_value = wellbore
+        client.get_welllog_data.return_value = data
         subject = WellLogService(client)
 
+        arg_welllog_id = "wellog_id argument"
+        arg_curves = ["ABC", "XYZ"]
+
         # Act
-        result = subject._convert_osdu_unit_to_raw_unit(osdu_unit)
+        result = subject.download_and_construct_las(arg_welllog_id, arg_curves)
 
         # Assert
-        assert result == expected
+        client.get_welllog_record.assert_called_once_with(arg_welllog_id)
+        client.get_wellbore_record.assert_called_once_with("well bore Id")
+        client.get_welllog_data.assert_called_once_with(arg_welllog_id, arg_curves)
+
+        assert result.well.UWI.value == "Some Name"
+        assert result.well.WELL.value == "Well name"
+
+        assert len(result.curves) == 3
+        assert result.curves[0].mnemonic == "ABC"
+        assert result.curves[0].unit == "M"
+        assert (result.curves[0].data == [1, 2, 3]).all()
+
+        assert result.curves[1].mnemonic == "XYZ"
+        assert result.curves[1].unit == "GAPI"
+        assert (result.curves[1].data == [9, 8, 7]).all()
+
+        assert result.curves[2].mnemonic == "IJK"
+        assert result.curves[2].unit is None
+        assert (result.curves[2].data == [-1, -2, -3]).all()
+
+    def test_download_and_construct_las_no_wellbore(self):
+        logdata = {
+            "WellboreID": None,
+            "Curves": [
+                {"CurveID": "ABC", "CurveUnit": "opendes:reference-data--UnitOfMeasure:M:"},
+                {"CurveID": "LMN", "CurveUnit": "opendes:reference-data--UnitOfMeasure:Ft:"},
+                {"CurveID": "XYZ", "CurveUnit": "opendes:reference-data--UnitOfMeasure:GAPI:"}]
+            }
+
+        welllog = Record("LogKind", {}, {}, logdata)
+        data = DataFrame({"ABC": [1, 2, 3], "XYZ": [9, 8, 7], "IJK": [-1, -2, -3]})
 
-    @pytest.mark.parametrize("osdu_unit", ["", None, "opendes:reference-data--UnitOfMeasure"])
-    def test_convert_osdu_unit_to_raw_unit_bad_data(self, osdu_unit):
-        # Assemble
         client = Mock(spec=OsduClient)
+
+        client.get_welllog_record.return_value = welllog
+        client.get_welllog_data.return_value = data
         subject = WellLogService(client)
 
+        arg_welllog_id = "wellog_id argument"
+        arg_curves = ["ABC", "XYZ"]
+
         # Act
-        result = subject._convert_osdu_unit_to_raw_unit(osdu_unit)
+        result = subject.download_and_construct_las(arg_welllog_id, arg_curves)
 
         # Assert
-        assert result is None
+        client.get_welllog_record.assert_called_once_with(arg_welllog_id)
+        client.get_wellbore_record.assert_not_called()
+        client.get_welllog_data.assert_called_once_with(arg_welllog_id, arg_curves)
+
+        assert result.well.UWI.value is None
+        assert result.well.WELL.value is None
+
+        assert len(result.curves) == 3
+        assert result.curves[0].mnemonic == "ABC"
+        assert result.curves[0].unit == "M"
+        assert (result.curves[0].data == [1, 2, 3]).all()
+
+        assert result.curves[1].mnemonic == "XYZ"
+        assert result.curves[1].unit == "GAPI"
+        assert (result.curves[1].data == [9, 8, 7]).all()
+
+        assert result.curves[2].mnemonic == "IJK"
+        assert result.curves[2].unit is None
+        assert (result.curves[2].data == [-1, -2, -3]).all()
 
 
 class TestWellBoreService:
@@ -155,7 +220,7 @@ class TestWellBoreService:
         mock_well_log_service = Mock(spec=WellLogService)
         mock_mapper = Mock(spec=LasToRecordMapper)
 
-        well_bore_record = Record("well_bore_kind", {}, {}, {})
+        well_bore_record = WellboreRecord({"Kind": "well_bore_kind", "acl": {}, "legal": {}, "data": {}})
         well_log_record = Record("well_log_kind", {}, {}, {})
         well_log_record_recognized = Record("well_log_rec_kind", {"blah": "blah"}, {}, {})
         well_log_record_returned = WellLogRecord({"kind": "returned_welllog_record"})
@@ -170,7 +235,7 @@ class TestWellBoreService:
         mock_mapper.extract_log_data.return_value = well_log_data
 
         mock_client.create_wellbore.return_value = well_bore_ids
-        mock_client.get_wellbore_record.return_value = vars(well_log_record)
+        mock_client.get_wellbore_record.return_value = well_bore_record
         mock_client.create_welllog.return_value = well_log_ids
         mock_client.get_welllog_record.return_value = well_log_record_returned