Merge branch 'develop' into 16-las-output-from-bulk-data-ingested-in-wellbore-ddms

# Conflicts:
#	src/lasloader/well_service.py
#	src/test/test_record_mapper.py
#	src/test/test_well_service.py

src/lascli.py

@@ -26,6 +26,7 @@ class LasCommandHelp(CLIHelp):
 
 class LasCommandLoader(CLICommandsLoader):
     """ Implementation of CLICommandsLoader for the las loader CLI """
+
     def __init__(self, *args, **kwargs):
         super(LasCommandLoader, self).__init__(
             *args,
@@ -36,10 +37,13 @@ class LasCommandLoader(CLICommandsLoader):
         """Load all commands"""
 
         with CommandGroup(self, 'fileload', 'lasloader.commands.file_load#{}') as group:
-            group.command('ingest', 'ingest')
             group.command('convert', 'convert')
             group.command('print', 'printlas')
 
+        with CommandGroup(self, 'ingest', 'lasloader.commands.ingest#{}') as group:
+            group.command('wellbore', 'wellbore')
+            group.command('data', 'welllog_data')
+
         with CommandGroup(self, 'list', 'lasloader.commands.list_osdu#{}') as group:
             group.command('wellbore', 'wellbore')
             group.command('welllog', 'welllog')
@@ -53,29 +57,37 @@ class LasCommandLoader(CLICommandsLoader):
     def load_arguments(self, command):
         """Load arguments for commands"""
 
+        with ArgumentsContext(self, '') as arg_context:
+            arg_context.argument('config_path', type=str, options_list=('-c', '--config_path'),
+                                 help='The path and filename of the configuration file.')
+
         with ArgumentsContext(self, 'fileload') as arg_context:
             arg_context.argument('input_path', type=str, options_list=('-p', '--path'),
                                  help='Path to a file or folder containing LAS file(s) to upload.')
 
-        with ArgumentsContext(self, '') as arg_context:
-            arg_context.argument('config_path', type=str, options_list=('-c', '--config_path'),
-                                 help='The path and filename of the configuration file.')
+        with ArgumentsContext(self, 'fileload convert') as arg_context:
+            arg_context.argument('wellbore_id', type=str, options_list=('--wellbore_id'),
+                                 help='The wellbore id to use when printing to file.')
 
-        with ArgumentsContext(self, 'fileload ingest') as arg_context:
+        with ArgumentsContext(self, 'ingest') as arg_context:
+            arg_context.argument('input_path', type=str, options_list=('-p', '--path'),
+                                 help='Path to a file or folder containing LAS file(s) to upload.')
+
+        with ArgumentsContext(self, 'ingest') as arg_context:
             arg_context.argument('url', type=str, options_list=('-u', '--url'),
                                  help='The base url of the OSDU instance.')
 
-        with ArgumentsContext(self, 'fileload ingest') as arg_context:
+        with ArgumentsContext(self, 'ingest') as arg_context:
             arg_context.argument('token', type=str, options_list=('-t', '--token'),
                                  help='A valid bearer token used to authenticate with the OSDU instance.')
 
-        with ArgumentsContext(self, 'fileload ingest') as arg_context:
+        with ArgumentsContext(self, 'ingest wellbore') as arg_context:
             arg_context.argument('no_recognize', options_list=('--norecognize'), action='store_true',
                                  help='If specified LASCLI won\'t attempt to recognize the curve families.')
 
-        with ArgumentsContext(self, 'fileload convert') as arg_context:
-            arg_context.argument('wellbore_id', type=str, options_list=('--wellbore_id'),
-                                 help='The wellbore id to use when printing to file.')
+        with ArgumentsContext(self, 'ingest data') as arg_context:
+            arg_context.argument('welllog_id', options_list=('--welllog_id'),
+                                 help='The welllog id of the record into which to write the data.')
 
         with ArgumentsContext(self, 'list') as arg_context:
             arg_context.argument('url', type=str, options_list=('-u', '--url'),

src/lasloader/commands/file_load.py

 from pathlib import Path
-from ntpath import basename
 from knack.log import get_logger
-from lasloader.file_loader import LasParser, LocalFileLoader, FileUtilities
+from lasloader.file_loader import LasParser, LocalFileLoader
 from lasloader.record_mapper import LasToRecordMapper
-from lasloader.osdu_client import OsduClient
 from lasloader.configuration import Configuration
 from lasloader.json_writer import JsonToFile
-from lasloader.well_service import WellBoreService, WellLogService
 
 
 logger = get_logger(__name__)
 
 
-def ingest(
-        input_path: str,
-        url: str,
-        token: str,
-        config_path: str,
-        no_recognize: bool = False):
-    """
-    Ingest a las file (single) or directory of las files (bulk) into OSDU
-    :param str input_path: Path and filename of a las file OR path to directory containing las and config files.
-    :param str url: The base URL of the OSDU instance to which to upload the file.
-    :param str token: A valid bearer token that is used to authenticate the ingest request.
-    :param str config_path: Path to the las metadata file.
-    :param bool no_recognize: If true don't attempt to recognize the curves, otherwise recognize the curves
-    """
-    config = Configuration(LocalFileLoader(), config_path)
-    las_parser = LasParser(LocalFileLoader())
-    client = OsduClient(url, token, config.data_partition_id)
-    service = WellBoreService(client, WellLogService(client))
-
-    failed_ingests = []
-
-    for file_path in FileUtilities.get_filenames(Path(input_path), '.las'):
-        logger.warning(f"Begining ingest of LAS file: {file_path}")
-
-        try:
-            las_data = las_parser.load_las_file(file_path)
-            las_mapper = LasToRecordMapper(las_data, config)
-            service.file_ingest(las_mapper, config.data_partition_id, no_recognize)
-
-            logger.warning(f"Ingest complete: {basename(file_path)}")
-
-        except Exception as e:
-            failed_ingests.append(f"{basename(file_path)} ({str(e)})")
-            logger.error(f"Ingest failed: {basename(file_path)} (see summary for details)")
-
-    if failed_ingests != []:
-        logger.error("SUMMARY - files not ingested: " + ", ".join(failed_ingests))
-
-
 def printlas(input_path):
     """
-    Print a las file header
-    :param str input_path: Path and filename of a las file
+    Print a LAS file header
+    :param str input_path: Path and filename of a LAS file
     """
 
     las_parser = LasParser(LocalFileLoader())
@@ -65,9 +23,9 @@ def printlas(input_path):
 
 def convert(input_path: str, config_path: str, wellbore_id: str):
     """
-    Convert a las file to Well Bore and Well log and write to json files.
-    :param str input_path: Path and filename of a las file
-    :param str config_path: Path to the las metadata file
+    Convert a LAS file to Wellbore and Well Log and write to json files.
+    :param str input_path: Path and filename of a LAS file
+    :param str config_path: Path to the LAS metadata file
     :param str wellbore_id: The wellbore id
     """
 

src/lasloader/commands/ingest.py

+from pathlib import Path
+from ntpath import basename
+from knack.log import get_logger
+from lasloader.file_loader import LasParser, LocalFileLoader, FileUtilities
+from lasloader.record_mapper import LasToRecordMapper
+from lasloader.osdu_client import OsduClient
+from lasloader.configuration import Configuration
+from lasloader.well_service import WellBoreService, WellLogService
+
+
+logger = get_logger(__name__)
+
+
+def wellbore(
+        input_path: str,
+        url: str,
+        token: str,
+        config_path: str,
+        no_recognize: bool = False):
+    """
+    Ingest a LAS file (single) or directory of LAS files (bulk) into OSDU
+    :param str input_path: Path and filename of a LAS file OR path to directory containing LAS and config files.
+    :param str url: The base URL of the OSDU instance to which to upload the file.
+    :param str token: A valid bearer token that is used to authenticate the ingest request.
+    :param str config_path: Path to the LAS metadata file.
+    :param bool no_recognize: If true don't attempt to recognize the curves, otherwise recognize the curves
+    """
+    config = Configuration(LocalFileLoader(), config_path)
+    las_parser = LasParser(LocalFileLoader())
+    client = OsduClient(url, token, config.data_partition_id)
+    service = WellBoreService(client, WellLogService(client))
+
+    failed_ingests = []
+
+    for file_path in FileUtilities.get_filenames(Path(input_path), '.las'):
+        logger.warning(f"Begining ingest of LAS file: {file_path}")
+
+        try:
+            las_data = las_parser.load_las_file(file_path)
+            las_mapper = LasToRecordMapper(las_data, config)
+            service.file_ingest(las_mapper, config.data_partition_id, no_recognize)
+
+            logger.warning(f"Ingest complete: {basename(file_path)}")
+
+        except Exception as e:
+            failed_ingests.append(f"{basename(file_path)} ({str(e)})")
+            logger.error(f"Ingest failed: {basename(file_path)} (see summary for details)")
+
+    if failed_ingests != []:
+        logger.error("SUMMARY - files not ingested: " + ", ".join(failed_ingests))
+
+
+def welllog_data(
+        welllog_id: str,
+        input_path: str,
+        url: str,
+        token: str,
+        config_path: str):
+    """
+    Write data from a LAS file to an existing Well Log.
+    :param str welllog_id: ID of well log to be updated.
+    :param str input_path: Path and filename of a LAS file containing data to be written to existing well log.
+    :param str url: The base URL of the OSDU instance to which to find the existing Well Log.
+    :param str token: A valid bearer token that is used to authenticate the update request.
+    :param str config_path: Path to the LAS metadata file.
+    """
+    config = Configuration(LocalFileLoader(), config_path)
+    client = OsduClient(url, token, config.data_partition_id)
+    service = WellBoreService(client, WellLogService(client))
+
+    service.ingest_welllog_data(input_path, welllog_id)
+
+    logger.warning(f"Ingest complete: Data written from {basename(input_path)} to Well Log ({welllog_id}).")

src/lasloader/commands/list_osdu.py

-
 import json
 from knack.log import get_logger
 from lasloader.osdu_client import OsduClient

src/lasloader/file_loader.py

@@ -79,12 +79,33 @@ class LasParser:
         """
         Validate that the Well Name attribute of the inputted LAS file has been populated.
 
-        :param las LASFile The LASFile object to be validated.
+        :param las LASFile: The LASFile object to be validated.
         """
         well_name = las.well.WELL.value
         if not well_name or well_name == " ":
             raise FileValidationError
 
+    def validate_las_file_against_record(self, las: LASFile, record_well_name: str, rec_curve_mnemonics: list[str]):
+        """
+        More extensive validation of a LAS file if it is to be used to write data to an existing record (as
+        opposed to creating new wellbore and well log records).
+        In addition to confirming the Well Name attribute of the LAS file has been populated, the well
+        name and curves are validated against the existing Wellbore and Well Log records respectively.
+
+        :param las LASFile: The LASFile object to be validated.
+        :param record_well_name str: Well name associated with the well log record that is to have data written to it.
+        :param rec_curve_mnemonics list[str]: List of available curves in well log record.
+        """
+        self.validate_las_file(las)
+
+        las_well_name = las.well.WELL.value
+        if not las_well_name == record_well_name:
+            raise FileValidationError("Well name associated with well log record does not match well name in LAS file.")
+
+        las_curve_mnemonics = [curve.mnemonic for curve in las.curves] if las.curves else []
+        if not all(curve in rec_curve_mnemonics for curve in las_curve_mnemonics):
+            raise FileValidationError("Curves available in well log record do not match those in LAS file.")
+
     def load_las_file(self, path: str) -> LASFile:
         """
         Parse a LAS format data into a LASFile object.

src/lasloader/osdu_client.py

@@ -2,7 +2,7 @@ from typing import Union
 import httpx
 import io
 from knack.log import get_logger
-from lasloader.record_mapper import Record, WellLogRecord
+from lasloader.record_mapper import Record, WellLogRecord, WellboreRecord
 from pandas import DataFrame, read_parquet
 
 
@@ -13,6 +13,7 @@ class LasLoaderWebResponseError(Exception):
     """
     Common class for web response errors
     """
+
     def __init__(self, http_code: str, url: str, message: str = ""):
         """
         Create a new instance of a LasLoaderWebResponseError
@@ -49,7 +50,7 @@ class OsduClient:
             "data-partition-id": self._data_partition_id,
         }
 
-    def create_wellbore(self, wellbore_record: Record) -> list[any]:
+    def create_wellbore(self, wellbore_record: Union[Record, WellboreRecord]) -> list[any]:
         """
         Make a post request to OSDU to create a new wellbore record.
 
@@ -57,7 +58,8 @@ class OsduClient:
         :return: the id of the new well bore
         :rtype: str
         """
-        return self._post_data_with_id_response("/api/os-wellbore-ddms/ddms/v3/wellbores", wellbore_record)
+        record = wellbore_record.to_record() if isinstance(wellbore_record, WellboreRecord) else wellbore_record
+        return self._post_data_with_id_response("/api/os-wellbore-ddms/ddms/v3/wellbores", record)
 
     def create_welllog(self, welllog_record: Union[Record, WellLogRecord]) -> list[any]:
         """
@@ -70,16 +72,16 @@ class OsduClient:
         record = welllog_record.to_record() if isinstance(welllog_record, WellLogRecord) else welllog_record
         return self._post_data_with_id_response("/api/os-wellbore-ddms/ddms/v3/welllogs", record)
 
-    def get_wellbore_record(self, wellbore_id: str) -> str:
+    def get_wellbore_record(self, wellbore_id: str) -> WellboreRecord:
         """
         Make a get request to OSDU to retreive an existing wellbore record.
 
         :param str wellbore_id: The wellbore id to be uploaded
         :return: the wellbore record
-        :rtype: str
+        :rtype: WellboreRecord
         """
         get_record_url = f"{self._base_url}/api/os-wellbore-ddms/ddms/v3/wellbores/{wellbore_id}"
-        return self._send_request_json_response("GET", get_record_url, None, None)
+        return WellboreRecord(self._send_request_json_response("GET", get_record_url, None, None))
 
     def get_welllog_record(self, welllog_id: str) -> WellLogRecord:
         """
@@ -87,7 +89,7 @@ class OsduClient:
 
         :param str welllog_id: The welllog id to be uploaded
         :return: the welllog record
-        :rtype: str
+        :rtype: WellLogRecord
         """
         get_record_url = f"{self._base_url}/api/os-wellbore-ddms/ddms/v3/welllogs/{welllog_id}"
         return WellLogRecord(self._send_request_json_response("GET", get_record_url, None, None))

src/lasloader/record_mapper.py

@@ -19,6 +19,34 @@ class Record:
     data: dict[str, any]
 
 
+class WellboreRecord(Record):
+    _raw_data: dict[str, any]
+
+    def __init__(self, data: dict[str, any]) -> None:
+        self.kind = data.get("kind", None)
+        self.data = data.get("data", {})
+        self.acl = data.get("acl", {})
+        self.legal = data.get("legal", {})
+
+        self._raw_data = data
+
+    def get_raw_data(self) -> dict[str, any]:
+        """
+        Get the wellbore data used to construct this object
+        :returns: The wellbore data
+        :rtype: dict[str, any]
+        """
+        return self._raw_data
+
+    def to_record(self) -> Record:
+        """
+        Convert this object to a base records type for upload to OSDU
+        :returns: A new instance of Record populated with data from this object
+        :rtype: Record
+        """
+        return Record(self.kind, self.acl, self.legal, self.data)
+
+
 class WellLogRecord(Record):
     _raw_data: dict[str, any]
 

src/lasloader/well_service.py

 import json
+from typing import Tuple
 import urllib
+
+from lasio.las import LASFile
 from knack.log import get_logger
+from lasloader.file_loader import FileValidationError, LasParser, LocalFileLoader
 from lasio.las import LASFile
 from lasloader.osdu_client import OsduClient, LasLoaderWebResponseError
 from lasloader.record_mapper import LasToRecordMapper, Record, MappingUtilities, MapWellLogToLas, WellBoreRecord
@@ -76,6 +80,32 @@ class WellLogService:
 
         return f"{data_partition_id}:reference-data--LogCurveFamily:{family_id}:"
 
+    def ingest_welllog_data(self, input_path: str, welllog_id: str) -> None:
+        """
+        Write data from a LAS file to an existing well log. First the LAS file is validated against the
+        existing well log record, then the data is extracted and written to the existing well log record.
+
+        :param str input_path: Path and filename of a LAS file containing data to be written to existing well log.
+        :param str welllog_id: ID of well log to be updated.
+        """
+        las_parser = LasParser(LocalFileLoader())
+        las_data = las_parser.load_las_file(input_path)
+
+        logger.info("Beginning LAS file validation.")
+        try:
+            self._validate_welllog_data_ingest_file(las_data, welllog_id)
+        except FileValidationError as e:
+            logger.error(f"Data not ingested - LAS file validation failed: {str(e)}")
+            return
+        except Exception as e:
+            logger.error(f"Data not ingested: {str(e)}")
+            return
+        logger.info("LAS file validation completed.")
+
+        logger.info("Extracting data from LAS file and writing to existing Well Log.")
+        data = las_data.df().reset_index()
+        self._client.add_welllog_data(data, welllog_id)
+
     def download_and_construct_las(self, welllog_id: str, curves: list[str] = None) -> LASFile:
         """
         Download wellbore and log data and convert to a LAS file.
@@ -108,6 +138,34 @@ class WellLogService:
         return mapper.build_las_file()
 
 
+    def _validate_welllog_data_ingest_file(self, las_data: LASFile, welllog_id: str) -> None:
+        """
+        Validate a LAS File against an existing well log record.
+        This should be done before attempting to write data from a LAS file to an existing record.
+
+        :param LASFile las_data: LASFile object to be validated.
+        :param str welllog_id: ID of the well log record to write data to.
+        """
+        logger.info("Retrieving well log record and associated information.")
+        welllog_well_name, welllog_curve_ids = self._get_data_ingest_validation_variables(welllog_id)
+
+        logger.info("Validating LAS file well name and curves against existing well log record.")
+        las_parser = LasParser(LocalFileLoader())
+        las_parser.validate_las_file_against_record(las_data, welllog_well_name, welllog_curve_ids)
+
+    def _get_data_ingest_validation_variables(self, welllog_id: str) -> Tuple[str, list[str]]:
+        """
+        Get the associated well name and curve IDs of the existing well log for validating against LAS file.
+        :param str welllog_id: ID of the well log record to validate a LAS file against.
+        :returns: Well name associated with well log AND curve IDs of the well log.
+        :rtype: Tuple[str, list[str]]
+        """
+        welllog_record = self._client.get_welllog_record(welllog_id)
+        wellbore_record = self._client.get_wellbore_record(welllog_record.data['WellboreID'])
+        welllog_well_name = wellbore_record.data['FacilityName']
+        welllog_curve_ids = welllog_record.get_curveids()
+
+        return welllog_well_name, welllog_curve_ids
 class WellBoreService:
     def __init__(self, client: OsduClient, well_log_service: WellLogService):
         """

src/test/test_file_loader.py

@@ -3,30 +3,47 @@ import lasio
 import pytest
 
 
-def test_LocalFileLoader_throws_when_file_not_exists():
-    fileReader = LocalFileLoader()
-
-    with pytest.raises(FileNotFoundError):
-        fileReader.load("non_existant_file.txt")
-
-
-def test_file_loader_returns_lasio_LASFile_object():
-    # Arrange
-    las_file = "test/test-las-files/15_9-19_SR_CPI.las"
-    loader = LasParser(LocalFileLoader())
-
-    # Act
-    las = loader.load_las_file(las_file)
-
-    # Assert
-    assert type(las) is lasio.LASFile
-
-
-def test_file_loader_raises_FileValidationError_when_well_name_not_populated_in_input_file():
-    # Arrange
-    invalid_las_file = "test/test-las-files/15_9-19_SR_CPI_no_well_name.las"
-    loader = LasParser(LocalFileLoader())
-
-    # Act/Assert
-    with pytest.raises(FileValidationError):
-        loader.load_las_file(invalid_las_file)
+class TestLasParser:
+    def test_file_loader_returns_lasio_LASFile_object(self):
+        # Arrange
+        las_file = "test/test-las-files/15_9-19_SR_CPI.las"
+        loader = LasParser(LocalFileLoader())
+
+        # Act
+        las = loader.load_las_file(las_file)
+
+        # Assert
+        assert type(las) is lasio.LASFile
+
+
+class TestLasFileValidation:
+    def test_LocalFileLoader_throws_when_file_not_exists(self):
+        fileReader = LocalFileLoader()
+
+        with pytest.raises(FileNotFoundError):
+            fileReader.load("non_existant_file.txt")
+
+    def test_FileValidationError_raised_when_well_name_not_populated_in_input_file(self):
+        # Arrange
+        invalid_las_file = "test/test-las-files/15_9-19_SR_CPI_no_well_name.las"
+        loader = LasParser(LocalFileLoader())
+
+        # Act/Assert
+        with pytest.raises(FileValidationError):
+            loader.load_las_file(invalid_las_file)
+
+    @pytest.mark.parametrize(
+        "rec_well_name, rec_curves",
+        [
+            ("different well name", ["DEPTH", "BWV", "DT", "KLOGH", "KLOGV", "PHIF", "SAND_FLAG", "SW", "VSH"]),
+            ("NPD-2105", ["DEPTH", "BWV"])
+        ]
+    )
+    def test_FileValidationError_raised_when_ingest_data_las_does_not_match_welllog_record(self, rec_well_name, rec_curves):
+        # Arrange
+        loader = LasParser(LocalFileLoader())
+        las_data = loader.load_las_file("test/test-las-files/15_9-19_SR_CPI.las")
+
+        # Act/Assert
+        with pytest.raises(FileValidationError):
+            loader.validate_las_file_against_record(las_data, rec_well_name, rec_curves)

src/test/test_osdu_client.py

@@ -184,6 +184,26 @@ class TestOsduClient:
         # Act
         client.get_wellbore_record(wellbore_id)
 
+    def test_get_wellbore_record_raises_on_error(self, respx_mock):
+        # Assemble
+        wellbore_id = "WB-ID123"
+
+        base_url = "http://test.bp.com"
+        url = f"{base_url}/api/os-wellbore-ddms/ddms/v3/wellbores/{wellbore_id}"
+
+        mock_response = httpx.Response(500)
+        respx_mock.get(url).mock(return_value=mock_response)
+
+        client = OsduClient(base_url, self._access_token, self._data_partition_id)
+
+        # Act
+        with pytest.raises(LasLoaderWebResponseError) as ex:
+            client.get_wellbore_record(wellbore_id)
+
+        # Assert
+        assert "500" in str(ex.value)
+        assert url in str(ex.value)
+
     def test_get_welllogs_record(self, respx_mock):
         # Assemble
         welllogs_id = "WL-ID123"
@@ -198,6 +218,26 @@ class TestOsduClient:
         # Act
         client.get_welllog_record(welllogs_id)
 
+    def test_get_welllog_record_raises_on_error(self, respx_mock):
+        # Assemble
+        welllog_id = "WL-ID123"
+
+        base_url = "http://test.bp.com"
+        url = f"{base_url}/api/os-wellbore-ddms/ddms/v3/welllogs/{welllog_id}"
+
+        mock_response = httpx.Response(500)
+        respx_mock.get(url).mock(return_value=mock_response)
+
+        client = OsduClient(base_url, self._access_token, self._data_partition_id)
+
+        # Act
+        with pytest.raises(LasLoaderWebResponseError) as ex:
+            client.get_welllog_record(welllog_id)
+
+        # Assert
+        assert "500" in str(ex.value)
+        assert url in str(ex.value)
+
     def test_get_welllog_data_record(self, respx_mock):
         # Assemble
         welllog_id = "WL-ID123"

src/test/test_record_mapper.py

@@ -3,7 +3,7 @@ import urllib
 from pandas import DataFrame
 from lasio import LASFile
 from unittest.mock import Mock
-from lasloader.record_mapper import Record, AttributeBuilder, LasToRecordMapper, WellLogRecord, MappingUtilities, MapWellLogToLas
+from lasloader.record_mapper import Record, AttributeBuilder, LasToRecordMapper, WellLogRecord, MappingUtilities, MapWellLogToLas, WellboreRecord
 from lasloader.file_loader import LasParser, LocalFileLoader
 from lasloader.configuration import Configuration
 
@@ -311,6 +311,37 @@ class TestLasToRecordMapper:
         assert expected_result.reset_index().equals(result)
 
 
+class TestWellboreRecord:
+    def test_construct_with_empty_dict(self):
+        record = WellboreRecord({})
+
+        assert record.data == {}
+        assert record.acl == {}
+        assert record.kind is None
+        assert record.legal == {}
+        assert record.get_raw_data() == {}
+
+    def test_construct_with_populated_dict(self):
+        wellbore = {
+            'data':
+            {
+                'FacilityName': 'a well name',
+                'NameAliases': [{'AliasName': 'name', 'AliasNameTypeID': 'typeID'}]
+            },
+            'acl': {'some': 'acl'},