Commit a44f2e17 authored by Chad Leong's avatar Chad Leong
Browse files

Merge branch '24-update-welllog-curve-families' into 'main'

Functionality to update existing well log records with curve families

Closes #24

See merge request osdu/platform/data-flow/data-loading/wellbore-ddms-las-loader!22
parents 6a7fa42a fd368aa3
Pipeline #73542 passed with stage
in 1 minute and 16 seconds
......@@ -52,7 +52,7 @@ To invoke the CLI using the virtual environment and from within the `src` folder
```
pipenv run python -m lascli <group> <command> <options>
```
Available groups are `fileload`, `ingest`, `list` and `download`. Help can be obtained with the `-h` option.
Available groups are `fileload`, `ingest`, `list`, `update` and `download`. Help can be obtained with the `-h` option.
```
pipenv run python -m lascli -h
pipenv run python -m lascli fileload -h
......
......@@ -49,6 +49,9 @@ class LasCommandLoader(CLICommandsLoader):
group.command('welllog', 'welllog')
group.command('curves', 'welllog_data')
with CommandGroup(self, 'update', 'lasloader.commands.update#{}') as group:
group.command('welllog', 'welllog')
with CommandGroup(self, 'download', 'lasloader.commands.download#{}') as group:
group.command('welllog', 'download_las')
......@@ -61,6 +64,7 @@ class LasCommandLoader(CLICommandsLoader):
arg_context.argument('config_path', type=str, options_list=('-c', '--config_path'),
help='The path and filename of the configuration file.')
# 'fileload'
with ArgumentsContext(self, 'fileload') as arg_context:
arg_context.argument('input_path', type=str, options_list=('-p', '--path'),
help='Path to a file or folder containing LAS file(s) to upload.')
......@@ -69,6 +73,7 @@ class LasCommandLoader(CLICommandsLoader):
arg_context.argument('wellbore_id', type=str, options_list=('--wellbore_id'),
help='The wellbore id to use when printing to file.')
# 'ingest'
with ArgumentsContext(self, 'ingest') as arg_context:
arg_context.argument('input_path', type=str, options_list=('-p', '--path'),
help='Path to a file or folder containing LAS file(s) to upload.')
......@@ -89,6 +94,7 @@ class LasCommandLoader(CLICommandsLoader):
arg_context.argument('welllog_id', options_list=('--welllog_id'),
help='The welllog id of the record into which to write the data.')
# 'list'
with ArgumentsContext(self, 'list') as arg_context:
arg_context.argument('url', type=str, options_list=('-u', '--url'),
help='Url for the OSDU instance.')
......@@ -117,6 +123,24 @@ class LasCommandLoader(CLICommandsLoader):
arg_context.argument('curves', type=str, options_list=('--curves'), nargs='*',
help='The list of curves to retrieve (space separated list), returns all curves if not specified.')
# 'update'
with ArgumentsContext(self, 'update') as arg_context:
arg_context.argument('url', type=str, options_list=('-u', '--url'),
help='Url for the OSDU instance.')
with ArgumentsContext(self, 'update') as arg_context:
arg_context.argument('token', type=str, options_list=('-t', '--token'),
help='A valid bearer token used to authenticate with the OSDU instance.')
with ArgumentsContext(self, 'update welllog') as arg_context:
arg_context.argument('welllog_id', type=str, options_list=('--welllog_id'),
help='The welllog ID of the record to updated.')
with ArgumentsContext(self, 'update welllog') as arg_context:
arg_context.argument('recognize', options_list=('--recognize'), action='store_true',
help='If specified or set to True LASCLI will attempt to recognize and update the curve families.')
# 'download'
with ArgumentsContext(self, 'download') as arg_context:
arg_context.argument('url', type=str, options_list=('-u', '--url'),
help='Url for the OSDU instance.')
......
from knack.log import get_logger
from lasloader.configuration import Configuration
from lasloader.file_loader import LocalFileLoader
from lasloader.osdu_client import LasLoaderWebResponseError, OsduClient
from lasloader.well_service import WellLogService
logger = get_logger(__name__)
def welllog(welllog_id: str, url: str, token: str, config_path: str, curve_families: bool = True):
"""
Update an existing well log record.
:param str welllog_id: ID of the well log to be updated.
:param str url: The base URL of the OSDU instance to which the well log was ingested.
:param str token: A valid bearer token that is used to authenticate the update request.
:param str config_path: Path to the LAS metadata file.
:param str curve_families: If true, recognize and update the curve families, otherwise don't.
"""
# This command currently only containes functionality to update the curve families of a well log.
# It is set up in such a way that it should be possible to extend to include other update operations in future if needed.
config = Configuration(LocalFileLoader(), config_path)
client = OsduClient(url, token, config.data_partition_id)
service = WellLogService(client)
if curve_families:
logger.info("Recognizing curve families and updating well log record.")
try:
service.update_log_family(welllog_id, config.data_partition_id)
except LasLoaderWebResponseError as ex:
logger.error(f"Error - record not updated: {str(ex)}")
return
logger.warning(f"Curve families recognized and updated for well log: {welllog_id}")
else:
logger.warning("No updates made - set command option 'curve_families' to True to update curve families.")
......@@ -61,12 +61,14 @@ class OsduClient:
record = wellbore_record.to_record() if isinstance(wellbore_record, WellboreRecord) else wellbore_record
return self._post_data_with_id_response("/api/os-wellbore-ddms/ddms/v3/wellbores", record)
def create_welllog(self, welllog_record: Union[Record, WellLogRecord]) -> list[any]:
def post_welllog(self, welllog_record: Union[Record, WellLogRecord]) -> list[any]:
"""
Make a post request to OSDU to create a new well log record.
Make a post request to OSDU to create a new or update an existing well log record.
If the `welllog_record` contains an `id` attribute, the well log with that ID shall be updated.
If the `welllog_record` does not contain an `id` attribute, a new record will be created.
:param Record welllog_record: The welllog record to be uploaded
:return: the id of the new well log
:param Union[Record, WellLogRecord] welllog_record: The well log record to be uploaded.
:return: The ID of the created/updated well log
:rtype: list[any]
"""
record = welllog_record.to_record() if isinstance(welllog_record, WellLogRecord) else welllog_record
......
from dataclasses import dataclass
from typing import Union
from typing import Union, Optional
import urllib
import lasio
from knack.log import get_logger
......@@ -17,6 +17,7 @@ class Record:
acl: dict[str, list[str]]
legal: dict[str, Union[list[str], str]]
data: dict[str, any]
id: Optional[str] = None
class WellboreRecord(Record):
......@@ -27,6 +28,7 @@ class WellboreRecord(Record):
self.data = data.get("data", {})
self.acl = data.get("acl", {})
self.legal = data.get("legal", {})
self.id = data.get("id", None)
self._raw_data = data
......@@ -44,7 +46,7 @@ class WellboreRecord(Record):
:returns: A new instance of Record populated with data from this object
:rtype: Record
"""
return Record(self.kind, self.acl, self.legal, self.data)
return Record(self.kind, self.acl, self.legal, self.data, self.id)
class WellLogRecord(Record):
......@@ -55,6 +57,7 @@ class WellLogRecord(Record):
self.data = data.get("data", {})
self.acl = data.get("acl", {})
self.legal = data.get("legal", {})
self.id = data.get("id", None)
self._raw_data = data
......@@ -81,7 +84,7 @@ class WellLogRecord(Record):
:returns: A new instance of Record populated with data from this object
:rtype: Record
"""
return Record(self.kind, self.acl, self.legal, self.data)
return Record(self.kind, self.acl, self.legal, self.data, self.id)
class AttributeBuilder:
......
......@@ -78,6 +78,27 @@ class WellLogService:
return f"{data_partition_id}:reference-data--LogCurveFamily:{family_id}:"
def update_log_family(self, welllog_id: str, data_partition_id: str) -> None:
"""
Update the recognized curve families for an existing well log record.
:param str welllog_id: ID of the well log record to be updated.
:param str data_partition_id: The data partition ID.
"""
logger.info("Retrieving existing well log record.")
welllog_record = self._client.get_welllog_record(welllog_id)
logger.info("Beginning recognition of curve families.")
enriched_welllog_record = self.recognize_log_family(welllog_record.to_record(), data_partition_id)
logger.info("Updating existing well log record with recognized curve families.")
welllog_ids = self._client.post_welllog(enriched_welllog_record)
# Print out updated record in logging for user to see.
wellbore_service = WellBoreService(self._client, self)
welllog_id = wellbore_service._safe_get_first_record(welllog_ids)
welllog_record = self._client.get_welllog_record(welllog_id)
logger.info(json.dumps(welllog_record.get_raw_data(), indent=4, sort_keys=True))
def ingest_welllog_data(self, input_path: str, welllog_id: str) -> None:
"""
Write data from a LAS file to an existing well log. First the LAS file is validated against the
......@@ -200,7 +221,7 @@ class WellBoreService:
logger.warning("Recognizing the curve families")
enriched_welllog_record = self._well_log_service.recognize_log_family(welllog_record, data_partition_id)
welllog_ids = self._client.create_welllog(enriched_welllog_record)
welllog_ids = self._client.post_welllog(enriched_welllog_record)
logger.warning(f"New welllog IDs: {welllog_ids}")
welllog_id = self._safe_get_first_record(welllog_ids)
......
......@@ -105,7 +105,7 @@ class TestOsduClient:
client = OsduClient(base_url, self._access_token, self._data_partition_id)
# Act
response = client.create_welllog(record)
response = client.post_welllog(record)
# Assert
assert response == self._expected_ids
......@@ -123,7 +123,7 @@ class TestOsduClient:
# Act
with pytest.raises(LasLoaderWebResponseError) as ex:
client.create_welllog(record)
client.post_welllog(record)
# Assert
assert "500" in str(ex.value)
......
......@@ -320,6 +320,7 @@ class TestWellboreRecord:
assert record.acl == {}
assert record.kind is None
assert record.legal == {}
assert record.id is None
assert record.get_raw_data() == {}
def test_construct_with_populated_dict(self):
......@@ -332,6 +333,7 @@ class TestWellboreRecord:
'acl': {'some': 'acl'},
'kind': 'A Kind',
'legal': {'some': 'legal'},
'id': 'WBID-123'
}
record = WellLogRecord(wellbore)
......@@ -340,6 +342,7 @@ class TestWellboreRecord:
assert record.acl == wellbore['acl']
assert record.kind == wellbore['kind']
assert record.legal == wellbore['legal']
assert record.id == wellbore['id']
assert record.get_raw_data() == wellbore
......@@ -351,6 +354,7 @@ class TestWellLogRecord:
assert record.acl == {}
assert record.kind is None
assert record.legal == {}
assert record.id is None
assert record.get_raw_data() == {}
assert record.get_curveids() == []
......@@ -364,6 +368,7 @@ class TestWellLogRecord:
'acl': {'some': 'acl'},
'kind': 'A Kind',
'legal': {'some': 'legal'},
'id': 'WLID-123'
}
record = WellLogRecord(welllog)
......@@ -372,6 +377,7 @@ class TestWellLogRecord:
assert record.acl == welllog['acl']
assert record.kind == welllog['kind']
assert record.legal == welllog['legal']
assert record.id == welllog['id']
assert record.get_raw_data() == welllog
assert record.get_curveids() == ['abc', 'xyz']
......@@ -407,7 +413,7 @@ class TestMapWellLogToLas:
{"CurveID": "ABC", "CurveUnit": "opendes:reference-data--UnitOfMeasure:M:"},
{"CurveID": "LMN", "CurveUnit": "opendes:reference-data--UnitOfMeasure:Ft:"},
{"CurveID": "XYZ", "CurveUnit": "opendes:reference-data--UnitOfMeasure:GAPI:"}]
}
}
welllog = Record("LogKind", {}, {}, logdata)
wellbore = Record("BoreKind", {}, {}, {"FacilityName": "Well name", "NameAliases": [{"AliasName": "Some Name"}]})
......@@ -442,7 +448,7 @@ class TestMapWellLogToLas:
{"CurveID": "ABC", "CurveUnit": "opendes:reference-data--UnitOfMeasure:M:"},
{"CurveID": "LMN", "CurveUnit": "opendes:reference-data--UnitOfMeasure:Ft:"},
{"CurveID": "XYZ", "CurveUnit": "opendes:reference-data--UnitOfMeasure:GAPI:"}]
}
}
welllog = Record("LogKind", {}, {}, logdata)
wellbore = Record("BoreKind", {}, {}, {"FacilityName": "Well name", "NameAliases": [{"AliasName": "Some Name"}]})
......
......@@ -89,6 +89,59 @@ class TestWellLogService:
client.post_log_recognition.assert_called_once_with(mnemonic, unit)
assert result == expected
def test_update_log_family(self):
# Arrange
welllog_id = "WL-ID-123"
data_partition_id = "dp-123"
curves = [
{"Mnemonic": "DEPT", "CurveUnit": "::Unit1"},
{"Mnemonic": "BWV", "CurveUnit": "::Unit2"},
{"Mnemonic": "DT", "CurveUnit": "::Unit3"}
]
welllog_record = WellLogRecord({
"kind": "a kind",
"data": {
"ReferenceCurveID": "DEPT",
"Curves": curves,
"WellboreID": "WB-ID-123",
},
"id": welllog_id
})
family_id_prefix = "dp-123:reference-data--LogCurveFamily"
enriched_curves = [
{"Mnemonic": "DEPT", "CurveUnit": "::Unit1", "LogCurveFamilyID": f"{family_id_prefix}:Fam-ID-1:"},
{"Mnemonic": "BWV", "CurveUnit": "::Unit2", "LogCurveFamilyID": f"{family_id_prefix}:Fam-ID-1:"},
{"Mnemonic": "DT", "CurveUnit": "::Unit3", "LogCurveFamilyID": f"{family_id_prefix}:Fam-ID-1:"}
]
enriched_welllog_record = WellLogRecord({
"kind": "a kind",
"data": {
"ReferenceCurveID": "DEPT",
"Curves": enriched_curves,
"WellboreID": "WB-ID-123",
},
"id": welllog_id
})
client = Mock(spec=OsduClient)
client.get_welllog_record.return_value = welllog_record
client.post_log_recognition.return_value = {"family": "Fam ID 1"}
client.post_welllog.return_value = [welllog_id]
subject = WellLogService(client)
# Act
subject.update_log_family(welllog_id, data_partition_id)
# Assert
assert client.get_welllog_record.call_count == 2
client.get_welllog_record.assert_called_with(welllog_id)
assert client.post_log_recognition.call_count == 3
client.post_log_recognition.assert_called_with("DT", "Unit3")
client.post_welllog.assert_called_once_with(enriched_welllog_record.to_record())
def test_get_data_ingest_validation_variables(self):
# Assemble
client = Mock(spec=OsduClient)
......@@ -126,7 +179,7 @@ class TestWellLogService:
{"CurveID": "ABC", "CurveUnit": "opendes:reference-data--UnitOfMeasure:M:"},
{"CurveID": "LMN", "CurveUnit": "opendes:reference-data--UnitOfMeasure:Ft:"},
{"CurveID": "XYZ", "CurveUnit": "opendes:reference-data--UnitOfMeasure:GAPI:"}]
}
}
welllog = Record("LogKind", {}, {}, logdata)
wellbore = WellboreRecord({"data": {"FacilityName": "Well name", "NameAliases": [{"AliasName": "Some Name"}]}})
......@@ -173,7 +226,7 @@ class TestWellLogService:
{"CurveID": "ABC", "CurveUnit": "opendes:reference-data--UnitOfMeasure:M:"},
{"CurveID": "LMN", "CurveUnit": "opendes:reference-data--UnitOfMeasure:Ft:"},
{"CurveID": "XYZ", "CurveUnit": "opendes:reference-data--UnitOfMeasure:GAPI:"}]
}
}
welllog = Record("LogKind", {}, {}, logdata)
data = DataFrame({"ABC": [1, 2, 3], "XYZ": [9, 8, 7], "IJK": [-1, -2, -3]})
......@@ -236,7 +289,7 @@ class TestWellBoreService:
mock_client.create_wellbore.return_value = well_bore_ids
mock_client.get_wellbore_record.return_value = well_bore_record
mock_client.create_welllog.return_value = well_log_ids
mock_client.post_welllog.return_value = well_log_ids
mock_client.get_welllog_record.return_value = well_log_record_returned
mock_well_log_service.recognize_log_family.return_value = well_log_record_recognized
......@@ -254,10 +307,10 @@ class TestWellBoreService:
if no_recognize:
mock_well_log_service.recognize_log_family.assert_not_called()
mock_client.create_welllog.called_once_with(well_log_record)
mock_client.post_welllog.called_once_with(well_log_record)
else:
mock_well_log_service.recognize_log_family.called_once_with(well_log_record, data_partition_id)
mock_client.create_welllog.called_once_with(well_log_record_recognized)
mock_client.post_welllog.called_once_with(well_log_record_recognized)
mock_client.get_welllog_record.called_once_with(well_log_ids[0])
mock_mapper.extract_log_data.called_once()
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment