Commit 64074d59 authored by Chad Leong's avatar Chad Leong
Browse files

Merge branch '16-las-output-from-bulk-data-ingested-in-wellbore-ddms' into 'main'

Resolve "LAS Output from bulk data ingested in wellbore DDMS"

Closes #16

See merge request osdu/platform/data-flow/data-loading/wellbore-ddms-las-loader!19
parents 26b4156b d26ee38c
Pipeline #72561 passed with stage
in 1 minute and 5 seconds
......@@ -49,6 +49,9 @@ class LasCommandLoader(CLICommandsLoader):
group.command('welllog', 'welllog')
group.command('curves', 'welllog_data')
with CommandGroup(self, 'download', 'lasloader.commands.download#{}') as group:
group.command('welllog', 'download_las')
return OrderedDict(self.command_table)
def load_arguments(self, command):
......@@ -114,6 +117,26 @@ class LasCommandLoader(CLICommandsLoader):
arg_context.argument('curves', type=str, options_list=('--curves'), nargs='*',
help='The list of curves to retrieve (space separated list), returns all curves if not specified.')
with ArgumentsContext(self, 'download') as arg_context:
arg_context.argument('url', type=str, options_list=('-u', '--url'),
help='Url for the OSDU instance.')
with ArgumentsContext(self, 'download') as arg_context:
arg_context.argument('token', type=str, options_list=('-t', '--token'),
help='A valid bearer token used to authenticate with the OSDU instance.')
with ArgumentsContext(self, 'download') as arg_context:
arg_context.argument('welllog_id', type=str, options_list=('--welllog_id'),
help='The welllog id of the record to retrieve.')
with ArgumentsContext(self, 'download') as arg_context:
arg_context.argument('curves', type=str, options_list=('--curves'), nargs='*',
help='The list of curves to retrieve (space separated list), returns all curves if not specified.')
with ArgumentsContext(self, 'download') as arg_context:
arg_context.argument('outfile', type=str, options_list=('--out'),
help='The output file path')
super(LasCommandLoader, self).load_arguments(command)
......
from knack.log import get_logger
from lasloader.osdu_client import OsduClient
from lasloader.configuration import Configuration
from lasloader.file_loader import LocalFileLoader
from lasloader.well_service import WellLogService
logger = get_logger(__name__)
def download_las(url: str,
token: str,
config_path: str,
welllog_id: str,
outfile: str,
curves: list[str] = None):
"""
Retrieve welllog data from an OSDU instance and save to a LAS format file
:param str url: The base URL of the OSDU instance
:param str token: a valid bearer token that is used to authenticate against the OSDU instance
:param str config_path: Path to the las metadata file
:param str welllog_id: The well bore id of the record to retrieve
:param str outfile: The path of the output file
:param list[str] curves: The curves to retrieve, use None to get all curves
"""
config = Configuration(LocalFileLoader(), config_path)
client = OsduClient(url, token, config.data_partition_id)
service = WellLogService(client)
las_file = service.download_and_construct_las(welllog_id, curves)
logger.warning(f"Writing to file {outfile}")
with open(outfile, mode='w') as f:
las_file.write(f, version=2)
......@@ -2,10 +2,15 @@ from dataclasses import dataclass
from typing import Union
import urllib
import lasio
from knack.log import get_logger
from lasio.las import LASFile
from pandas.core.frame import DataFrame
from lasloader.configuration import Configuration
logger = get_logger(__name__)
@dataclass
class Record:
kind: str
......@@ -289,3 +294,67 @@ class LasToRecordMapper:
"""
return self.las.df().reset_index()
class MapWellLogToLas:
def __init__(self, wellbore: Record, welllog: Record, data: DataFrame) -> None:
"""
Construct a new instance of MapWellLogToLas
:param wellbore Record: The wellbore record
:param welllog Record: The well log record
:param data DataFrame: The well log curve data
"""
self._wellbore = wellbore
self._welllog = welllog
self._data = data
def build_las_file(self) -> LASFile:
"""
Build a new instance of LASFile and populate it with data from the welllog
:returns: A new instance of LASFile.
:rtype: LASFile
"""
las = lasio.LASFile()
# Add wellbore info
nameAlias = self._wellbore.data.get("NameAliases")
las.well.UWI.value = nameAlias[0].get("AliasName") if nameAlias is not None and len(nameAlias) > 0 else None
las.well.WELL.value = self._wellbore.data.get("FacilityName")
# Get wellog curve info
wl_curves = self._welllog.data.get("Curves")
# Add curves to las file
for curve in self._data:
# Try to get the curve units from the welllog
if wl_curves is not None:
wl_curve_matches = [c for c in wl_curves if c.get("CurveID") == curve]
wl_curve = wl_curve_matches[0] if wl_curve_matches is not None and len(wl_curve_matches) > 0 else {}
wl_unit = wl_curve.get("CurveUnit")
las_units = MappingUtilities.convert_osdu_unit_to_raw_unit(wl_unit)
else:
las_units = None
las.append_curve(curve, self._data[curve], unit=las_units)
return las
class MappingUtilities:
@staticmethod
def convert_osdu_unit_to_raw_unit(osdu_unit: str) -> str:
"""
Extract and return the unit section from a OSDU qualified unit.
:param str osdu_unit: The osdu qualified unit
:return: The unit part or None if the unit cannot be extracted.
:rtype: str
"""
if osdu_unit is None:
return None
split_units = osdu_unit.split(':')
if (split_units is None or len(split_units) < 3):
logger.warning(f"Could not extract units from {osdu_unit}")
return None
return urllib.parse.unquote(split_units[2])
import json
from typing import Tuple
import urllib
from lasio.las import LASFile
from knack.log import get_logger
from lasio.las import LASFile
from lasloader.file_loader import FileValidationError, LasParser, LocalFileLoader
from lasloader.osdu_client import OsduClient, LasLoaderWebResponseError
from lasloader.record_mapper import LasToRecordMapper, Record
from lasloader.record_mapper import LasToRecordMapper, Record, MappingUtilities, MapWellLogToLas
logger = get_logger(__name__)
......@@ -48,7 +47,7 @@ class WellLogService:
for curve in curves:
mnemonic = curve["Mnemonic"]
osdu_unit = curve["CurveUnit"]
unit = self._convert_osdu_unit_to_raw_unit(osdu_unit)
unit = MappingUtilities.convert_osdu_unit_to_raw_unit(osdu_unit)
curve["LogCurveFamilyID"] = self.recognize_curve_family(mnemonic, unit, data_partition_id) if unit is not None else None
......@@ -105,22 +104,36 @@ class WellLogService:
data = las_data.df().reset_index()
self._client.add_welllog_data(data, welllog_id)
def _convert_osdu_unit_to_raw_unit(self, osdu_unit) -> str:
def download_and_construct_las(self, welllog_id: str, curves: list[str] = None) -> LASFile:
"""
Extract and return the unit section from a OSDU qualified unit.
:param str osdu_unit: The osdu qualified unit
:return: The unit part or None if the unit cannot be extracted.
:rtype: str
Download wellbore and log data and convert to a LAS file.
:param str welllog_id: The welllog_id
:param list[str] curves: The Curves to get, or None for all curves
:return: A new instance of the LAS file object
:rtype: LASFile
"""
if osdu_unit is None:
return None
split_units = osdu_unit.split(':')
if (split_units is None or len(split_units) < 3):
logger.warning(f"Could not extract units from {osdu_unit}")
return None
logger.warning(f"Getting welllog ID {welllog_id}")
welllog = self._client.get_welllog_record(welllog_id)
wellbore_id = welllog.data.get("WellboreID")
if wellbore_id is None:
logger.error("The welllog records contained no wellbore Id, cannot get wellbore")
wellbore = Record(None, {}, {}, {})
else:
logger.warning(f"Getting wellbore ID {wellbore_id}")
wellbore = self._client.get_wellbore_record(wellbore_id)
logger.warning(f"Getting curve data for welllog ID {welllog_id}")
if curves:
logger.warning(f"Curves: {curves}")
welllog_data = self._client.get_welllog_data(welllog_id, curves)
mapper = MapWellLogToLas(wellbore, welllog, welllog_data)
return urllib.parse.unquote(split_units[2])
return mapper.build_las_file()
def _validate_welllog_data_ingest_file(self, las_data: LASFile, welllog_id: str) -> None:
"""
......@@ -176,7 +189,7 @@ class WellBoreService:
logger.warning(f"New wellbore IDs: {ids}")
wellbore_id = self._safe_get_first_record(ids)
logger.info(json.dumps(self._client.get_wellbore_record(wellbore_id), indent=4, sort_keys=True))
logger.info(json.dumps(self._client.get_wellbore_record(wellbore_id).get_raw_data(), indent=4, sort_keys=True))
welllog_record = mapper.map_to_well_log_record(wellbore_id)
......
......@@ -3,7 +3,8 @@ import urllib
from pandas import DataFrame
from lasio import LASFile
from unittest.mock import Mock
from lasloader.record_mapper import Record, AttributeBuilder, LasToRecordMapper, WellLogRecord, WellboreRecord
from lasloader.record_mapper import Record, AttributeBuilder, LasToRecordMapper, WellLogRecord, MappingUtilities
from lasloader.record_mapper import MapWellLogToLas, WellboreRecord
from lasloader.file_loader import LasParser, LocalFileLoader
from lasloader.configuration import Configuration
......@@ -373,3 +374,116 @@ class TestWellLogRecord:
assert record.legal == welllog['legal']
assert record.get_raw_data() == welllog
assert record.get_curveids() == ['abc', 'xyz']
class TestMappingUtilities:
@pytest.mark.parametrize("osdu_unit,expected",
[("opendes:reference-data--UnitOfMeasure:M:", "M"), ("opendes:reference-data--UnitOfMeasure:GAPI:", "GAPI"),
("opendes:reference-data--UnitOfMeasure:US%2FF:", "US/F"),
("opendes:reference-data--UnitOfMeasure:G%2FC3:", "G/C3")])
def test_convert_osdu_unit_to_raw_unit(self, osdu_unit, expected):
# Assemble
# Act
result = MappingUtilities.convert_osdu_unit_to_raw_unit(osdu_unit)
# Assert
assert result == expected
@pytest.mark.parametrize("osdu_unit", ["", None, "opendes:reference-data--UnitOfMeasure"])
def test_convert_osdu_unit_to_raw_unit_bad_data(self, osdu_unit):
# Assemble
# Act
result = MappingUtilities.convert_osdu_unit_to_raw_unit(osdu_unit)
# Assert
assert result is None
class TestMapWellLogToLas:
def test_build_las_file_happy(self):
# Assemble
logdata = {
"Curves": [
{"CurveID": "ABC", "CurveUnit": "opendes:reference-data--UnitOfMeasure:M:"},
{"CurveID": "LMN", "CurveUnit": "opendes:reference-data--UnitOfMeasure:Ft:"},
{"CurveID": "XYZ", "CurveUnit": "opendes:reference-data--UnitOfMeasure:GAPI:"}]
}
welllog = Record("LogKind", {}, {}, logdata)
wellbore = Record("BoreKind", {}, {}, {"FacilityName": "Well name", "NameAliases": [{"AliasName": "Some Name"}]})
data = DataFrame({"ABC": [1, 2, 3], "XYZ": [9, 8, 7], "IJK": [-1, -2, -3]})
subject = MapWellLogToLas(wellbore, welllog, data)
# ACT
result = subject.build_las_file()
# Assert
assert result.well.UWI.value == "Some Name"
assert result.well.WELL.value == "Well name"
assert len(result.curves) == 3
assert result.curves[0].mnemonic == "ABC"
assert result.curves[0].unit == "M"
assert (result.curves[0].data == [1, 2, 3]).all()
assert result.curves[1].mnemonic == "XYZ"
assert result.curves[1].unit == "GAPI"
assert (result.curves[1].data == [9, 8, 7]).all()
assert result.curves[2].mnemonic == "IJK"
assert result.curves[2].unit is None
assert (result.curves[2].data == [-1, -2, -3]).all()
def test_build_las_file_no_curve_data(self):
# Assemble
logdata = {
"Curves": [
{"CurveID": "ABC", "CurveUnit": "opendes:reference-data--UnitOfMeasure:M:"},
{"CurveID": "LMN", "CurveUnit": "opendes:reference-data--UnitOfMeasure:Ft:"},
{"CurveID": "XYZ", "CurveUnit": "opendes:reference-data--UnitOfMeasure:GAPI:"}]
}
welllog = Record("LogKind", {}, {}, logdata)
wellbore = Record("BoreKind", {}, {}, {"FacilityName": "Well name", "NameAliases": [{"AliasName": "Some Name"}]})
data = DataFrame({})
subject = MapWellLogToLas(wellbore, welllog, data)
# ACT
result = subject.build_las_file()
# Assert
assert result.well.UWI.value == "Some Name"
assert result.well.WELL.value == "Well name"
assert len(result.curves) == 0
def test_build_las_file_empty_log_and_bore(self):
# Assemble
welllog = Record("LogKind", {}, {}, {})
wellbore = Record("BoreKind", {}, {}, {})
data = DataFrame({"ABC": [1, 2, 3], "XYZ": [9, 8, 7], "IJK": [-1, -2, -3]})
subject = MapWellLogToLas(wellbore, welllog, data)
# ACT
result = subject.build_las_file()
# Assert
assert result.well.UWI.value is None
assert result.well.WELL.value is None
assert len(result.curves) == 3
assert result.curves[0].mnemonic == "ABC"
assert result.curves[0].unit is None
assert (result.curves[0].data == [1, 2, 3]).all()
assert result.curves[1].mnemonic == "XYZ"
assert result.curves[1].unit is None
assert (result.curves[1].data == [9, 8, 7]).all()
assert result.curves[2].mnemonic == "IJK"
assert result.curves[2].unit is None
assert (result.curves[2].data == [-1, -2, -3]).all()
......@@ -119,32 +119,97 @@ class TestWellLogService:
assert welllog_well_name == "Well-ABC"
assert welllog_curve_ids == ["DEPT", "BWV", "DT"]
@pytest.mark.parametrize("osdu_unit,expected",
[("opendes:reference-data--UnitOfMeasure:M:", "M"), ("opendes:reference-data--UnitOfMeasure:GAPI:", "GAPI"),
("opendes:reference-data--UnitOfMeasure:US%2FF:", "US/F"),
("opendes:reference-data--UnitOfMeasure:G%2FC3:", "G/C3")])
def test_convert_osdu_unit_to_raw_unit(self, osdu_unit, expected):
# Assemble
def test_download_and_construct_las(self):
logdata = {
"WellboreID": "well bore Id",
"Curves": [
{"CurveID": "ABC", "CurveUnit": "opendes:reference-data--UnitOfMeasure:M:"},
{"CurveID": "LMN", "CurveUnit": "opendes:reference-data--UnitOfMeasure:Ft:"},
{"CurveID": "XYZ", "CurveUnit": "opendes:reference-data--UnitOfMeasure:GAPI:"}]
}
welllog = Record("LogKind", {}, {}, logdata)
wellbore = WellboreRecord({"data": {"FacilityName": "Well name", "NameAliases": [{"AliasName": "Some Name"}]}})
data = DataFrame({"ABC": [1, 2, 3], "XYZ": [9, 8, 7], "IJK": [-1, -2, -3]})
client = Mock(spec=OsduClient)
client.get_welllog_record.return_value = welllog
client.get_wellbore_record.return_value = wellbore
client.get_welllog_data.return_value = data
subject = WellLogService(client)
arg_welllog_id = "wellog_id argument"
arg_curves = ["ABC", "XYZ"]
# Act
result = subject._convert_osdu_unit_to_raw_unit(osdu_unit)
result = subject.download_and_construct_las(arg_welllog_id, arg_curves)
# Assert
assert result == expected
client.get_welllog_record.assert_called_once_with(arg_welllog_id)
client.get_wellbore_record.assert_called_once_with("well bore Id")
client.get_welllog_data.assert_called_once_with(arg_welllog_id, arg_curves)
assert result.well.UWI.value == "Some Name"
assert result.well.WELL.value == "Well name"
assert len(result.curves) == 3
assert result.curves[0].mnemonic == "ABC"
assert result.curves[0].unit == "M"
assert (result.curves[0].data == [1, 2, 3]).all()
assert result.curves[1].mnemonic == "XYZ"
assert result.curves[1].unit == "GAPI"
assert (result.curves[1].data == [9, 8, 7]).all()
assert result.curves[2].mnemonic == "IJK"
assert result.curves[2].unit is None
assert (result.curves[2].data == [-1, -2, -3]).all()
def test_download_and_construct_las_no_wellbore(self):
logdata = {
"WellboreID": None,
"Curves": [
{"CurveID": "ABC", "CurveUnit": "opendes:reference-data--UnitOfMeasure:M:"},
{"CurveID": "LMN", "CurveUnit": "opendes:reference-data--UnitOfMeasure:Ft:"},
{"CurveID": "XYZ", "CurveUnit": "opendes:reference-data--UnitOfMeasure:GAPI:"}]
}
welllog = Record("LogKind", {}, {}, logdata)
data = DataFrame({"ABC": [1, 2, 3], "XYZ": [9, 8, 7], "IJK": [-1, -2, -3]})
@pytest.mark.parametrize("osdu_unit", ["", None, "opendes:reference-data--UnitOfMeasure"])
def test_convert_osdu_unit_to_raw_unit_bad_data(self, osdu_unit):
# Assemble
client = Mock(spec=OsduClient)
client.get_welllog_record.return_value = welllog
client.get_welllog_data.return_value = data
subject = WellLogService(client)
arg_welllog_id = "wellog_id argument"
arg_curves = ["ABC", "XYZ"]
# Act
result = subject._convert_osdu_unit_to_raw_unit(osdu_unit)
result = subject.download_and_construct_las(arg_welllog_id, arg_curves)
# Assert
assert result is None
client.get_welllog_record.assert_called_once_with(arg_welllog_id)
client.get_wellbore_record.assert_not_called()
client.get_welllog_data.assert_called_once_with(arg_welllog_id, arg_curves)
assert result.well.UWI.value is None
assert result.well.WELL.value is None
assert len(result.curves) == 3
assert result.curves[0].mnemonic == "ABC"
assert result.curves[0].unit == "M"
assert (result.curves[0].data == [1, 2, 3]).all()
assert result.curves[1].mnemonic == "XYZ"
assert result.curves[1].unit == "GAPI"
assert (result.curves[1].data == [9, 8, 7]).all()
assert result.curves[2].mnemonic == "IJK"
assert result.curves[2].unit is None
assert (result.curves[2].data == [-1, -2, -3]).all()
class TestWellBoreService:
......@@ -155,7 +220,7 @@ class TestWellBoreService:
mock_well_log_service = Mock(spec=WellLogService)
mock_mapper = Mock(spec=LasToRecordMapper)
well_bore_record = Record("well_bore_kind", {}, {}, {})
well_bore_record = WellboreRecord({"Kind": "well_bore_kind", "acl": {}, "legal": {}, "data": {}})
well_log_record = Record("well_log_kind", {}, {}, {})
well_log_record_recognized = Record("well_log_rec_kind", {"blah": "blah"}, {}, {})
well_log_record_returned = WellLogRecord({"kind": "returned_welllog_record"})
......@@ -170,7 +235,7 @@ class TestWellBoreService:
mock_mapper.extract_log_data.return_value = well_log_data
mock_client.create_wellbore.return_value = well_bore_ids
mock_client.get_wellbore_record.return_value = vars(well_log_record)
mock_client.get_wellbore_record.return_value = well_bore_record
mock_client.create_welllog.return_value = well_log_ids
mock_client.get_welllog_record.return_value = well_log_record_returned
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment