Commit d4507d8e authored by Gregory Harris's avatar Gregory Harris
Browse files

Merge branch 'develop' into 16-las-output-from-bulk-data-ingested-in-wellbore-ddms

# Conflicts:
#	src/lasloader/well_service.py
#	src/test/test_record_mapper.py
#	src/test/test_well_service.py
parents bd11bae2 27a374f3
......@@ -26,6 +26,7 @@ class LasCommandHelp(CLIHelp):
class LasCommandLoader(CLICommandsLoader):
""" Implementation of CLICommandsLoader for the las loader CLI """
def __init__(self, *args, **kwargs):
super(LasCommandLoader, self).__init__(
*args,
......@@ -36,10 +37,13 @@ class LasCommandLoader(CLICommandsLoader):
"""Load all commands"""
with CommandGroup(self, 'fileload', 'lasloader.commands.file_load#{}') as group:
group.command('ingest', 'ingest')
group.command('convert', 'convert')
group.command('print', 'printlas')
with CommandGroup(self, 'ingest', 'lasloader.commands.ingest#{}') as group:
group.command('wellbore', 'wellbore')
group.command('data', 'welllog_data')
with CommandGroup(self, 'list', 'lasloader.commands.list_osdu#{}') as group:
group.command('wellbore', 'wellbore')
group.command('welllog', 'welllog')
......@@ -53,29 +57,37 @@ class LasCommandLoader(CLICommandsLoader):
def load_arguments(self, command):
"""Load arguments for commands"""
with ArgumentsContext(self, '') as arg_context:
arg_context.argument('config_path', type=str, options_list=('-c', '--config_path'),
help='The path and filename of the configuration file.')
with ArgumentsContext(self, 'fileload') as arg_context:
arg_context.argument('input_path', type=str, options_list=('-p', '--path'),
help='Path to a file or folder containing LAS file(s) to upload.')
with ArgumentsContext(self, '') as arg_context:
arg_context.argument('config_path', type=str, options_list=('-c', '--config_path'),
help='The path and filename of the configuration file.')
with ArgumentsContext(self, 'fileload convert') as arg_context:
arg_context.argument('wellbore_id', type=str, options_list=('--wellbore_id'),
help='The wellbore id to use when printing to file.')
with ArgumentsContext(self, 'fileload ingest') as arg_context:
with ArgumentsContext(self, 'ingest') as arg_context:
arg_context.argument('input_path', type=str, options_list=('-p', '--path'),
help='Path to a file or folder containing LAS file(s) to upload.')
with ArgumentsContext(self, 'ingest') as arg_context:
arg_context.argument('url', type=str, options_list=('-u', '--url'),
help='The base url of the OSDU instance.')
with ArgumentsContext(self, 'fileload ingest') as arg_context:
with ArgumentsContext(self, 'ingest') as arg_context:
arg_context.argument('token', type=str, options_list=('-t', '--token'),
help='A valid bearer token used to authenticate with the OSDU instance.')
with ArgumentsContext(self, 'fileload ingest') as arg_context:
with ArgumentsContext(self, 'ingest wellbore') as arg_context:
arg_context.argument('no_recognize', options_list=('--norecognize'), action='store_true',
help='If specified LASCLI won\'t attempt to recognize the curve families.')
with ArgumentsContext(self, 'fileload convert') as arg_context:
arg_context.argument('wellbore_id', type=str, options_list=('--wellbore_id'),
help='The wellbore id to use when printing to file.')
with ArgumentsContext(self, 'ingest data') as arg_context:
arg_context.argument('welllog_id', options_list=('--welllog_id'),
help='The welllog id of the record into which to write the data.')
with ArgumentsContext(self, 'list') as arg_context:
arg_context.argument('url', type=str, options_list=('-u', '--url'),
......
from pathlib import Path
from ntpath import basename
from knack.log import get_logger
from lasloader.file_loader import LasParser, LocalFileLoader, FileUtilities
from lasloader.file_loader import LasParser, LocalFileLoader
from lasloader.record_mapper import LasToRecordMapper
from lasloader.osdu_client import OsduClient
from lasloader.configuration import Configuration
from lasloader.json_writer import JsonToFile
from lasloader.well_service import WellBoreService, WellLogService
logger = get_logger(__name__)
def ingest(
input_path: str,
url: str,
token: str,
config_path: str,
no_recognize: bool = False):
"""
Ingest a las file (single) or directory of las files (bulk) into OSDU
:param str input_path: Path and filename of a las file OR path to directory containing las and config files.
:param str url: The base URL of the OSDU instance to which to upload the file.
:param str token: A valid bearer token that is used to authenticate the ingest request.
:param str config_path: Path to the las metadata file.
:param bool no_recognize: If true don't attempt to recognize the curves, otherwise recognize the curves
"""
config = Configuration(LocalFileLoader(), config_path)
las_parser = LasParser(LocalFileLoader())
client = OsduClient(url, token, config.data_partition_id)
service = WellBoreService(client, WellLogService(client))
failed_ingests = []
for file_path in FileUtilities.get_filenames(Path(input_path), '.las'):
logger.warning(f"Begining ingest of LAS file: {file_path}")
try:
las_data = las_parser.load_las_file(file_path)
las_mapper = LasToRecordMapper(las_data, config)
service.file_ingest(las_mapper, config.data_partition_id, no_recognize)
logger.warning(f"Ingest complete: {basename(file_path)}")
except Exception as e:
failed_ingests.append(f"{basename(file_path)} ({str(e)})")
logger.error(f"Ingest failed: {basename(file_path)} (see summary for details)")
if failed_ingests != []:
logger.error("SUMMARY - files not ingested: " + ", ".join(failed_ingests))
def printlas(input_path):
"""
Print a las file header
:param str input_path: Path and filename of a las file
Print a LAS file header
:param str input_path: Path and filename of a LAS file
"""
las_parser = LasParser(LocalFileLoader())
......@@ -65,9 +23,9 @@ def printlas(input_path):
def convert(input_path: str, config_path: str, wellbore_id: str):
"""
Convert a las file to Well Bore and Well log and write to json files.
:param str input_path: Path and filename of a las file
:param str config_path: Path to the las metadata file
Convert a LAS file to Wellbore and Well Log and write to json files.
:param str input_path: Path and filename of a LAS file
:param str config_path: Path to the LAS metadata file
:param str wellbore_id: The wellbore id
"""
......
from pathlib import Path
from ntpath import basename
from knack.log import get_logger
from lasloader.file_loader import LasParser, LocalFileLoader, FileUtilities
from lasloader.record_mapper import LasToRecordMapper
from lasloader.osdu_client import OsduClient
from lasloader.configuration import Configuration
from lasloader.well_service import WellBoreService, WellLogService
logger = get_logger(__name__)
def wellbore(
input_path: str,
url: str,
token: str,
config_path: str,
no_recognize: bool = False):
"""
Ingest a LAS file (single) or directory of LAS files (bulk) into OSDU
:param str input_path: Path and filename of a LAS file OR path to directory containing LAS and config files.
:param str url: The base URL of the OSDU instance to which to upload the file.
:param str token: A valid bearer token that is used to authenticate the ingest request.
:param str config_path: Path to the LAS metadata file.
:param bool no_recognize: If true don't attempt to recognize the curves, otherwise recognize the curves
"""
config = Configuration(LocalFileLoader(), config_path)
las_parser = LasParser(LocalFileLoader())
client = OsduClient(url, token, config.data_partition_id)
service = WellBoreService(client, WellLogService(client))
failed_ingests = []
for file_path in FileUtilities.get_filenames(Path(input_path), '.las'):
logger.warning(f"Begining ingest of LAS file: {file_path}")
try:
las_data = las_parser.load_las_file(file_path)
las_mapper = LasToRecordMapper(las_data, config)
service.file_ingest(las_mapper, config.data_partition_id, no_recognize)
logger.warning(f"Ingest complete: {basename(file_path)}")
except Exception as e:
failed_ingests.append(f"{basename(file_path)} ({str(e)})")
logger.error(f"Ingest failed: {basename(file_path)} (see summary for details)")
if failed_ingests != []:
logger.error("SUMMARY - files not ingested: " + ", ".join(failed_ingests))
def welllog_data(
welllog_id: str,
input_path: str,
url: str,
token: str,
config_path: str):
"""
Write data from a LAS file to an existing Well Log.
:param str welllog_id: ID of well log to be updated.
:param str input_path: Path and filename of a LAS file containing data to be written to existing well log.
:param str url: The base URL of the OSDU instance to which to find the existing Well Log.
:param str token: A valid bearer token that is used to authenticate the update request.
:param str config_path: Path to the LAS metadata file.
"""
config = Configuration(LocalFileLoader(), config_path)
client = OsduClient(url, token, config.data_partition_id)
service = WellBoreService(client, WellLogService(client))
service.ingest_welllog_data(input_path, welllog_id)
logger.warning(f"Ingest complete: Data written from {basename(input_path)} to Well Log ({welllog_id}).")
import json
from knack.log import get_logger
from lasloader.osdu_client import OsduClient
......
......@@ -79,12 +79,33 @@ class LasParser:
"""
Validate that the Well Name attribute of the inputted LAS file has been populated.
:param las LASFile The LASFile object to be validated.
:param las LASFile: The LASFile object to be validated.
"""
well_name = las.well.WELL.value
if not well_name or well_name == " ":
raise FileValidationError
def validate_las_file_against_record(self, las: LASFile, record_well_name: str, rec_curve_mnemonics: list[str]):
"""
More extensive validation of a LAS file if it is to be used to write data to an existing record (as
opposed to creating new wellbore and well log records).
In addition to confirming the Well Name attribute of the LAS file has been populated, the well
name and curves are validated against the existing Wellbore and Well Log records respectively.
:param las LASFile: The LASFile object to be validated.
:param record_well_name str: Well name associated with the well log record that is to have data written to it.
:param rec_curve_mnemonics list[str]: List of available curves in well log record.
"""
self.validate_las_file(las)
las_well_name = las.well.WELL.value
if not las_well_name == record_well_name:
raise FileValidationError("Well name associated with well log record does not match well name in LAS file.")
las_curve_mnemonics = [curve.mnemonic for curve in las.curves] if las.curves else []
if not all(curve in rec_curve_mnemonics for curve in las_curve_mnemonics):
raise FileValidationError("Curves available in well log record do not match those in LAS file.")
def load_las_file(self, path: str) -> LASFile:
"""
Parse a LAS format data into a LASFile object.
......
......@@ -2,7 +2,7 @@ from typing import Union
import httpx
import io
from knack.log import get_logger
from lasloader.record_mapper import Record, WellLogRecord
from lasloader.record_mapper import Record, WellLogRecord, WellboreRecord
from pandas import DataFrame, read_parquet
......@@ -13,6 +13,7 @@ class LasLoaderWebResponseError(Exception):
"""
Common class for web response errors
"""
def __init__(self, http_code: str, url: str, message: str = ""):
"""
Create a new instance of a LasLoaderWebResponseError
......@@ -49,7 +50,7 @@ class OsduClient:
"data-partition-id": self._data_partition_id,
}
def create_wellbore(self, wellbore_record: Record) -> list[any]:
def create_wellbore(self, wellbore_record: Union[Record, WellboreRecord]) -> list[any]:
"""
Make a post request to OSDU to create a new wellbore record.
......@@ -57,7 +58,8 @@ class OsduClient:
:return: the id of the new well bore
:rtype: str
"""
return self._post_data_with_id_response("/api/os-wellbore-ddms/ddms/v3/wellbores", wellbore_record)
record = wellbore_record.to_record() if isinstance(wellbore_record, WellboreRecord) else wellbore_record
return self._post_data_with_id_response("/api/os-wellbore-ddms/ddms/v3/wellbores", record)
def create_welllog(self, welllog_record: Union[Record, WellLogRecord]) -> list[any]:
"""
......@@ -70,16 +72,16 @@ class OsduClient:
record = welllog_record.to_record() if isinstance(welllog_record, WellLogRecord) else welllog_record
return self._post_data_with_id_response("/api/os-wellbore-ddms/ddms/v3/welllogs", record)
def get_wellbore_record(self, wellbore_id: str) -> str:
def get_wellbore_record(self, wellbore_id: str) -> WellboreRecord:
"""
Make a get request to OSDU to retreive an existing wellbore record.
:param str wellbore_id: The wellbore id to be uploaded
:return: the wellbore record
:rtype: str
:rtype: WellboreRecord
"""
get_record_url = f"{self._base_url}/api/os-wellbore-ddms/ddms/v3/wellbores/{wellbore_id}"
return self._send_request_json_response("GET", get_record_url, None, None)
return WellboreRecord(self._send_request_json_response("GET", get_record_url, None, None))
def get_welllog_record(self, welllog_id: str) -> WellLogRecord:
"""
......@@ -87,7 +89,7 @@ class OsduClient:
:param str welllog_id: The welllog id to be uploaded
:return: the welllog record
:rtype: str
:rtype: WellLogRecord
"""
get_record_url = f"{self._base_url}/api/os-wellbore-ddms/ddms/v3/welllogs/{welllog_id}"
return WellLogRecord(self._send_request_json_response("GET", get_record_url, None, None))
......
......@@ -19,6 +19,34 @@ class Record:
data: dict[str, any]
class WellboreRecord(Record):
_raw_data: dict[str, any]
def __init__(self, data: dict[str, any]) -> None:
self.kind = data.get("kind", None)
self.data = data.get("data", {})
self.acl = data.get("acl", {})
self.legal = data.get("legal", {})
self._raw_data = data
def get_raw_data(self) -> dict[str, any]:
"""
Get the wellbore data used to construct this object
:returns: The wellbore data
:rtype: dict[str, any]
"""
return self._raw_data
def to_record(self) -> Record:
"""
Convert this object to a base records type for upload to OSDU
:returns: A new instance of Record populated with data from this object
:rtype: Record
"""
return Record(self.kind, self.acl, self.legal, self.data)
class WellLogRecord(Record):
_raw_data: dict[str, any]
......
import json
from typing import Tuple
import urllib
from lasio.las import LASFile
from knack.log import get_logger
from lasloader.file_loader import FileValidationError, LasParser, LocalFileLoader
from lasio.las import LASFile
from lasloader.osdu_client import OsduClient, LasLoaderWebResponseError
from lasloader.record_mapper import LasToRecordMapper, Record, MappingUtilities, MapWellLogToLas, WellBoreRecord
......@@ -76,6 +80,32 @@ class WellLogService:
return f"{data_partition_id}:reference-data--LogCurveFamily:{family_id}:"
def ingest_welllog_data(self, input_path: str, welllog_id: str) -> None:
"""
Write data from a LAS file to an existing well log. First the LAS file is validated against the
existing well log record, then the data is extracted and written to the existing well log record.
:param str input_path: Path and filename of a LAS file containing data to be written to existing well log.
:param str welllog_id: ID of well log to be updated.
"""
las_parser = LasParser(LocalFileLoader())
las_data = las_parser.load_las_file(input_path)
logger.info("Beginning LAS file validation.")
try:
self._validate_welllog_data_ingest_file(las_data, welllog_id)
except FileValidationError as e:
logger.error(f"Data not ingested - LAS file validation failed: {str(e)}")
return
except Exception as e:
logger.error(f"Data not ingested: {str(e)}")
return
logger.info("LAS file validation completed.")
logger.info("Extracting data from LAS file and writing to existing Well Log.")
data = las_data.df().reset_index()
self._client.add_welllog_data(data, welllog_id)
def download_and_construct_las(self, welllog_id: str, curves: list[str] = None) -> LASFile:
"""
Download wellbore and log data and convert to a LAS file.
......@@ -108,6 +138,34 @@ class WellLogService:
return mapper.build_las_file()
def _validate_welllog_data_ingest_file(self, las_data: LASFile, welllog_id: str) -> None:
"""
Validate a LAS File against an existing well log record.
This should be done before attempting to write data from a LAS file to an existing record.
:param LASFile las_data: LASFile object to be validated.
:param str welllog_id: ID of the well log record to write data to.
"""
logger.info("Retrieving well log record and associated information.")
welllog_well_name, welllog_curve_ids = self._get_data_ingest_validation_variables(welllog_id)
logger.info("Validating LAS file well name and curves against existing well log record.")
las_parser = LasParser(LocalFileLoader())
las_parser.validate_las_file_against_record(las_data, welllog_well_name, welllog_curve_ids)
def _get_data_ingest_validation_variables(self, welllog_id: str) -> Tuple[str, list[str]]:
"""
Get the associated well name and curve IDs of the existing well log for validating against LAS file.
:param str welllog_id: ID of the well log record to validate a LAS file against.
:returns: Well name associated with well log AND curve IDs of the well log.
:rtype: Tuple[str, list[str]]
"""
welllog_record = self._client.get_welllog_record(welllog_id)
wellbore_record = self._client.get_wellbore_record(welllog_record.data['WellboreID'])
welllog_well_name = wellbore_record.data['FacilityName']
welllog_curve_ids = welllog_record.get_curveids()
return welllog_well_name, welllog_curve_ids
class WellBoreService:
def __init__(self, client: OsduClient, well_log_service: WellLogService):
"""
......
......@@ -3,30 +3,47 @@ import lasio
import pytest
def test_LocalFileLoader_throws_when_file_not_exists():
fileReader = LocalFileLoader()
with pytest.raises(FileNotFoundError):
fileReader.load("non_existant_file.txt")
def test_file_loader_returns_lasio_LASFile_object():
# Arrange
las_file = "test/test-las-files/15_9-19_SR_CPI.las"
loader = LasParser(LocalFileLoader())
# Act
las = loader.load_las_file(las_file)
# Assert
assert type(las) is lasio.LASFile
def test_file_loader_raises_FileValidationError_when_well_name_not_populated_in_input_file():
# Arrange
invalid_las_file = "test/test-las-files/15_9-19_SR_CPI_no_well_name.las"
loader = LasParser(LocalFileLoader())
# Act/Assert
with pytest.raises(FileValidationError):
loader.load_las_file(invalid_las_file)
class TestLasParser:
def test_file_loader_returns_lasio_LASFile_object(self):
# Arrange
las_file = "test/test-las-files/15_9-19_SR_CPI.las"
loader = LasParser(LocalFileLoader())
# Act
las = loader.load_las_file(las_file)
# Assert
assert type(las) is lasio.LASFile
class TestLasFileValidation:
def test_LocalFileLoader_throws_when_file_not_exists(self):
fileReader = LocalFileLoader()
with pytest.raises(FileNotFoundError):
fileReader.load("non_existant_file.txt")
def test_FileValidationError_raised_when_well_name_not_populated_in_input_file(self):
# Arrange
invalid_las_file = "test/test-las-files/15_9-19_SR_CPI_no_well_name.las"
loader = LasParser(LocalFileLoader())
# Act/Assert
with pytest.raises(FileValidationError):
loader.load_las_file(invalid_las_file)
@pytest.mark.parametrize(
"rec_well_name, rec_curves",
[
("different well name", ["DEPTH", "BWV", "DT", "KLOGH", "KLOGV", "PHIF", "SAND_FLAG", "SW", "VSH"]),
("NPD-2105", ["DEPTH", "BWV"])
]
)
def test_FileValidationError_raised_when_ingest_data_las_does_not_match_welllog_record(self, rec_well_name, rec_curves):
# Arrange
loader = LasParser(LocalFileLoader())
las_data = loader.load_las_file("test/test-las-files/15_9-19_SR_CPI.las")
# Act/Assert
with pytest.raises(FileValidationError):
loader.validate_las_file_against_record(las_data, rec_well_name, rec_curves)
......@@ -184,6 +184,26 @@ class TestOsduClient:
# Act
client.get_wellbore_record(wellbore_id)
def test_get_wellbore_record_raises_on_error(self, respx_mock):
# Assemble
wellbore_id = "WB-ID123"
base_url = "http://test.bp.com"
url = f"{base_url}/api/os-wellbore-ddms/ddms/v3/wellbores/{wellbore_id}"
mock_response = httpx.Response(500)
respx_mock.get(url).mock(return_value=mock_response)
client = OsduClient(base_url, self._access_token, self._data_partition_id)
# Act
with pytest.raises(LasLoaderWebResponseError) as ex:
client.get_wellbore_record(wellbore_id)
# Assert
assert "500" in str(ex.value)
assert url in str(ex.value)
def test_get_welllogs_record(self, respx_mock):
# Assemble
welllogs_id = "WL-ID123"
......@@ -198,6 +218,26 @@ class TestOsduClient:
# Act
client.get_welllog_record(welllogs_id)
def test_get_welllog_record_raises_on_error(self, respx_mock):
# Assemble
welllog_id = "WL-ID123"
base_url = "http://test.bp.com"
url = f"{base_url}/api/os-wellbore-ddms/ddms/v3/welllogs/{welllog_id}"
mock_response = httpx.Response(500)
respx_mock.get(url).mock(return_value=mock_response)
client = OsduClient(base_url, self._access_token, self._data_partition_id)
# Act
with pytest.raises(LasLoaderWebResponseError) as ex:
client.get_welllog_record(welllog_id)
# Assert
assert "500" in str(ex.value)
assert url in str(ex.value)
def test_get_welllog_data_record(self, respx_mock):
# Assemble
welllog_id = "WL-ID123"
......
......@@ -3,7 +3,7 @@ import urllib
from pandas import DataFrame
from lasio import LASFile
from unittest.mock import Mock
from lasloader.record_mapper import Record, AttributeBuilder, LasToRecordMapper, WellLogRecord, MappingUtilities, MapWellLogToLas
from lasloader.record_mapper import Record, AttributeBuilder, LasToRecordMapper, WellLogRecord, MappingUtilities, MapWellLogToLas, WellboreRecord
from lasloader.file_loader import LasParser, LocalFileLoader
from lasloader.configuration import Configuration
......@@ -311,6 +311,37 @@ class TestLasToRecordMapper:
assert expected_result.reset_index().equals(result)
class TestWellboreRecord:
def test_construct_with_empty_dict(self):
record = WellboreRecord({})
assert record.data == {}
assert record.acl == {}
assert record.kind is None
assert record.legal == {}
assert record.get_raw_data() == {}
def test_construct_with_populated_dict(self):
wellbore = {
'data':
{
'FacilityName': 'a well name',
'NameAliases': [{'AliasName': 'name', 'AliasNameTypeID': 'typeID'}]
},
'acl': {'some': 'acl'},