Commit 6efc6ddd authored by Niall McDaid's avatar Niall McDaid
Browse files

Restructure commands & add data ingest capability

parent d95c222a
from pathlib import Path
from ntpath import basename
from knack.log import get_logger
from lasloader.file_loader import LasParser, LocalFileLoader, FileUtilities
from lasloader.file_loader import LasParser, LocalFileLoader
from lasloader.record_mapper import LasToRecordMapper
from lasloader.osdu_client import OsduClient
from lasloader.configuration import Configuration
from lasloader.json_writer import JsonToFile
from lasloader.well_service import WellBoreService, WellLogService
logger = get_logger(__name__)
def ingest(
input_path: str,
url: str,
token: str,
config_path: str,
no_recognize: bool = False):
"""
Ingest a las file (single) or directory of las files (bulk) into OSDU
:param str input_path: Path and filename of a las file OR path to directory containing las and config files.
:param str url: The base URL of the OSDU instance to which to upload the file.
:param str token: A valid bearer token that is used to authenticate the ingest request.
:param str config_path: Path to the las metadata file.
:param bool no_recognize: If true don't attempt to recognize the curves, otherwise recognize the curves
"""
config = Configuration(LocalFileLoader(), config_path)
las_parser = LasParser(LocalFileLoader())
client = OsduClient(url, token, config.data_partition_id)
service = WellBoreService(client, WellLogService(client))
failed_ingests = []
for file_path in FileUtilities.get_filenames(Path(input_path), '.las'):
logger.warning(f"Begining ingest of LAS file: {file_path}")
try:
las_data = las_parser.load_las_file(file_path)
las_mapper = LasToRecordMapper(las_data, config)
service.file_ingest(las_mapper, config.data_partition_id, no_recognize)
logger.warning(f"Ingest complete: {basename(file_path)}")
except Exception as e:
failed_ingests.append(f"{basename(file_path)} ({str(e)})")
logger.error(f"Ingest failed: {basename(file_path)} (see summary for details)")
if failed_ingests != []:
logger.error("SUMMARY - files not ingested: " + ", ".join(failed_ingests))
def printlas(input_path):
"""
Print a las file header
:param str input_path: Path and filename of a las file
Print a LAS file header
:param str input_path: Path and filename of a LAS file
"""
las_parser = LasParser(LocalFileLoader())
......@@ -65,9 +23,9 @@ def printlas(input_path):
def convert(input_path: str, config_path: str, wellbore_id: str):
"""
Convert a las file to Well Bore and Well log and write to json files.
:param str input_path: Path and filename of a las file
:param str config_path: Path to the las metadata file
Convert a LAS file to Wellbore and Well Log and write to json files.
:param str input_path: Path and filename of a LAS file
:param str config_path: Path to the LAS metadata file
:param str wellbore_id: The wellbore id
"""
......
from pathlib import Path
from ntpath import basename
from knack.log import get_logger
from lasloader.file_loader import FileValidationError, LasParser, LocalFileLoader, FileUtilities
from lasloader.record_mapper import LasToRecordMapper
from lasloader.osdu_client import LasLoaderWebResponseError, OsduClient
from lasloader.configuration import Configuration
from lasloader.well_service import WellBoreService, WellLogService
logger = get_logger(__name__)
def wellbore(
input_path: str,
url: str,
token: str,
config_path: str,
no_recognize: bool = False):
"""
Ingest a LAS file (single) or directory of LAS files (bulk) into OSDU
:param str input_path: Path and filename of a LAS file OR path to directory containing LAS and config files.
:param str url: The base URL of the OSDU instance to which to upload the file.
:param str token: A valid bearer token that is used to authenticate the ingest request.
:param str config_path: Path to the LAS metadata file.
:param bool no_recognize: If true don't attempt to recognize the curves, otherwise recognize the curves
"""
config = Configuration(LocalFileLoader(), config_path)
las_parser = LasParser(LocalFileLoader())
client = OsduClient(url, token, config.data_partition_id)
service = WellBoreService(client, WellLogService(client))
failed_ingests = []
for file_path in FileUtilities.get_filenames(Path(input_path), '.las'):
logger.warning(f"Begining ingest of LAS file: {file_path}")
try:
las_data = las_parser.load_las_file(file_path)
las_mapper = LasToRecordMapper(las_data, config)
service.file_ingest(las_mapper, config.data_partition_id, no_recognize)
logger.warning(f"Ingest complete: {basename(file_path)}")
except Exception as e:
failed_ingests.append(f"{basename(file_path)} ({str(e)})")
logger.error(f"Ingest failed: {basename(file_path)} (see summary for details)")
if failed_ingests != []:
logger.error("SUMMARY - files not ingested: " + ", ".join(failed_ingests))
def welllog_data(
welllog_id: str,
input_path: str,
url: str,
token: str,
config_path: str):
"""
Write data from a LAS file to an existing Well Log.
:param str welllog_id: ID of Well Log to be updated.
:param str input_path: Path and filename of a LAS file containing data to be written to existing Well Log.
:param str url: The base URL of the OSDU instance to which to find the existing Well Log.
:param str token: A valid bearer token that is used to authenticate the update request.
:param str config_path: Path to the LAS metadata file.
"""
config = Configuration(LocalFileLoader(), config_path)
las_parser = LasParser(LocalFileLoader())
client = OsduClient(url, token, config.data_partition_id)
logger.warning("Retrieving Well Log record and associated information.")
try:
welllog_record = client.get_welllog_record(welllog_id)
wellbore_record = client.get_wellbore_record(welllog_record.data['WellboreID'])
except LasLoaderWebResponseError as e:
logger.error(f"Data not ingested: {str(e)}")
return
welllog_well_name = wellbore_record.data['FacilityName']
welllog_curve_ids = welllog_record.get_curveids()
logger.warning("Validating LAS file against existing Well Log record.")
las_data = las_parser.load_las_file(input_path)
try:
las_parser.validate_data_ingest_las_file(las_data, welllog_well_name, welllog_curve_ids)
except FileValidationError as e:
logger.error(f"Data not ingested - LAS file validation failed: {str(e)}")
return
except Exception as e:
logger.error(f"Data not ingested - unexpected error validating file: {str(e)}")
return
logger.warning("Extracting data from LAS file and writing to existing Well Log.")
data = las_data.df().reset_index()
client.add_welllog_data(data, welllog_id)
logger.warning(f"Ingest complete: Data written from {basename(input_path)} to Well Log ({welllog_id}).")
......@@ -79,12 +79,33 @@ class LasParser:
"""
Validate that the Well Name attribute of the inputted LAS file has been populated.
:param las LASFile The LASFile object to be validated.
:param las LASFile: The LASFile object to be validated.
"""
well_name = las.well.WELL.value
if not well_name or well_name == " ":
raise FileValidationError
def validate_data_ingest_las_file(self, las: LASFile, record_well_name: str, rec_curve_mnemonics: list[str]):
"""
More extensive validation of a LAS file if it is to be used to write data to an existing record (as
opposed to creating new wellbore and well log records).
In addition to confirming the Well Name attribute of the LAS file has been populated, the well
name and curves are validated against the existing Wellbore and Well Log records respectively.
:param las LASFile: The LASFile object to be validated.
:param record_well_name str: Well name associated with the well log record that is to have data written to it.
:param rec_curve_mnemonics list[str]: List of available curves in well log record.
"""
self.validate_las_file(las)
las_well_name = las.well.WELL.value
if not las_well_name == record_well_name:
raise FileValidationError("Well name associated with well log record does not match well name in LAS file.")
las_curve_mnemonics = [curve.mnemonic for curve in las.curves] if las.curves else []
if not all(curve in rec_curve_mnemonics for curve in las_curve_mnemonics):
raise FileValidationError("Curves available in well log record do not match those in LAS file.")
def load_las_file(self, path: str) -> LASFile:
"""
Parse a LAS format data into a LASFile object.
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment