Commit 6b45d6b4 authored by Chad Leong's avatar Chad Leong
Browse files

Merge branch '20-allow-use-of-environment-variables' into 'main'

Resolve "Allow use of environment variable for storing access token"

Closes #20

See merge request osdu/platform/data-flow/data-loading/wellbore-ddms-las-loader!26
parents e1ea7cb0 ba8fcb44
Pipeline #77221 passed with stage
in 58 seconds
env/
__pycache__/
.vs/
.vscode/
\ No newline at end of file
.vscode/
lasloader.egg-info/
\ No newline at end of file
......@@ -59,28 +59,24 @@ pipenv run python -m lasloader.lascli fileload -h
pipenv run python -m lasloader.lascli fileload convert -h
```
A token is required to access an OSDU instance. The `-t/--token` option is used to specify the token, this is a string. As tokens are long strings we recommend that the token is stored in an environment variable. For example, on a Windows PowerShell prompt:
A token is required to access an OSDU instance. The `-t/--token` option is used to specify the token, this is a string. As tokens are long strings we recommend that the token is stored in an environment variable.
For example, on a Windows PowerShell prompt:
```
set OSDUTOKEN <token>
pipenv run python -m lasloader.lascli ingest wellbore -t $OSDUTOKEN
```
Or Windows command prompt
Or Windows command prompt:
```
set OSDUTOKEN=<token>
pipenv run python -m lasloader.lascli ingest wellbore -t %OSDUTOKEN%
```
This keeps the command line short. This can also be done for other commonly used options (e.g. `-u/--url`).
Many commands require a bearer token and filepath to a configuration file. If these are set as environment variables (named
'OSDUTOKEN' and 'CONFIGPATH' respectively), it is not necessary to provide them as arguments for commands (the CLI defaults to
use the environment variable).
## Deploy
Before build and deploy ensure that the setup.py and the pipfile.lock dependencies synced, by running:
```
pipenv-setup sync
```
Check that the version number in the `src/lasloader/__init__.py` file is correct.
### Install from repository source
The lasloader module can be installed in edit mode on the command line by running the following command in the root folder of the repository:
......@@ -103,6 +99,13 @@ This installs the package in a virtual environment. The CLI can then be run usin
pipenv run lascli
```
Before build and deploy ensure that the setup.py and the pipfile.lock dependencies synced, by running:
```
pipenv-setup sync
```
Check that the version number in the `src/lasloader/__init__.py` file is correct.
### Creating distribution archives
To create a distribution run:
......@@ -125,6 +128,7 @@ To be determined...
The LASCLI requires a configuration file that has the following JSON structure:
```
{
"base_url": "https://osdu-ship.msft-osdu-test.org",
"data_partition_id": "opendes",
"acl_domain": "contoso.com",
"legal":
......
......@@ -47,20 +47,27 @@ setup(
include_package_data=True,
entry_points={"console_scripts": ["lascli=lasloader.__main__:main"]},
install_requires=[
"anyio==3.3.4; python_full_version >= '3.6.2'",
"argcomplete==1.12.3",
"colorama==0.4.4; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
"certifi==2021.10.8",
"charset-normalizer==2.0.7; python_version >= '3.5'",
"h11==0.12.0; python_version >= '3.6'",
"httpcore==0.13.7; python_version >= '3.6'",
"httpx==0.20.0",
"idna==3.3",
"jmespath==0.10.0; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
"knack==0.8.2",
"knack==0.9.0",
"lasio==0.29",
"numpy==1.21.3; python_version < '3.10' and platform_machine != 'aarch64' and platform_machine != 'arm64'",
"numpy==1.21.4; python_version < '3.10' and platform_machine != 'aarch64' and platform_machine != 'arm64'",
"pandas==1.3.4",
"pyarrow==6.0.0",
"pygments==2.10.0; python_version >= '3.5'",
"python-dateutil==2.8.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
"pytz==2021.3",
"pyyaml==6.0; python_version >= '3.6'",
"rfc3986[idna2008]==1.5.0",
"six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
"sniffio==1.2.0; python_version >= '3.5'",
"tabulate==0.8.9",
"toml==0.10.2",
],
)
......@@ -8,6 +8,7 @@ lasio = "*"
knack = "*"
pandas = "*"
pyarrow = "*"
httpx = "*"
[dev-packages]
flake8 = "*"
......
This diff is collapsed.
{
"base_url": "https://osdu-ship.msft-osdu-test.org",
"data_partition_id": "opendes",
"acl_domain": "contoso.com",
"legal":
{
"legal": {
"legaltags": ["opendes-public-usa-dataset-7643990"],
"otherRelevantDataCountries": ["US"],
"status": "compliant"
"status": "compliant"
},
"data": {
"default": {
"viewers": [ "data.default.viewers@opendes.contoso.com" ],
"owners": [ "data.default.owners@opendes.contoso.com" ]
"viewers": ["data.default.viewers@opendes.contoso.com"],
"owners": ["data.default.owners@opendes.contoso.com"]
}
}
}
}
\ No newline at end of file
from typing import List
from knack.log import get_logger
from lasloader.environment_var_helper import EnvironmentVarHelper
from lasloader.osdu_client import OsduClient
from lasloader.configuration import Configuration
from lasloader.file_loader import LocalFileLoader
......@@ -10,23 +10,24 @@ from lasloader.well_service import WellLogService
logger = get_logger(__name__)
def download_las(url: str,
token: str,
config_path: str,
welllog_id: str,
def download_las(welllog_id: str,
outfile: str,
token: str = None,
config_path: str = None,
curves: List[str] = None):
"""
Retrieve welllog data from an OSDU instance and save to a LAS format file
:param str url: The base URL of the OSDU instance
:param str token: a valid bearer token that is used to authenticate against the OSDU instance
:param str config_path: Path to the las metadata file
:param str welllog_id: The well bore id of the record to retrieve
:param str outfile: The path of the output file
:param str token: a valid bearer token that is used to authenticate against the OSDU instance
:param str config_path: Path to the las metadata file
:param List[str] curves: The curves to retrieve, use None to get all curves
"""
token = EnvironmentVarHelper.get_token(token)
config_path = EnvironmentVarHelper.get_config_path(config_path)
config = Configuration(LocalFileLoader(), config_path)
client = OsduClient(url, token, config.data_partition_id)
client = OsduClient(config.base_url, token, config.data_partition_id)
service = WellLogService(client)
......
from pathlib import Path
from knack.log import get_logger
from lasloader.environment_var_helper import EnvironmentVarHelper
from lasloader.file_loader import LasParser, LocalFileLoader, FileUtilities
from lasloader.record_mapper import LasToRecordMapper
from lasloader.configuration import Configuration
......@@ -12,7 +13,7 @@ logger = get_logger(__name__)
def printlas(input_path):
"""
Print a LAS file header
:param str input_path: Path and filename of a LAS file or folder or LAS files
:param str input_path: Path and filename of a LAS file or folder containing LAS files
"""
las_parser = LasParser(LocalFileLoader())
......@@ -23,13 +24,14 @@ def printlas(input_path):
print(las_data.header)
def convert(input_path: str, config_path: str, wellbore_id: str):
def convert(input_path: str, wellbore_id: str, config_path: str = None):
"""
Convert a LAS file to Wellbore and Well Log and write to json files.
:param str input_path: Path and filename of a LAS file or folder containing LAS files
:param str config_path: Path to the LAS metadata file
:param str wellbore_id: The wellbore id
:param str config_path: Path to the LAS metadata file
"""
config_path = EnvironmentVarHelper.get_config_path(config_path)
las_parser = LasParser(LocalFileLoader())
config = Configuration(LocalFileLoader(), config_path)
......
......@@ -2,6 +2,7 @@ import json
from pathlib import Path
from ntpath import basename
from knack.log import get_logger
from lasloader.environment_var_helper import EnvironmentVarHelper
from lasloader.file_loader import LasParser, LocalFileLoader, FileUtilities
from lasloader.record_mapper import LasToRecordMapper
from lasloader.osdu_client import OsduClient
......@@ -14,21 +15,22 @@ logger = get_logger(__name__)
def wellbore(
input_path: str,
url: str,
token: str,
config_path: str,
token: str = None,
config_path: str = None,
no_recognize: bool = False):
"""
Ingest a LAS file (single) or directory of LAS files (bulk) into OSDU
:param str input_path: Path and filename of a LAS file OR path to directory containing LAS and config files.
:param str url: The base URL of the OSDU instance to which to upload the file.
:param str token: A valid bearer token that is used to authenticate the ingest request.
:param str config_path: Path to the LAS metadata file.
:param bool no_recognize: If true don't attempt to recognize the curves, otherwise recognize the curves
"""
token = EnvironmentVarHelper.get_token(token)
config_path = EnvironmentVarHelper.get_config_path(config_path)
config = Configuration(LocalFileLoader(), config_path)
las_parser = LasParser(LocalFileLoader())
client = OsduClient(url, token, config.data_partition_id)
client = OsduClient(config.base_url, token, config.data_partition_id)
service = WellBoreService(client, WellLogService(client))
failed_ingests = []
......@@ -59,19 +61,20 @@ def wellbore(
def welllog_data(
welllog_id: str,
input_path: str,
url: str,
token: str,
config_path: str):
token: str = None,
config_path: str = None):
"""
Write data from a LAS file to an existing Well Log.
:param str welllog_id: ID of well log to be updated.
:param str input_path: Path and filename of a LAS file containing data to be written to existing well log.
:param str url: The base URL of the OSDU instance to which to find the existing Well Log.
:param str token: A valid bearer token that is used to authenticate the update request.
:param str config_path: Path to the LAS metadata file.
"""
token = EnvironmentVarHelper.get_token(token)
config_path = EnvironmentVarHelper.get_config_path(config_path)
config = Configuration(LocalFileLoader(), config_path)
client = OsduClient(url, token, config.data_partition_id)
client = OsduClient(config.base_url, token, config.data_partition_id)
service = WellLogService(client)
service.ingest_welllog_data(input_path, welllog_id)
......
import json
from typing import List
from knack.log import get_logger
from lasloader.environment_var_helper import EnvironmentVarHelper
from lasloader.osdu_client import OsduClient
from lasloader.configuration import Configuration
from lasloader.file_loader import LocalFileLoader
......@@ -9,39 +10,42 @@ from lasloader.file_loader import LocalFileLoader
logger = get_logger(__name__)
def wellbore(url: str,
token: str,
config_path: str,
wellbore_id: str):
def wellbore(wellbore_id: str,
token: str = None,
config_path: str = None):
"""
Retrieve and print wellbore record from an OSDU instance
:param str url: The base URL of the OSDU instance
:param str wellbore_id: The well bore id of the record to retrieve
:param str token: a valid bearer token that is used to authenticate against the OSDU instance
:param str config_path: Path to the las metadata file
:param str wellbore_id: The well bore id of the record to retrieve
"""
token = EnvironmentVarHelper.get_token(token)
config_path = EnvironmentVarHelper.get_config_path(config_path)
config = Configuration(LocalFileLoader(), config_path)
client = OsduClient(url, token, config.data_partition_id)
client = OsduClient(config.base_url, token, config.data_partition_id)
logger.warning(f"Getting wellbore ID {wellbore_id}")
wellbore = client.get_wellbore_record(wellbore_id)
print(json.dumps(wellbore.get_raw_data(), indent=4, sort_keys=True))
def welllog(url: str,
token: str,
config_path: str,
welllog_id: str,
def welllog(welllog_id: str,
token: str = None,
config_path: str = None,
curves: bool = False):
"""
Retrieve and print welllog record from an OSDU instance
:param str url: The base URL of the OSDU instance
:param str wellbore_id: The well bore id of the record to retrieve
:param str token: a valid bearer token that is used to authenticate against the OSDU instance
:param str config_path: Path to the las metadata file
:param str wellbore_id: The well bore id of the record to retrieve
:param bool curves: Boolean to determine whether to list curves
"""
token = EnvironmentVarHelper.get_token(token)
config_path = EnvironmentVarHelper.get_config_path(config_path)
config = Configuration(LocalFileLoader(), config_path)
client = OsduClient(url, token, config.data_partition_id)
client = OsduClient(config.base_url, token, config.data_partition_id)
logger.warning(f"Getting welllog ID {welllog_id}")
welllog = client.get_welllog_record(welllog_id)
......@@ -52,21 +56,22 @@ def welllog(url: str,
print(json.dumps(welllog.get_raw_data(), indent=4, sort_keys=True))
def welllog_data(url: str,
token: str,
config_path: str,
welllog_id: str,
def welllog_data(welllog_id: str,
token: str = None,
config_path: str = None,
curves: List[str] = None):
"""
Retrieve and print welllog data from an OSDU instance
:param str url: The base URL of the OSDU instance
:param str welllog_id: The welllog id of the record to retrieve
:param str token: a valid bearer token that is used to authenticate against the OSDU instance
:param str config_path: Path to the las metadata file
:param str welllog_id: The welllog id of the record to retrieve
:param List[str] curves: The curves to retrieve, use None to get all curves
"""
token = EnvironmentVarHelper.get_token(token)
config_path = EnvironmentVarHelper.get_config_path(config_path)
config = Configuration(LocalFileLoader(), config_path)
client = OsduClient(url, token, config.data_partition_id)
client = OsduClient(config.base_url, token, config.data_partition_id)
logger.warning(f"Getting data for welllog ID {welllog_id}")
if curves:
......
import json
from knack.log import get_logger
from lasloader.environment_var_helper import EnvironmentVarHelper
from lasloader.osdu_client import OsduClient
from lasloader.configuration import Configuration
from lasloader.file_loader import LocalFileLoader
......@@ -9,20 +10,20 @@ from lasloader.well_service import WellBoreService, WellLogService
logger = get_logger(__name__)
def wellbore_search(url: str,
token: str,
config_path: str,
wellbore_name: str):
def wellbore_search(wellbore_name: str,
token: str = None,
config_path: str = None):
"""
Retrieve and print the ids of wellbores that match the specified name
:param str url: The base URL of the OSDU instance
:param str wellbore_name: The well bore name to search for
:param str token: a valid bearer token that is used to authenticate against the OSDU instance
:param str config_path: Path to the las metadata file
:param str wellbore_name: The well bore name to search for
"""
token = EnvironmentVarHelper.get_token(token)
config_path = EnvironmentVarHelper.get_config_path(config_path)
config = Configuration(LocalFileLoader(), config_path)
client = OsduClient(url, token, config.data_partition_id)
client = OsduClient(config.base_url, token, config.data_partition_id)
service = WellBoreService(client, WellLogService(client))
data = service.search_for_wellbore(wellbore_name)
......
from knack.log import get_logger
from lasloader.environment_var_helper import EnvironmentVarHelper
from lasloader.configuration import Configuration
from lasloader.file_loader import LocalFileLoader
from lasloader.osdu_client import LasLoaderWebResponseError, OsduClient
......@@ -8,19 +9,24 @@ from lasloader.well_service import WellLogService
logger = get_logger(__name__)
def welllog(welllog_id: str, url: str, token: str, config_path: str, curve_families: bool = True):
def welllog(welllog_id: str,
token: str = None,
config_path: str = None,
curve_families: bool = True):
"""
Update an existing well log record.
:param str welllog_id: ID of the well log to be updated.
:param str url: The base URL of the OSDU instance to which the well log was ingested.
:param str token: A valid bearer token that is used to authenticate the update request.
:param str config_path: Path to the LAS metadata file.
:param str curve_families: If true, recognize and update the curve families, otherwise don't.
"""
# This command currently only containes functionality to update the curve families of a well log.
# It is set up in such a way that it should be possible to extend to include other update operations in future if needed.
token = EnvironmentVarHelper.get_token(token)
config_path = EnvironmentVarHelper.get_config_path(config_path)
config = Configuration(LocalFileLoader(), config_path)
client = OsduClient(url, token, config.data_partition_id)
client = OsduClient(config.base_url, token, config.data_partition_id)
service = WellLogService(client)
if curve_families:
......
......@@ -17,6 +17,15 @@ class Configuration:
json_parser = JsonLoader(file_loader)
self._config = json_parser.load(path)
@property
def base_url(self) -> str:
"""
Gets the base url.
return: the base url for API calls
rtype: str
"""
return self._config.get("base_url")
@property
def data_partition_id(self) -> str:
"""
......
import os
from knack.log import get_logger
logger = get_logger(__name__)
class MissingArgumentError(Exception):
"""
Custom error for missing arguments.
"""
def __init__(self, message="Required arguments not provided. See error logs for details."):
super().__init__(message)
class EnvironmentVarHelper:
def get_token(arg_token: str) -> str:
if arg_token is not None:
return arg_token
token = os.environ.get('OSDUTOKEN')
if token is not None:
return token
logger.error("A bearer token was not provided as an environment variable ('OSDUTOKEN') or command argument.")
raise MissingArgumentError
def get_config_path(arg_config_path: str) -> str:
if arg_config_path is not None:
return arg_config_path
config_path = os.environ.get('CONFIGPATH')
if config_path is not None:
return config_path
logger.error("A path to a configuration file was not provided as an environment variable ('CONFIGPATH') or command argument.")
raise MissingArgumentError
......@@ -53,7 +53,6 @@ class LasCommandLoader(CLICommandsLoader):
group.command('wellbore', 'wellbore')
group.command('welllog', 'welllog')
group.command('curves', 'welllog_data')
group.command('wellbore_search', 'wellbore_search')
with CommandGroup(self, 'update', 'lasloader.commands.update#{}') as group:
group.command('welllog', 'welllog')
......@@ -71,7 +70,8 @@ class LasCommandLoader(CLICommandsLoader):
with ArgumentsContext(self, '') as arg_context:
arg_context.argument('config_path', type=str, options_list=('-c', '--config_path'),
help='The path and filename of the configuration file.')
help='''The path and filename of the configuration file.
Defaults to \'CONFIGPATH\' environment variable if not set.''')
# 'fileload'
with ArgumentsContext(self, 'fileload') as arg_context:
......@@ -83,7 +83,7 @@ class LasCommandLoader(CLICommandsLoader):
# 'ingest'
with ArgumentsContext(self, 'ingest') as arg_context:
self._register_input_path_argument(arg_context)
self._register_url_and_token_arguments(arg_context)
self._register_token_argument(arg_context)
with ArgumentsContext(self, 'ingest wellbore') as arg_context:
arg_context.argument('no_recognize', options_list=('--norecognize'), action='store_true',
......@@ -94,7 +94,7 @@ class LasCommandLoader(CLICommandsLoader):
# 'list'
with ArgumentsContext(self, 'list') as arg_context:
self._register_url_and_token_arguments(arg_context)
self._register_token_argument(arg_context)
with ArgumentsContext(self, 'list wellbore') as arg_context:
self._register_wellbore_id_flag(arg_context)
......@@ -110,7 +110,7 @@ class LasCommandLoader(CLICommandsLoader):
# 'update'
with ArgumentsContext(self, 'update') as arg_context:
self._register_url_and_token_arguments(arg_context)
self._register_token_argument(arg_context)
with ArgumentsContext(self, 'update welllog') as arg_context:
self._register_welllog_id_argument(arg_context)
......@@ -119,7 +119,7 @@ class LasCommandLoader(CLICommandsLoader):
# 'Search'
with ArgumentsContext(self, 'search') as arg_context:
self._register_url_and_token_arguments(arg_context)
self._register_token_argument(arg_context)
with ArgumentsContext(self, 'search wellbore') as arg_context:
arg_context.argument('wellbore_name', type=str, options_list=('--name'),
......@@ -127,7 +127,7 @@ class LasCommandLoader(CLICommandsLoader):
# 'download'
with ArgumentsContext(self, 'download') as arg_context:
self._register_url_and_token_arguments(arg_context)
self._register_token_argument(arg_context)
self._register_welllog_id_argument(arg_context)
self._register_curves_argument(arg_context)
arg_context.argument('outfile', type=str, options_list=('--out'),
......@@ -151,11 +151,10 @@ class LasCommandLoader(CLICommandsLoader):
arg_context.argument('welllog_id', type=str, options_list=('--welllog_id'),
help='The welllog id of the record.')
def _register_url_and_token_arguments(self, arg_context):
arg_context.argument('url', type=str, options_list=('-u', '--url'),
help='The base url of the OSDU instance.')
def _register_token_argument(self, arg_context):
arg_context.argument('token', type=str, options_list=('-t', '--token'),
help='A valid bearer token used to authenticate with the OSDU instance.')
help='''A valid bearer token used to authenticate with the OSDU instance.
Defaults to \'OSDUTOKEN\' environment variable if not set.''')
def main():
......
......@@ -262,7 +262,7 @@ class WellBoreService:
def _get_wellbore_by_name(self, wellbore_name: str) -> str:
"""
Search for existing wellbores by name and return appropriate value for
the three cases - 1 result, >1 results, 0 results.
the three cases - 1 result, >1 result, 0 results.
:param str wellbore_name: The wellbore name to search for
:return: The id of the matching wellbore
:rtype: str
......
import os
import pytest
from lasloader.environment_var_helper import EnvironmentVarHelper, MissingArgumentError
class TestGetToken:
def test_returns_original_token_if_not_None(self):
arg_token = "a token"
token = EnvironmentVarHelper.get_token(arg_token)
assert token == arg_token
def test_returns_environment_variable_if_original_token_is_None(self):
# Arrange
arg_token = None
env_var_token = "token stored as env variable"
os.environ['OSDUTOKEN'] = env_var_token
# Act
token = EnvironmentVarHelper.get_token(arg_token)
# Assert
assert token == env_var_token
# Teardown
os.environ.pop('OSDUTOKEN')
def test_raises_MissingArgumentError_when_original_token_is_None_and_environment_var_not_set(self):
arg_token = None
with pytest.raises(MissingArgumentError):
EnvironmentVarHelper.get_token(arg_token)
class TestGetConfigPath:
def test_returns_original_config_path_if_not_None(self):
arg_config_path = "path/to/config/file.json"
config_path = EnvironmentVarHelper.get_config_path(arg_config_path)
assert config_path == arg_config_path
def test_return