Commit 74c95250 authored by Mark Hewitt's avatar Mark Hewitt
Browse files

Merge branch 'feature/storage' into 'main'

Feature/storage

See merge request !11
parents d1873062 e395e1f9
Pipeline #85467 passed with stages
in 2 minutes and 55 seconds
......@@ -28,6 +28,11 @@ For more information, specify the `-h` flag:
Change Log
==========
0.0.29
------
- storage commands
0.0.28
------
......
......@@ -213,9 +213,7 @@ spelling-store-unknown-words=no
# List of note tags to take in consideration, separated by a comma.
notes=FIXME,
XXX,
TODO
XXX
[SIMILARITIES]
......
......@@ -44,7 +44,7 @@ EXIT /B 0
:: define function to launch tests
:test_func
echo testing
nose2 -v --with-coverage --coverage src
nose2 -v --with-coverage --coverage src --coverage-report term --coverage-report html
EXIT /B 0
ENDLOCAL
......
......@@ -12,4 +12,4 @@
""" OSDU command line environment"""
__VERSION__ = "0.0.28"
__VERSION__ = "0.0.29"
......@@ -49,7 +49,6 @@ def query(
"""
connection = CliOsduClient(state.config)
search_client = SearchClient(connection)
print(custom_query)
json_response = search_client.query(kind, id, custom_query, limit)
return json_response
......@@ -31,6 +31,7 @@ from osducli.commands.file._const import FILE_SERVICE_NAME, FILE_STATUS_PATH
from osducli.commands.legal._const import LEGAL_SERVICE_NAME, LEGAL_STATUS_PATH
from osducli.commands.schema._const import SCHEMA_SERVICE_NAME, SCHEMA_STATUS_PATH
from osducli.commands.search._const import SEARCH_SERVICE_NAME, SEARCH_STATUS_PATH
from osducli.commands.storage._const import STORAGE_SERVICE_NAME, STORAGE_STATUS_PATH
from osducli.commands.unit._const import UNIT_SERVICE_NAME, UNIT_STATUS_PATH
from osducli.commands.workflow._const import WORKFLOW_SERVICE_NAME, WORKFLOW_STATUS_PATH
from osducli.config import (
......@@ -76,7 +77,7 @@ def status(state: State): # pylint: disable=unused-argument
(LEGAL_SERVICE_NAME, CONFIG_LEGAL_URL, LEGAL_STATUS_PATH),
(SCHEMA_SERVICE_NAME, CONFIG_SCHEMA_URL, SCHEMA_STATUS_PATH),
(SEARCH_SERVICE_NAME, CONFIG_SEARCH_URL, SEARCH_STATUS_PATH),
("Storage service", CONFIG_STORAGE_URL, "health"),
(STORAGE_SERVICE_NAME, CONFIG_STORAGE_URL, STORAGE_STATUS_PATH),
(UNIT_SERVICE_NAME, CONFIG_UNIT_URL, UNIT_STATUS_PATH),
(WORKFLOW_SERVICE_NAME, CONFIG_WORKFLOW_URL, WORKFLOW_STATUS_PATH),
]
......
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Storage service commands"""
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Storage constants"""
STORAGE_SERVICE_NAME = "Storage service"
STORAGE_STATUS_PATH = "health"
STORAGE_SWAGGER_PATH = "swagger-ui.html"
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Version command"""
import json
import click
from osducli.click_cli import CustomClickCommand, State, command_with_output
from osducli.cliclient import CliOsduClient, handle_cli_exceptions
from osducli.config import CONFIG_STORAGE_URL
from osducli.log import get_logger
from osducli.util.file import get_files_from_path
logger = get_logger(__name__)
# click entry point
@click.command(cls=CustomClickCommand)
@click.option(
"-p",
"--path",
help="Path to a record or records to add.",
type=click.Path(exists=True, file_okay=True, dir_okay=True, readable=True, resolve_path=True),
required=True,
)
@click.option(
"-b",
"--batch",
help="Number of records to add per API call. If not specified records are uploaded as is.",
is_flag=False,
flag_value=200,
type=int,
default=None,
show_default=True,
)
@handle_cli_exceptions
@command_with_output(None)
def _click_command(state: State, path: str, batch: int):
"""Add or update a record"""
return add_records(state, path, batch)
# pylint: disable=too-many-locals
def add_records(state: State, path: str, batch: int) -> dict:
"""Add or update a record
Args:
state (State): Global state
path (str): Path to a record or records to add.
batch (int): Batch size per API call. If None then ingest as is
Returns:
dict: Response from service
"""
if batch is not None:
raise NotImplementedError("--batch is not supported yet for storage add")
connection = CliOsduClient(state.config)
files = get_files_from_path(path)
logger.debug("Files list: %s", files)
# TODO: Check if loaded file is already an array, or a single file
# TODO: Batch uploads
responses = []
for filepath in files:
if filepath.endswith(".json"):
with open(filepath) as file:
storage_object = json.load(file)
logger.info("Processing file %s.", filepath)
if isinstance(storage_object, list):
payload = storage_object
else:
payload = [storage_object]
response_json = None
response_json = connection.cli_put_returning_json(
CONFIG_STORAGE_URL, "records", payload, [200, 201]
)
responses.append(response_json)
return responses
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Storage service versions command"""
import click
from osducli.click_cli import CustomClickCommand, State, global_params
from osducli.cliclient import CliOsduClient, handle_cli_exceptions
from osducli.config import CONFIG_STORAGE_URL
# click entry point
@click.command(cls=CustomClickCommand)
@click.option("-id", "--id", "_id", required=True, help="id to delete")
@handle_cli_exceptions
@global_params
def _click_command(state: State, _id: str):
"""Delete records"""
return delete(state, _id)
def delete(state: State, id: str): # pylint: disable=invalid-name,redefined-builtin
"""Delete records
Args:
state (State): Global state
"""
connection = CliOsduClient(state.config)
url = "records/" + id
connection.cli_delete(CONFIG_STORAGE_URL, url)
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Schema service get command"""
import click
from osducli.click_cli import CustomClickCommand, State, command_with_output
from osducli.cliclient import CliOsduClient, handle_cli_exceptions
from osducli.config import CONFIG_STORAGE_URL
# click entry point
@click.command(cls=CustomClickCommand)
@click.option("-k", "--kind", required=True, help="Get records by kind")
@handle_cli_exceptions
@command_with_output("results")
def _click_command(state: State, kind: str):
"""Get records"""
return get(state, kind)
def get(state: State, kind: str):
"""Get records
Args:
state (State): Global state
kind (str): Kind of the schema
"""
print("NOTE: storage get is still a work in progress")
connection = CliOsduClient(state.config)
# NOTE: there is a difference between records and query endpoints
# url = "records/id"
# url = "query/records?limit=10000&kind=osdu:wks:work-product-component--WellLog:1.0.0"
# TODO: What do we want - a list of id's or the actual records? Perhaps move id list to 'storage list'
if kind is not None:
url = "query/records?kind=" + kind
json = connection.cli_get_returning_json(CONFIG_STORAGE_URL, url)
return json
request_data = {}
# if identifier is not None:
# request_data["query"] = f'id:("{identifier}")'
request_data["records"] = [
"opendes:work-product-component--WellLog:14667082fc7e4dceb17af802904069fb"
]
json = connection.cli_post_returning_json(CONFIG_STORAGE_URL, "query/records", request_data)
return json
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Storage info command"""
import click
from osducli.click_cli import CustomClickCommand, State, command_with_output
from osducli.cliclient import handle_cli_exceptions
from osducli.commands.storage._const import (
STORAGE_SERVICE_NAME,
STORAGE_STATUS_PATH,
STORAGE_SWAGGER_PATH,
)
from osducli.config import CONFIG_STORAGE_URL
from osducli.util.service_info import info
# click entry point
@click.command(cls=CustomClickCommand)
@handle_cli_exceptions
@command_with_output(None)
def _click_command(state: State) -> dict:
"""Information about the service"""
return info(
state, STORAGE_SERVICE_NAME, CONFIG_STORAGE_URL, STORAGE_STATUS_PATH, STORAGE_SWAGGER_PATH
)
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Storage service versions command"""
import click
from osducli.click_cli import CustomClickCommand, State, command_with_output
from osducli.cliclient import CliOsduClient, handle_cli_exceptions
from osducli.config import CONFIG_STORAGE_URL
# click entry point
@click.command(cls=CustomClickCommand)
@click.option("-id", "--id", "_id", required=True, help="id to get versions for")
@handle_cli_exceptions
@command_with_output("versions")
def _click_command(state: State, _id: str):
"""List record versions"""
return versions(state, _id)
def versions(state: State, id: str): # pylint: disable=invalid-name,redefined-builtin
"""List record versions
Args:
state (State): Global state
"""
connection = CliOsduClient(state.config)
url = "records/versions/" + id
json = connection.cli_get_returning_json(CONFIG_STORAGE_URL, url)
return json
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Base class for testing"""
import json
import shutil
import tempfile
import unittest
from os import path
import mock
from click.testing import CliRunner
from osducli.__main__ import cli
class BaseTestCase(unittest.TestCase):
"""Base class for testing"""
def assert_cli_calls_invoked_function(
self, cli_args: list, invoked_function_name: str, *invoked_args
):
"""Test that when called from the cli a specified function is called.
Args:
cli_args (list): list of arguments to pass when calling from the cli
invoked_function_name (str): name of the invoked function
*invoked_args: What we expect passed to invoked_function_name.
"""
with mock.patch(invoked_function_name, return_value=None) as mock_invoked:
runner = CliRunner()
result = runner.invoke(cli, cli_args)
mock_invoked.assert_called_once_with(*invoked_args)
self.assertEqual(result.exit_code, 0)
@staticmethod
def get_local_path(filename: str, calling_path: str) -> dict:
"""Get the full path of the named file from the relevant data folder.
Args:
filename (str): Json file to load from data folder
__file__ (str): Calling file path.
Returns:
str: Path to the given file
"""
data_folder = path.join(path.dirname(path.realpath(calling_path)), "data")
data_path = path.join(data_folder, filename)
return data_path
@staticmethod
def get_local_json_file(filename: str, calling_path: str) -> dict:
"""Get the named file from the relevant data folder.
Args:
filename (str): Json file to load from data folder
__file__ (str): Calling file path.
Returns:
dict: Loaded json file.
"""
data_path = BaseTestCase.get_local_path(filename, calling_path)
return BaseTestCase.json_from_path(data_path)
@staticmethod
def json_from_path(data_path: str) -> dict:
"""Loads the json from the specified path.
Args:
data_path (str): path
Returns:
dict: loaded json
"""
with open(data_path) as file:
return json.load(file)
def create_temp_dir(self, *files) -> str:
"""
Create a temporary directory for testing. The test harness will delete the directory during
tearing down.
Args:
files (str): list of files to copy
Returns:
str: path of temporary directory
"""
temp_dir = tempfile.mkdtemp()
self.addCleanup(lambda: shutil.rmtree(temp_dir, ignore_errors=True))
for file in files:
shutil.copy(file, temp_dir)
return temp_dir
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests for OSDU CLI"""
{
"id": "opendes:reference-data--MaterialType:WTStest",
"kind": "osdu:wks:reference-data--MaterialType:1.0.0",
"acl": {
"owners": [
"data.default.owners@opendes.contoso.com"
],
"viewers": [
"data.default.viewers@opendes.contoso.com"
]
},
"legal": {
"legaltags": [
"opendes-public-usa-dataset-7643990"
],
"otherRelevantDataCountries": [
"US"
]
},
"data": {
"Name": "WTStest",
"Description": "opendes:reference-data--MaterialType:WTStest:",
"ID": "WTStest",
"Code": "WTStest",
"InactiveIndicator": false,
"AttributionAuthority": "Equinor",
"AttributionPublication": "Equinor Automatic Reference Data Dictionary V1.0",
"AttributionRevision": "1.0",
"Source": "Automatically generated by create_missing_id_manuifest.py"
}
}
\ No newline at end of file
[
{
"id": "opendes:reference-data--MaterialType:WTStest",
"kind": "osdu:wks:reference-data--MaterialType:1.0.0",
"acl": {
"owners": [
"data.default.owners@opendes.contoso.com"
],
"viewers": [
"data.default.viewers@opendes.contoso.com"
]
},
"legal": {
"legaltags": [
"opendes-public-usa-dataset-7643990"
],
"otherRelevantDataCountries": [
"US"
]
},
"data": {
"Name": "WTStest",
"Description": "opendes:reference-data--MaterialType:WTStest:",
"ID": "WTStest",
"Code": "WTStest",
"InactiveIndicator": false,
"AttributionAuthority": "Equinor",
"AttributionPublication": "Equinor Automatic Reference Data Dictionary V1.0",
"AttributionRevision": "1.0",
"Source": "Automatically generated by create_missing_id_manuifest.py"
}
}
]
\ No newline at end of file
{
"results": [
"opendes:reference-data--MaterialType:WTS",
"opendes:reference-data--MaterialType:Oil",
"opendes:reference-data--MaterialType:Water",
"opendes:reference-data--MaterialType:Gas",
"opendes:reference-data--MaterialType:WTStest"
]
}
\ No newline at end of file
{
"recordId": "opendes:reference-data--MaterialType:test",
"versions": [
1641374503513365,
1641374544113195,
1641378831231105
]
}
\ No newline at end of file
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Test cases for storage add"""
import mock
from mock import patch
from nose2.tools import params
from osducli.click_cli import State
from osducli.cliclient import CliOsduClient
from osducli.commands.storage.add import add_records