Merge branch 'feature/storage' into 'main'

Feature/storage

See merge request !11

README.rst

@@ -28,6 +28,11 @@ For more information, specify the `-h` flag:
 Change Log
 ==========
 
+0.0.29
+------
+
+- storage commands
+
 0.0.28
 ------
 

pylintrc

@@ -213,9 +213,7 @@ spelling-store-unknown-words=no
 
 # List of note tags to take in consideration, separated by a comma.
 notes=FIXME,
-      XXX,
-      TODO
-
+      XXX
 
 [SIMILARITIES]
 

scripts/verify.bat

@@ -44,7 +44,7 @@ EXIT /B 0
 :: define function to launch tests
 :test_func
 echo testing
-nose2 -v --with-coverage --coverage src
+nose2 -v --with-coverage --coverage src --coverage-report term --coverage-report html
 EXIT /B 0
 
 ENDLOCAL

src/osducli/__init__.py

@@ -12,4 +12,4 @@
 
 """ OSDU command line environment"""
 
-__VERSION__ = "0.0.28"
+__VERSION__ = "0.0.29"

src/osducli/commands/search/query.py

@@ -49,7 +49,6 @@ def query(
     """
     connection = CliOsduClient(state.config)
     search_client = SearchClient(connection)
-    print(custom_query)
     json_response = search_client.query(kind, id, custom_query, limit)
 
     return json_response

src/osducli/commands/status/status.py

@@ -31,6 +31,7 @@ from osducli.commands.file._const import FILE_SERVICE_NAME, FILE_STATUS_PATH
 from osducli.commands.legal._const import LEGAL_SERVICE_NAME, LEGAL_STATUS_PATH
 from osducli.commands.schema._const import SCHEMA_SERVICE_NAME, SCHEMA_STATUS_PATH
 from osducli.commands.search._const import SEARCH_SERVICE_NAME, SEARCH_STATUS_PATH
+from osducli.commands.storage._const import STORAGE_SERVICE_NAME, STORAGE_STATUS_PATH
 from osducli.commands.unit._const import UNIT_SERVICE_NAME, UNIT_STATUS_PATH
 from osducli.commands.workflow._const import WORKFLOW_SERVICE_NAME, WORKFLOW_STATUS_PATH
 from osducli.config import (
@@ -76,7 +77,7 @@ def status(state: State):  # pylint: disable=unused-argument
         (LEGAL_SERVICE_NAME, CONFIG_LEGAL_URL, LEGAL_STATUS_PATH),
         (SCHEMA_SERVICE_NAME, CONFIG_SCHEMA_URL, SCHEMA_STATUS_PATH),
         (SEARCH_SERVICE_NAME, CONFIG_SEARCH_URL, SEARCH_STATUS_PATH),
-        ("Storage service", CONFIG_STORAGE_URL, "health"),
+        (STORAGE_SERVICE_NAME, CONFIG_STORAGE_URL, STORAGE_STATUS_PATH),
         (UNIT_SERVICE_NAME, CONFIG_UNIT_URL, UNIT_STATUS_PATH),
         (WORKFLOW_SERVICE_NAME, CONFIG_WORKFLOW_URL, WORKFLOW_STATUS_PATH),
     ]

src/osducli/commands/storage/__init__.py

+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""Storage service commands"""

src/osducli/commands/storage/_const.py

+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""Storage constants"""
+
+STORAGE_SERVICE_NAME = "Storage service"
+STORAGE_STATUS_PATH = "health"
+STORAGE_SWAGGER_PATH = "swagger-ui.html"

src/osducli/commands/storage/add.py

+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""Version command"""
+
+import json
+
+import click
+
+from osducli.click_cli import CustomClickCommand, State, command_with_output
+from osducli.cliclient import CliOsduClient, handle_cli_exceptions
+from osducli.config import CONFIG_STORAGE_URL
+from osducli.log import get_logger
+from osducli.util.file import get_files_from_path
+
+logger = get_logger(__name__)
+
+
+# click entry point
+@click.command(cls=CustomClickCommand)
+@click.option(
+    "-p",
+    "--path",
+    help="Path to a record or records to add.",
+    type=click.Path(exists=True, file_okay=True, dir_okay=True, readable=True, resolve_path=True),
+    required=True,
+)
+@click.option(
+    "-b",
+    "--batch",
+    help="Number of records to add per API call. If not specified records are uploaded as is.",
+    is_flag=False,
+    flag_value=200,
+    type=int,
+    default=None,
+    show_default=True,
+)
+@handle_cli_exceptions
+@command_with_output(None)
+def _click_command(state: State, path: str, batch: int):
+    """Add or update a record"""
+    return add_records(state, path, batch)
+
+
+# pylint: disable=too-many-locals
+def add_records(state: State, path: str, batch: int) -> dict:
+    """Add or update a record
+
+    Args:
+        state (State): Global state
+        path (str): Path to a record or records to add.
+        batch (int): Batch size per API call. If None then ingest as is
+    Returns:
+        dict: Response from service
+    """
+    if batch is not None:
+        raise NotImplementedError("--batch is not supported yet for storage add")
+
+    connection = CliOsduClient(state.config)
+
+    files = get_files_from_path(path)
+    logger.debug("Files list: %s", files)
+
+    # TODO: Check if loaded file is already an array, or a single file
+    # TODO: Batch uploads
+    responses = []
+    for filepath in files:
+        if filepath.endswith(".json"):
+            with open(filepath) as file:
+                storage_object = json.load(file)
+
+                logger.info("Processing file %s.", filepath)
+                if isinstance(storage_object, list):
+                    payload = storage_object
+                else:
+                    payload = [storage_object]
+
+                response_json = None
+                response_json = connection.cli_put_returning_json(
+                    CONFIG_STORAGE_URL, "records", payload, [200, 201]
+                )
+                responses.append(response_json)
+    return responses

src/osducli/commands/storage/delete.py

+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""Storage service versions command"""
+
+import click
+
+from osducli.click_cli import CustomClickCommand, State, global_params
+from osducli.cliclient import CliOsduClient, handle_cli_exceptions
+from osducli.config import CONFIG_STORAGE_URL
+
+
+# click entry point
+@click.command(cls=CustomClickCommand)
+@click.option("-id", "--id", "_id", required=True, help="id to delete")
+@handle_cli_exceptions
+@global_params
+def _click_command(state: State, _id: str):
+    """Delete records"""
+    return delete(state, _id)
+
+
+def delete(state: State, id: str):  # pylint: disable=invalid-name,redefined-builtin
+    """Delete records
+
+    Args:
+        state (State): Global state
+    """
+    connection = CliOsduClient(state.config)
+    url = "records/" + id
+    connection.cli_delete(CONFIG_STORAGE_URL, url)

src/osducli/commands/storage/get.py

+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""Schema service get command"""
+
+import click
+
+from osducli.click_cli import CustomClickCommand, State, command_with_output
+from osducli.cliclient import CliOsduClient, handle_cli_exceptions
+from osducli.config import CONFIG_STORAGE_URL
+
+
+# click entry point
+@click.command(cls=CustomClickCommand)
+@click.option("-k", "--kind", required=True, help="Get records by kind")
+@handle_cli_exceptions
+@command_with_output("results")
+def _click_command(state: State, kind: str):
+    """Get records"""
+    return get(state, kind)
+
+
+def get(state: State, kind: str):
+    """Get records
+
+    Args:
+        state (State): Global state
+        kind (str): Kind of the schema
+    """
+    print("NOTE: storage get is still a work in progress")
+    connection = CliOsduClient(state.config)
+    # NOTE: there is a difference between records and query endpoints
+    # url = "records/id"
+    # url = "query/records?limit=10000&kind=osdu:wks:work-product-component--WellLog:1.0.0"
+    # TODO: What do we want - a list of id's or the actual records? Perhaps move id list to 'storage list'
+    if kind is not None:
+        url = "query/records?kind=" + kind
+        json = connection.cli_get_returning_json(CONFIG_STORAGE_URL, url)
+        return json
+
+    request_data = {}
+    # if identifier is not None:
+    #     request_data["query"] = f'id:("{identifier}")'
+    request_data["records"] = [
+        "opendes:work-product-component--WellLog:14667082fc7e4dceb17af802904069fb"
+    ]
+    json = connection.cli_post_returning_json(CONFIG_STORAGE_URL, "query/records", request_data)
+    return json

src/osducli/commands/storage/info.py

+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""Storage info command"""
+
+import click
+
+from osducli.click_cli import CustomClickCommand, State, command_with_output
+from osducli.cliclient import handle_cli_exceptions
+from osducli.commands.storage._const import (
+    STORAGE_SERVICE_NAME,
+    STORAGE_STATUS_PATH,
+    STORAGE_SWAGGER_PATH,
+)
+from osducli.config import CONFIG_STORAGE_URL
+from osducli.util.service_info import info
+
+
+# click entry point
+@click.command(cls=CustomClickCommand)
+@handle_cli_exceptions
+@command_with_output(None)
+def _click_command(state: State) -> dict:
+    """Information about the service"""
+    return info(
+        state, STORAGE_SERVICE_NAME, CONFIG_STORAGE_URL, STORAGE_STATUS_PATH, STORAGE_SWAGGER_PATH
+    )

src/osducli/commands/storage/versions.py

+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""Storage service versions command"""
+
+import click
+
+from osducli.click_cli import CustomClickCommand, State, command_with_output
+from osducli.cliclient import CliOsduClient, handle_cli_exceptions
+from osducli.config import CONFIG_STORAGE_URL
+
+
+# click entry point
+@click.command(cls=CustomClickCommand)
+@click.option("-id", "--id", "_id", required=True, help="id to get versions for")
+@handle_cli_exceptions
+@command_with_output("versions")
+def _click_command(state: State, _id: str):
+    """List record versions"""
+    return versions(state, _id)
+
+
+def versions(state: State, id: str):  # pylint: disable=invalid-name,redefined-builtin
+    """List record versions
+
+    Args:
+        state (State): Global state
+    """
+    connection = CliOsduClient(state.config)
+    url = "records/versions/" + id
+    json = connection.cli_get_returning_json(CONFIG_STORAGE_URL, url)
+    return json

tests/base_test_case.py

+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""Base class for testing"""
+
+import json
+import shutil
+import tempfile
+import unittest
+from os import path
+
+import mock
+from click.testing import CliRunner
+
+from osducli.__main__ import cli
+
+
+class BaseTestCase(unittest.TestCase):
+    """Base class for testing"""
+
+    def assert_cli_calls_invoked_function(
+        self, cli_args: list, invoked_function_name: str, *invoked_args
+    ):
+        """Test that when called from the cli a specified function is called.
+
+        Args:
+            cli_args (list): list of arguments to pass when calling from the cli
+            invoked_function_name (str): name of the invoked function
+            *invoked_args: What we expect passed to invoked_function_name.
+        """
+        with mock.patch(invoked_function_name, return_value=None) as mock_invoked:
+            runner = CliRunner()
+            result = runner.invoke(cli, cli_args)
+            mock_invoked.assert_called_once_with(*invoked_args)
+            self.assertEqual(result.exit_code, 0)
+
+    @staticmethod
+    def get_local_path(filename: str, calling_path: str) -> dict:
+        """Get the full path of the named file from the relevant data folder.
+
+        Args:
+            filename (str): Json file to load from data folder
+            __file__ (str): Calling file path.
+
+        Returns:
+            str: Path to the given file
+        """
+        data_folder = path.join(path.dirname(path.realpath(calling_path)), "data")
+        data_path = path.join(data_folder, filename)
+        return data_path
+
+    @staticmethod
+    def get_local_json_file(filename: str, calling_path: str) -> dict:
+        """Get the named file from the relevant data folder.
+
+        Args:
+            filename (str): Json file to load from data folder
+            __file__ (str): Calling file path.
+
+        Returns:
+            dict: Loaded json file.
+        """
+        data_path = BaseTestCase.get_local_path(filename, calling_path)
+        return BaseTestCase.json_from_path(data_path)
+
+    @staticmethod
+    def json_from_path(data_path: str) -> dict:
+        """Loads the json from the specified path.
+
+        Args:
+            data_path (str): path
+
+        Returns:
+            dict: loaded json
+        """
+        with open(data_path) as file:
+            return json.load(file)
+
+    def create_temp_dir(self, *files) -> str:
+        """
+        Create a temporary directory for testing. The test harness will delete the directory during
+        tearing down.
+
+        Args:
+            files (str): list of files to copy
+
+        Returns:
+            str: path of temporary directory
+        """
+        temp_dir = tempfile.mkdtemp()
+        self.addCleanup(lambda: shutil.rmtree(temp_dir, ignore_errors=True))
+
+        for file in files:
+            shutil.copy(file, temp_dir)
+
+        return temp_dir

tests/commands/storage/__init__.py

+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""Tests for OSDU CLI"""

tests/commands/storage/data/add1record-noarray.json

+{
+    "id": "opendes:reference-data--MaterialType:WTStest",
+    "kind": "osdu:wks:reference-data--MaterialType:1.0.0",
+    "acl": {
+    "owners": [
+    "data.default.owners@opendes.contoso.com"
+    ],
+    "viewers": [
+    "data.default.viewers@opendes.contoso.com"
+    ]
+    },
+    "legal": {
+    "legaltags": [
+    "opendes-public-usa-dataset-7643990"
+    ],
+    "otherRelevantDataCountries": [
+    "US"
+    ]
+    },
+    "data": {
+    "Name": "WTStest",
+    "Description": "opendes:reference-data--MaterialType:WTStest:",
+    "ID": "WTStest",
+    "Code": "WTStest",
+    "InactiveIndicator": false,
+    "AttributionAuthority": "Equinor",
+    "AttributionPublication": "Equinor Automatic Reference Data Dictionary V1.0",
+    "AttributionRevision": "1.0",
+    "Source": "Automatically generated by create_missing_id_manuifest.py"
+    }
+}
\ No newline at end of file

tests/commands/storage/data/add1record.json

+[
+  {
+    "id": "opendes:reference-data--MaterialType:WTStest",
+    "kind": "osdu:wks:reference-data--MaterialType:1.0.0",
+    "acl": {
+    "owners": [
+      "data.default.owners@opendes.contoso.com"
+    ],
+    "viewers": [
+      "data.default.viewers@opendes.contoso.com"
+    ]
+    },
+    "legal": {
+    "legaltags": [
+      "opendes-public-usa-dataset-7643990"
+    ],
+    "otherRelevantDataCountries": [
+      "US"
+    ]
+    },
+    "data": {
+    "Name": "WTStest",
+    "Description": "opendes:reference-data--MaterialType:WTStest:",
+    "ID": "WTStest",
+    "Code": "WTStest",
+    "InactiveIndicator": false,
+    "AttributionAuthority": "Equinor",
+    "AttributionPublication": "Equinor Automatic Reference Data Dictionary V1.0",
+    "AttributionRevision": "1.0",
+    "Source": "Automatically generated by create_missing_id_manuifest.py"
+    }
+  }
+]
\ No newline at end of file

tests/commands/storage/data/get-response.json

+{
+  "results": [
+    "opendes:reference-data--MaterialType:WTS",
+    "opendes:reference-data--MaterialType:Oil",
+    "opendes:reference-data--MaterialType:Water",
+    "opendes:reference-data--MaterialType:Gas",
+    "opendes:reference-data--MaterialType:WTStest"
+  ]
+}
\ No newline at end of file

tests/commands/storage/data/versions-response.json

+{
+    "recordId": "opendes:reference-data--MaterialType:test",
+    "versions": [
+      1641374503513365,
+      1641374544113195,
+      1641378831231105
+    ]
+  }
\ No newline at end of file

tests/commands/storage/test_add.py

+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+
+"""Test cases for storage add"""
+
+import mock
+from mock import patch
+from nose2.tools import params
+
+from osducli.click_cli import State
+from osducli.cliclient import CliOsduClient
+from osducli.commands.storage.add import add_records
+