Merge branch 'feature/workflow-commands' into 'main'

API documentation pages & Workflow commands

See merge request !12

README.rst

@@ -28,6 +28,11 @@ For more information, specify the `-h` flag:
 Change Log
 ==========
 
+0.0.30
+
+- API documentation pages are shown in info commands
+- workflow get, runs and status commands
+
 0.0.29
 ------
 

src/osducli/__init__.py

@@ -12,4 +12,4 @@
 
 """ OSDU command line environment"""
 
-__VERSION__ = "0.0.29"
+__VERSION__ = "0.0.30"

src/osducli/commands/dataload/ingest.py

@@ -20,8 +20,8 @@ import requests
 
 from osducli.click_cli import CustomClickCommand, State, command_with_output
 from osducli.cliclient import CliOsduClient, handle_cli_exceptions
-from osducli.commands.dataload.status import check_status
 from osducli.commands.dataload.verify import batch_verify
+from osducli.commands.workflow.status import check_status
 from osducli.config import (
     CONFIG_ACL_OWNER,
     CONFIG_ACL_VIEWER,

src/osducli/commands/dataload/status.py

@@ -12,14 +12,11 @@
 
 """Dataload status command"""
 
-import sys
-import time
-
 import click
 
 from osducli.click_cli import CustomClickCommand, State, command_with_output
-from osducli.cliclient import CliOsduClient, handle_cli_exceptions
-from osducli.config import CONFIG_WORKFLOW_URL, CLIConfig
+from osducli.cliclient import handle_cli_exceptions
+from osducli.commands.workflow.status import status
 from osducli.log import get_logger
 
 START_TIME = "startTimeStamp"
@@ -50,91 +47,3 @@ logger = get_logger(__name__)
 def _click_command(state: State, runid: str = None, runid_log: str = None, wait: bool = False):
     """Get status of workflow runs."""
     return status(state, runid, runid_log, wait)
-
-
-def status(state: State, runid: str = None, runid_log: str = None, wait: bool = False) -> dict:
-    """Get status of workflow runs
-
-    Args:
-        state (State): Global state
-        group (str): Email address of the group
-
-    Returns:
-        dict: Response from service
-    """
-    runids = []
-    if runid is not None:
-        runids = [runid]
-    elif runid_log is not None:
-        with open(runid_log) as handle:
-            runids = [run_id.rstrip() for run_id in handle]
-    else:
-        logger.error("Specify either runid or runid_log")
-        sys.exit(1)
-
-    return check_status(state.config, runids, wait)
-
-
-def check_status(config: CLIConfig, runids: list, wait: bool) -> list:
-    """Check statis for a list of runids
-
-    Args:
-        config (CLIConfig): configuration
-        runids (list): list of runids
-        wait (bool): whether to wait for status to change out of running
-
-    Returns:
-        list: list containing runid and status.
-    """
-    results = _check_status(config, runids)
-
-    if wait:
-        # parse the results to see if the ingestion is complete.
-        while True:
-            ingestion_complete = True
-            for result in results:
-                if result.get("status") == "running":
-                    ingestion_complete = False
-                    logger.debug(
-                        "Not all records finished. Checking again in 30s. Tried upto %s and found running",
-                        result.get("runId"),
-                    )
-                    break
-
-            if ingestion_complete:
-                break
-
-            print(results)
-            time.sleep(30)  # 30 seconds sleep.
-            results = _check_status(config, runids)  # recheck the status.
-
-    return results
-
-
-def _check_status(config: CLIConfig, run_id_list: list):
-    logger.debug("list of run-ids: %s", run_id_list)
-
-    results = []
-    for run_id in run_id_list:
-        connection = CliOsduClient(config)
-        response_json = connection.cli_get_returning_json(
-            CONFIG_WORKFLOW_URL, "workflow/Osdu_ingest/workflowRun/" + run_id
-        )
-        if response_json is not None:
-            run_status = response_json.get(STATUS)
-            if run_status == "running":
-                results.append({RUN_ID: run_id, STATUS: run_status})
-            else:
-                time_taken = response_json.get(END_TIME) - response_json.get(START_TIME)
-                results.append(
-                    {
-                        RUN_ID: run_id,
-                        END_TIME: response_json.get(END_TIME),
-                        START_TIME: response_json.get(START_TIME),
-                        STATUS: run_status,
-                        TIME_TAKEN: time_taken / 1000,
-                    }
-                )
-        else:
-            results.append({RUN_ID: run_id, STATUS: "Unable To fetch status"})
-    return results

src/osducli/commands/entitlements/_const.py

@@ -15,3 +15,4 @@
 ENTITLEMENTS_SERVICE_NAME = "Entitlements Service"
 ENTITLEMENTS_STATUS_PATH = "_ah/readiness_check"
 ENTITLEMENTS_SWAGGER_PATH = "swagger-ui.html"
+ENTITLEMENTS_DOCUMENTATION = "https://community.opengroup.org/osdu/platform/security-and-compliance/entitlements/-/blob/master/docs/tutorial/Entitlements-Service.md"  # noqa: E501 pylint: disable=line-too-long

src/osducli/commands/entitlements/info.py

@@ -17,6 +17,7 @@ import click
 from osducli.click_cli import CustomClickCommand, State, command_with_output
 from osducli.cliclient import handle_cli_exceptions
 from osducli.commands.entitlements._const import (
+    ENTITLEMENTS_DOCUMENTATION,
     ENTITLEMENTS_SERVICE_NAME,
     ENTITLEMENTS_STATUS_PATH,
     ENTITLEMENTS_SWAGGER_PATH,
@@ -37,4 +38,5 @@ def _click_command(state: State) -> dict:
         CONFIG_ENTITLEMENTS_URL,
         ENTITLEMENTS_STATUS_PATH,
         ENTITLEMENTS_SWAGGER_PATH,
+        ENTITLEMENTS_DOCUMENTATION,
     )

src/osducli/commands/legal/_const.py

@@ -15,3 +15,4 @@
 LEGAL_SERVICE_NAME = "Legal service"
 LEGAL_STATUS_PATH = "_ah/readiness_check"
 LEGAL_SWAGGER_PATH = "swagger-ui.html"
+LEGAL_DOCUMENTATION = "https://community.opengroup.org/osdu/platform/security-and-compliance/legal/-/blob/master/docs/tutorial/ComplianceService.md"  # noqa: E501 pylint: disable=line-too-long

src/osducli/commands/legal/info.py

@@ -17,6 +17,7 @@ import click
 from osducli.click_cli import CustomClickCommand, State, command_with_output
 from osducli.cliclient import handle_cli_exceptions
 from osducli.commands.legal._const import (
+    LEGAL_DOCUMENTATION,
     LEGAL_SERVICE_NAME,
     LEGAL_STATUS_PATH,
     LEGAL_SWAGGER_PATH,
@@ -31,4 +32,11 @@ from osducli.util.service_info import info
 @command_with_output(None)
 def _click_command(state: State) -> dict:
     """Information about the service"""
-    return info(state, LEGAL_SERVICE_NAME, CONFIG_LEGAL_URL, LEGAL_STATUS_PATH, LEGAL_SWAGGER_PATH)
+    return info(
+        state,
+        LEGAL_SERVICE_NAME,
+        CONFIG_LEGAL_URL,
+        LEGAL_STATUS_PATH,
+        LEGAL_SWAGGER_PATH,
+        LEGAL_DOCUMENTATION,
+    )

src/osducli/commands/schema/_const.py

@@ -15,3 +15,4 @@
 SCHEMA_SERVICE_NAME = "Schema service"
 SCHEMA_STATUS_PATH = "schema?limit=1"
 SCHEMA_SWAGGER_PATH = "swagger-ui.html"
+SCHEMA_DOCUMENTATION = "https://community.opengroup.org/osdu/platform/system/schema-service/-/blob/master/docs/SchemaService-OSDU.md"  # noqa: E501 pylint: disable=line-too-long

src/osducli/commands/schema/info.py

@@ -17,6 +17,7 @@ import click
 from osducli.click_cli import CustomClickCommand, State, command_with_output
 from osducli.cliclient import handle_cli_exceptions
 from osducli.commands.schema._const import (
+    SCHEMA_DOCUMENTATION,
     SCHEMA_SERVICE_NAME,
     SCHEMA_STATUS_PATH,
     SCHEMA_SWAGGER_PATH,
@@ -32,5 +33,10 @@ from osducli.util.service_info import info
 def _click_command(state: State) -> dict:
     """Information about the service"""
     return info(
-        state, SCHEMA_SERVICE_NAME, CONFIG_SCHEMA_URL, SCHEMA_STATUS_PATH, SCHEMA_SWAGGER_PATH
+        state,
+        SCHEMA_SERVICE_NAME,
+        CONFIG_SCHEMA_URL,
+        SCHEMA_STATUS_PATH,
+        SCHEMA_SWAGGER_PATH,
+        SCHEMA_DOCUMENTATION,
     )

src/osducli/commands/search/_const.py

@@ -15,3 +15,4 @@
 SEARCH_SERVICE_NAME = "Search service"
 SEARCH_STATUS_PATH = "health/readiness_check"
 SEARCH_SWAGGER_PATH = "swagger-ui.html"
+SEARCH_DOCUMENTATION = "https://community.opengroup.org/osdu/platform/system/search-service/-/blob/master/docs/tutorial/SearchService.md"  # noqa: E501 pylint: disable=line-too-long

src/osducli/commands/search/info.py

@@ -17,6 +17,7 @@ import click
 from osducli.click_cli import CustomClickCommand, State, command_with_output
 from osducli.cliclient import handle_cli_exceptions
 from osducli.commands.search._const import (
+    SEARCH_DOCUMENTATION,
     SEARCH_SERVICE_NAME,
     SEARCH_STATUS_PATH,
     SEARCH_SWAGGER_PATH,
@@ -32,5 +33,10 @@ from osducli.util.service_info import info
 def _click_command(state: State) -> dict:
     """Information about the service"""
     return info(
-        state, SEARCH_SERVICE_NAME, CONFIG_SEARCH_URL, SEARCH_STATUS_PATH, SEARCH_SWAGGER_PATH
+        state,
+        SEARCH_SERVICE_NAME,
+        CONFIG_SEARCH_URL,
+        SEARCH_STATUS_PATH,
+        SEARCH_SWAGGER_PATH,
+        SEARCH_DOCUMENTATION,
     )

src/osducli/commands/storage/_const.py

@@ -15,3 +15,4 @@
 STORAGE_SERVICE_NAME = "Storage service"
 STORAGE_STATUS_PATH = "health"
 STORAGE_SWAGGER_PATH = "swagger-ui.html"
+STORAGE_DOCUMENTATION = "https://community.opengroup.org/osdu/platform/system/storage/-/blob/master/docs/tutorial/StorageService.md"  # noqa: E501 pylint: disable=line-too-long

src/osducli/commands/storage/info.py

@@ -17,6 +17,7 @@ import click
 from osducli.click_cli import CustomClickCommand, State, command_with_output
 from osducli.cliclient import handle_cli_exceptions
 from osducli.commands.storage._const import (
+    STORAGE_DOCUMENTATION,
     STORAGE_SERVICE_NAME,
     STORAGE_STATUS_PATH,
     STORAGE_SWAGGER_PATH,
@@ -32,5 +33,10 @@ from osducli.util.service_info import info
 def _click_command(state: State) -> dict:
     """Information about the service"""
     return info(
-        state, STORAGE_SERVICE_NAME, CONFIG_STORAGE_URL, STORAGE_STATUS_PATH, STORAGE_SWAGGER_PATH
+        state,
+        STORAGE_SERVICE_NAME,
+        CONFIG_STORAGE_URL,
+        STORAGE_STATUS_PATH,
+        STORAGE_SWAGGER_PATH,
+        STORAGE_DOCUMENTATION,
     )

src/osducli/commands/workflow/get.py

+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""Workflow get command"""
+
+import click
+
+from osducli.click_cli import CustomClickCommand, State, command_with_output
+from osducli.cliclient import CliOsduClient, handle_cli_exceptions
+from osducli.config import CONFIG_WORKFLOW_URL
+
+
+# click entry point
+@click.command(cls=CustomClickCommand)
+@click.option("-n", "--name", help="Workflow (DAG) name", required=True)
+@handle_cli_exceptions
+@command_with_output(None)
+def _click_command(state: State, name: str) -> dict:
+    """Get information about a workflow"""
+    return get_workflow(state, name)
+
+
+def get_workflow(state: State, name: str) -> dict:
+    """Get a workflows
+
+    Args:
+        state (State): Global state
+
+    Returns:
+        dict: Response from service
+    """
+    connection = CliOsduClient(state.config)
+    url = "workflow/" + name
+    response_json = connection.cli_get_returning_json(CONFIG_WORKFLOW_URL, url)
+    return response_json

src/osducli/commands/workflow/register.py

@@ -21,7 +21,7 @@ from osducli.config import CONFIG_WORKFLOW_URL
 
 # click entry point
 @click.command(cls=CustomClickCommand)
-@click.option("-n", "--name", help="DAG Name", required=True)
+@click.option("-n", "--name", help="Workflow (DAG) name", required=True)
 @click.option("-d", "--description", help="Description", required=True)
 @handle_cli_exceptions
 @command_with_output(None)

src/osducli/commands/workflow/runs.py

+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""Workflow runs command"""
+
+import click
+
+from osducli.click_cli import CustomClickCommand, State, command_with_output
+from osducli.cliclient import CliOsduClient, handle_cli_exceptions
+from osducli.config import CONFIG_WORKFLOW_URL
+
+
+# click entry point
+@click.command(cls=CustomClickCommand)
+@click.option("-n", "--name", help="Workflow (DAG) name", required=True)
+@handle_cli_exceptions
+@command_with_output(None)
+def _click_command(state: State, name: str) -> dict:
+    """Get runs of a workflow"""
+    return runs(state, name)
+
+
+def runs(state: State, name: str) -> dict:
+    """Get runs
+
+    Args:
+        state (State): Global state
+
+    Returns:
+        dict: Response from service
+    """
+    connection = CliOsduClient(state.config)
+    url = f"workflow/{name}/workflowRun"
+    response_json = connection.cli_get_returning_json(CONFIG_WORKFLOW_URL, url)
+    return response_json

src/osducli/commands/workflow/status.py

+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""Dataload status command"""
+
+import sys
+import time
+
+import click
+
+from osducli.click_cli import CustomClickCommand, State, command_with_output
+from osducli.cliclient import CliOsduClient, handle_cli_exceptions
+from osducli.config import CONFIG_WORKFLOW_URL, CLIConfig
+from osducli.log import get_logger
+
+START_TIME = "startTimeStamp"
+END_TIME = "endTimeStamp"
+STATUS = "status"
+RUN_ID = "runId"
+TIME_TAKEN = "timeTaken"
+FINISHED = "finished"
+FAILED = "failed"
+
+logger = get_logger(__name__)
+
+
+# click entry point
+@click.command(cls=CustomClickCommand)
+@click.option("-r", "--runid", help="Runid to query status of.")
+@click.option(
+    "-rl",
+    "--runid-log",
+    help="Path to a file containing run ids to get status of (see dataload ingest -h).",
+    type=click.Path(exists=True, file_okay=True, readable=True, resolve_path=True),
+)
+@click.option(
+    "-w", "--wait", help="Whether to wait for runs to complete.", is_flag=True, show_default=True
+)
+@handle_cli_exceptions
+@command_with_output(None)
+def _click_command(state: State, runid: str = None, runid_log: str = None, wait: bool = False):
+    """Get status of workflow runs."""
+    return status(state, runid, runid_log, wait)
+
+
+def status(state: State, runid: str = None, runid_log: str = None, wait: bool = False) -> dict:
+    """Get status of workflow runs
+
+    Args:
+        state (State): Global state
+        group (str): Email address of the group
+
+    Returns:
+        dict: Response from service
+    """
+    runids = []
+    if runid is not None:
+        runids = [runid]
+    elif runid_log is not None:
+        with open(runid_log) as handle:
+            runids = [run_id.rstrip() for run_id in handle]
+    else:
+        logger.error("Specify either runid or runid-log")
+        sys.exit(1)
+
+    return check_status(state.config, runids, wait)
+
+
+def check_status(config: CLIConfig, runids: list, wait: bool) -> list:
+    """Check statis for a list of runids
+
+    Args:
+        config (CLIConfig): configuration
+        runids (list): list of runids
+        wait (bool): whether to wait for status to change out of running
+
+    Returns:
+        list: list containing runid and status.
+    """
+    results = _check_status(config, runids)
+
+    if wait:
+        # parse the results to see if the ingestion is complete.
+        while True:
+            ingestion_complete = True
+            for result in results:
+                if result.get("status") == "running":
+                    ingestion_complete = False
+                    logger.debug(
+                        "Not all records finished. Checking again in 30s. Tried upto %s and found running",
+                        result.get("runId"),
+                    )
+                    break
+
+            if ingestion_complete:
+                break
+
+            print(results)
+            time.sleep(30)  # 30 seconds sleep.
+            results = _check_status(config, runids)  # recheck the status.
+
+    return results
+
+
+def _check_status(config: CLIConfig, run_id_list: list):
+    logger.debug("list of run-ids: %s", run_id_list)
+
+    results = []
+    for run_id in run_id_list:
+        connection = CliOsduClient(config)
+        response_json = connection.cli_get_returning_json(
+            CONFIG_WORKFLOW_URL, "workflow/Osdu_ingest/workflowRun/" + run_id
+        )
+        if response_json is not None:
+            run_status = response_json.get(STATUS)
+            if run_status == "running":
+                results.append({RUN_ID: run_id, STATUS: run_status})
+            else:
+                time_taken = response_json.get(END_TIME) - response_json.get(START_TIME)
+                results.append(
+                    {
+                        RUN_ID: run_id,
+                        END_TIME: response_json.get(END_TIME),
+                        START_TIME: response_json.get(START_TIME),
+                        STATUS: run_status,
+                        TIME_TAKEN: time_taken / 1000,
+                    }
+                )
+        else:
+            results.append({RUN_ID: run_id, STATUS: "Unable To fetch status"})
+    return results

src/osducli/commands/workflow/unregister.py

@@ -21,7 +21,7 @@ from osducli.config import CONFIG_WORKFLOW_URL
 
 # click entry point
 @click.command(cls=CustomClickCommand)
-@click.option("-n", "--name", help="DAG Name", required=True)
+@click.option("-n", "--name", help="Workflow (DAG) name", required=True)
 @handle_cli_exceptions
 @global_params
 def _click_command(state: State, name: str) -> dict:

tests/commands/entitlements/test_info.py

@@ -19,6 +19,7 @@ from click.testing import CliRunner
 
 from osducli.__main__ import cli
 from osducli.commands.entitlements._const import (
+    ENTITLEMENTS_DOCUMENTATION,
     ENTITLEMENTS_SERVICE_NAME,
     ENTITLEMENTS_STATUS_PATH,
     ENTITLEMENTS_SWAGGER_PATH,
@@ -40,6 +41,7 @@ class ServiceInfoTests(unittest.TestCase):
                 CONFIG_ENTITLEMENTS_URL,
                 ENTITLEMENTS_STATUS_PATH,
                 ENTITLEMENTS_SWAGGER_PATH,
+                ENTITLEMENTS_DOCUMENTATION,
             )
             self.assertEqual(result.exit_code, 0)