Commit c34f5f58 authored by Mark Hewitt's avatar Mark Hewitt
Browse files

Merge branch 'feature/workflow-commands' into 'main'

API documentation pages & Workflow commands

See merge request !12
parents 74c95250 9190f1b6
Pipeline #85536 passed with stages
in 2 minutes and 54 seconds
......@@ -28,6 +28,11 @@ For more information, specify the `-h` flag:
Change Log
==========
0.0.30
- API documentation pages are shown in info commands
- workflow get, runs and status commands
0.0.29
------
......
......@@ -12,4 +12,4 @@
""" OSDU command line environment"""
__VERSION__ = "0.0.29"
__VERSION__ = "0.0.30"
......@@ -20,8 +20,8 @@ import requests
from osducli.click_cli import CustomClickCommand, State, command_with_output
from osducli.cliclient import CliOsduClient, handle_cli_exceptions
from osducli.commands.dataload.status import check_status
from osducli.commands.dataload.verify import batch_verify
from osducli.commands.workflow.status import check_status
from osducli.config import (
CONFIG_ACL_OWNER,
CONFIG_ACL_VIEWER,
......
......@@ -12,14 +12,11 @@
"""Dataload status command"""
import sys
import time
import click
from osducli.click_cli import CustomClickCommand, State, command_with_output
from osducli.cliclient import CliOsduClient, handle_cli_exceptions
from osducli.config import CONFIG_WORKFLOW_URL, CLIConfig
from osducli.cliclient import handle_cli_exceptions
from osducli.commands.workflow.status import status
from osducli.log import get_logger
START_TIME = "startTimeStamp"
......@@ -50,91 +47,3 @@ logger = get_logger(__name__)
def _click_command(state: State, runid: str = None, runid_log: str = None, wait: bool = False):
"""Get status of workflow runs."""
return status(state, runid, runid_log, wait)
def status(state: State, runid: str = None, runid_log: str = None, wait: bool = False) -> dict:
"""Get status of workflow runs
Args:
state (State): Global state
group (str): Email address of the group
Returns:
dict: Response from service
"""
runids = []
if runid is not None:
runids = [runid]
elif runid_log is not None:
with open(runid_log) as handle:
runids = [run_id.rstrip() for run_id in handle]
else:
logger.error("Specify either runid or runid_log")
sys.exit(1)
return check_status(state.config, runids, wait)
def check_status(config: CLIConfig, runids: list, wait: bool) -> list:
"""Check statis for a list of runids
Args:
config (CLIConfig): configuration
runids (list): list of runids
wait (bool): whether to wait for status to change out of running
Returns:
list: list containing runid and status.
"""
results = _check_status(config, runids)
if wait:
# parse the results to see if the ingestion is complete.
while True:
ingestion_complete = True
for result in results:
if result.get("status") == "running":
ingestion_complete = False
logger.debug(
"Not all records finished. Checking again in 30s. Tried upto %s and found running",
result.get("runId"),
)
break
if ingestion_complete:
break
print(results)
time.sleep(30) # 30 seconds sleep.
results = _check_status(config, runids) # recheck the status.
return results
def _check_status(config: CLIConfig, run_id_list: list):
logger.debug("list of run-ids: %s", run_id_list)
results = []
for run_id in run_id_list:
connection = CliOsduClient(config)
response_json = connection.cli_get_returning_json(
CONFIG_WORKFLOW_URL, "workflow/Osdu_ingest/workflowRun/" + run_id
)
if response_json is not None:
run_status = response_json.get(STATUS)
if run_status == "running":
results.append({RUN_ID: run_id, STATUS: run_status})
else:
time_taken = response_json.get(END_TIME) - response_json.get(START_TIME)
results.append(
{
RUN_ID: run_id,
END_TIME: response_json.get(END_TIME),
START_TIME: response_json.get(START_TIME),
STATUS: run_status,
TIME_TAKEN: time_taken / 1000,
}
)
else:
results.append({RUN_ID: run_id, STATUS: "Unable To fetch status"})
return results
......@@ -15,3 +15,4 @@
ENTITLEMENTS_SERVICE_NAME = "Entitlements Service"
ENTITLEMENTS_STATUS_PATH = "_ah/readiness_check"
ENTITLEMENTS_SWAGGER_PATH = "swagger-ui.html"
ENTITLEMENTS_DOCUMENTATION = "https://community.opengroup.org/osdu/platform/security-and-compliance/entitlements/-/blob/master/docs/tutorial/Entitlements-Service.md" # noqa: E501 pylint: disable=line-too-long
......@@ -17,6 +17,7 @@ import click
from osducli.click_cli import CustomClickCommand, State, command_with_output
from osducli.cliclient import handle_cli_exceptions
from osducli.commands.entitlements._const import (
ENTITLEMENTS_DOCUMENTATION,
ENTITLEMENTS_SERVICE_NAME,
ENTITLEMENTS_STATUS_PATH,
ENTITLEMENTS_SWAGGER_PATH,
......@@ -37,4 +38,5 @@ def _click_command(state: State) -> dict:
CONFIG_ENTITLEMENTS_URL,
ENTITLEMENTS_STATUS_PATH,
ENTITLEMENTS_SWAGGER_PATH,
ENTITLEMENTS_DOCUMENTATION,
)
......@@ -15,3 +15,4 @@
LEGAL_SERVICE_NAME = "Legal service"
LEGAL_STATUS_PATH = "_ah/readiness_check"
LEGAL_SWAGGER_PATH = "swagger-ui.html"
LEGAL_DOCUMENTATION = "https://community.opengroup.org/osdu/platform/security-and-compliance/legal/-/blob/master/docs/tutorial/ComplianceService.md" # noqa: E501 pylint: disable=line-too-long
......@@ -17,6 +17,7 @@ import click
from osducli.click_cli import CustomClickCommand, State, command_with_output
from osducli.cliclient import handle_cli_exceptions
from osducli.commands.legal._const import (
LEGAL_DOCUMENTATION,
LEGAL_SERVICE_NAME,
LEGAL_STATUS_PATH,
LEGAL_SWAGGER_PATH,
......@@ -31,4 +32,11 @@ from osducli.util.service_info import info
@command_with_output(None)
def _click_command(state: State) -> dict:
"""Information about the service"""
return info(state, LEGAL_SERVICE_NAME, CONFIG_LEGAL_URL, LEGAL_STATUS_PATH, LEGAL_SWAGGER_PATH)
return info(
state,
LEGAL_SERVICE_NAME,
CONFIG_LEGAL_URL,
LEGAL_STATUS_PATH,
LEGAL_SWAGGER_PATH,
LEGAL_DOCUMENTATION,
)
......@@ -15,3 +15,4 @@
SCHEMA_SERVICE_NAME = "Schema service"
SCHEMA_STATUS_PATH = "schema?limit=1"
SCHEMA_SWAGGER_PATH = "swagger-ui.html"
SCHEMA_DOCUMENTATION = "https://community.opengroup.org/osdu/platform/system/schema-service/-/blob/master/docs/SchemaService-OSDU.md" # noqa: E501 pylint: disable=line-too-long
......@@ -17,6 +17,7 @@ import click
from osducli.click_cli import CustomClickCommand, State, command_with_output
from osducli.cliclient import handle_cli_exceptions
from osducli.commands.schema._const import (
SCHEMA_DOCUMENTATION,
SCHEMA_SERVICE_NAME,
SCHEMA_STATUS_PATH,
SCHEMA_SWAGGER_PATH,
......@@ -32,5 +33,10 @@ from osducli.util.service_info import info
def _click_command(state: State) -> dict:
"""Information about the service"""
return info(
state, SCHEMA_SERVICE_NAME, CONFIG_SCHEMA_URL, SCHEMA_STATUS_PATH, SCHEMA_SWAGGER_PATH
state,
SCHEMA_SERVICE_NAME,
CONFIG_SCHEMA_URL,
SCHEMA_STATUS_PATH,
SCHEMA_SWAGGER_PATH,
SCHEMA_DOCUMENTATION,
)
......@@ -15,3 +15,4 @@
SEARCH_SERVICE_NAME = "Search service"
SEARCH_STATUS_PATH = "health/readiness_check"
SEARCH_SWAGGER_PATH = "swagger-ui.html"
SEARCH_DOCUMENTATION = "https://community.opengroup.org/osdu/platform/system/search-service/-/blob/master/docs/tutorial/SearchService.md" # noqa: E501 pylint: disable=line-too-long
......@@ -17,6 +17,7 @@ import click
from osducli.click_cli import CustomClickCommand, State, command_with_output
from osducli.cliclient import handle_cli_exceptions
from osducli.commands.search._const import (
SEARCH_DOCUMENTATION,
SEARCH_SERVICE_NAME,
SEARCH_STATUS_PATH,
SEARCH_SWAGGER_PATH,
......@@ -32,5 +33,10 @@ from osducli.util.service_info import info
def _click_command(state: State) -> dict:
"""Information about the service"""
return info(
state, SEARCH_SERVICE_NAME, CONFIG_SEARCH_URL, SEARCH_STATUS_PATH, SEARCH_SWAGGER_PATH
state,
SEARCH_SERVICE_NAME,
CONFIG_SEARCH_URL,
SEARCH_STATUS_PATH,
SEARCH_SWAGGER_PATH,
SEARCH_DOCUMENTATION,
)
......@@ -15,3 +15,4 @@
STORAGE_SERVICE_NAME = "Storage service"
STORAGE_STATUS_PATH = "health"
STORAGE_SWAGGER_PATH = "swagger-ui.html"
STORAGE_DOCUMENTATION = "https://community.opengroup.org/osdu/platform/system/storage/-/blob/master/docs/tutorial/StorageService.md" # noqa: E501 pylint: disable=line-too-long
......@@ -17,6 +17,7 @@ import click
from osducli.click_cli import CustomClickCommand, State, command_with_output
from osducli.cliclient import handle_cli_exceptions
from osducli.commands.storage._const import (
STORAGE_DOCUMENTATION,
STORAGE_SERVICE_NAME,
STORAGE_STATUS_PATH,
STORAGE_SWAGGER_PATH,
......@@ -32,5 +33,10 @@ from osducli.util.service_info import info
def _click_command(state: State) -> dict:
"""Information about the service"""
return info(
state, STORAGE_SERVICE_NAME, CONFIG_STORAGE_URL, STORAGE_STATUS_PATH, STORAGE_SWAGGER_PATH
state,
STORAGE_SERVICE_NAME,
CONFIG_STORAGE_URL,
STORAGE_STATUS_PATH,
STORAGE_SWAGGER_PATH,
STORAGE_DOCUMENTATION,
)
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Workflow get command"""
import click
from osducli.click_cli import CustomClickCommand, State, command_with_output
from osducli.cliclient import CliOsduClient, handle_cli_exceptions
from osducli.config import CONFIG_WORKFLOW_URL
# click entry point
@click.command(cls=CustomClickCommand)
@click.option("-n", "--name", help="Workflow (DAG) name", required=True)
@handle_cli_exceptions
@command_with_output(None)
def _click_command(state: State, name: str) -> dict:
"""Get information about a workflow"""
return get_workflow(state, name)
def get_workflow(state: State, name: str) -> dict:
"""Get a workflows
Args:
state (State): Global state
Returns:
dict: Response from service
"""
connection = CliOsduClient(state.config)
url = "workflow/" + name
response_json = connection.cli_get_returning_json(CONFIG_WORKFLOW_URL, url)
return response_json
......@@ -21,7 +21,7 @@ from osducli.config import CONFIG_WORKFLOW_URL
# click entry point
@click.command(cls=CustomClickCommand)
@click.option("-n", "--name", help="DAG Name", required=True)
@click.option("-n", "--name", help="Workflow (DAG) name", required=True)
@click.option("-d", "--description", help="Description", required=True)
@handle_cli_exceptions
@command_with_output(None)
......
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Workflow runs command"""
import click
from osducli.click_cli import CustomClickCommand, State, command_with_output
from osducli.cliclient import CliOsduClient, handle_cli_exceptions
from osducli.config import CONFIG_WORKFLOW_URL
# click entry point
@click.command(cls=CustomClickCommand)
@click.option("-n", "--name", help="Workflow (DAG) name", required=True)
@handle_cli_exceptions
@command_with_output(None)
def _click_command(state: State, name: str) -> dict:
"""Get runs of a workflow"""
return runs(state, name)
def runs(state: State, name: str) -> dict:
"""Get runs
Args:
state (State): Global state
Returns:
dict: Response from service
"""
connection = CliOsduClient(state.config)
url = f"workflow/{name}/workflowRun"
response_json = connection.cli_get_returning_json(CONFIG_WORKFLOW_URL, url)
return response_json
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Dataload status command"""
import sys
import time
import click
from osducli.click_cli import CustomClickCommand, State, command_with_output
from osducli.cliclient import CliOsduClient, handle_cli_exceptions
from osducli.config import CONFIG_WORKFLOW_URL, CLIConfig
from osducli.log import get_logger
START_TIME = "startTimeStamp"
END_TIME = "endTimeStamp"
STATUS = "status"
RUN_ID = "runId"
TIME_TAKEN = "timeTaken"
FINISHED = "finished"
FAILED = "failed"
logger = get_logger(__name__)
# click entry point
@click.command(cls=CustomClickCommand)
@click.option("-r", "--runid", help="Runid to query status of.")
@click.option(
"-rl",
"--runid-log",
help="Path to a file containing run ids to get status of (see dataload ingest -h).",
type=click.Path(exists=True, file_okay=True, readable=True, resolve_path=True),
)
@click.option(
"-w", "--wait", help="Whether to wait for runs to complete.", is_flag=True, show_default=True
)
@handle_cli_exceptions
@command_with_output(None)
def _click_command(state: State, runid: str = None, runid_log: str = None, wait: bool = False):
"""Get status of workflow runs."""
return status(state, runid, runid_log, wait)
def status(state: State, runid: str = None, runid_log: str = None, wait: bool = False) -> dict:
"""Get status of workflow runs
Args:
state (State): Global state
group (str): Email address of the group
Returns:
dict: Response from service
"""
runids = []
if runid is not None:
runids = [runid]
elif runid_log is not None:
with open(runid_log) as handle:
runids = [run_id.rstrip() for run_id in handle]
else:
logger.error("Specify either runid or runid-log")
sys.exit(1)
return check_status(state.config, runids, wait)
def check_status(config: CLIConfig, runids: list, wait: bool) -> list:
"""Check statis for a list of runids
Args:
config (CLIConfig): configuration
runids (list): list of runids
wait (bool): whether to wait for status to change out of running
Returns:
list: list containing runid and status.
"""
results = _check_status(config, runids)
if wait:
# parse the results to see if the ingestion is complete.
while True:
ingestion_complete = True
for result in results:
if result.get("status") == "running":
ingestion_complete = False
logger.debug(
"Not all records finished. Checking again in 30s. Tried upto %s and found running",
result.get("runId"),
)
break
if ingestion_complete:
break
print(results)
time.sleep(30) # 30 seconds sleep.
results = _check_status(config, runids) # recheck the status.
return results
def _check_status(config: CLIConfig, run_id_list: list):
logger.debug("list of run-ids: %s", run_id_list)
results = []
for run_id in run_id_list:
connection = CliOsduClient(config)
response_json = connection.cli_get_returning_json(
CONFIG_WORKFLOW_URL, "workflow/Osdu_ingest/workflowRun/" + run_id
)
if response_json is not None:
run_status = response_json.get(STATUS)
if run_status == "running":
results.append({RUN_ID: run_id, STATUS: run_status})
else:
time_taken = response_json.get(END_TIME) - response_json.get(START_TIME)
results.append(
{
RUN_ID: run_id,
END_TIME: response_json.get(END_TIME),
START_TIME: response_json.get(START_TIME),
STATUS: run_status,
TIME_TAKEN: time_taken / 1000,
}
)
else:
results.append({RUN_ID: run_id, STATUS: "Unable To fetch status"})
return results
......@@ -21,7 +21,7 @@ from osducli.config import CONFIG_WORKFLOW_URL
# click entry point
@click.command(cls=CustomClickCommand)
@click.option("-n", "--name", help="DAG Name", required=True)
@click.option("-n", "--name", help="Workflow (DAG) name", required=True)
@handle_cli_exceptions
@global_params
def _click_command(state: State, name: str) -> dict:
......
......@@ -19,6 +19,7 @@ from click.testing import CliRunner
from osducli.__main__ import cli
from osducli.commands.entitlements._const import (
ENTITLEMENTS_DOCUMENTATION,
ENTITLEMENTS_SERVICE_NAME,
ENTITLEMENTS_STATUS_PATH,
ENTITLEMENTS_SWAGGER_PATH,
......@@ -40,6 +41,7 @@ class ServiceInfoTests(unittest.TestCase):
CONFIG_ENTITLEMENTS_URL,
ENTITLEMENTS_STATUS_PATH,
ENTITLEMENTS_SWAGGER_PATH,
ENTITLEMENTS_DOCUMENTATION,
)
self.assertEqual(result.exit_code, 0)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment