Commit c34f5f58 authored by Mark Hewitt's avatar Mark Hewitt
Browse files

Merge branch 'feature/workflow-commands' into 'main'

API documentation pages & Workflow commands

See merge request !12
parents 74c95250 9190f1b6
Pipeline #85536 passed with stages
in 2 minutes and 54 seconds
...@@ -28,6 +28,11 @@ For more information, specify the `-h` flag: ...@@ -28,6 +28,11 @@ For more information, specify the `-h` flag:
Change Log Change Log
========== ==========
0.0.30
- API documentation pages are shown in info commands
- workflow get, runs and status commands
0.0.29 0.0.29
------ ------
......
...@@ -12,4 +12,4 @@ ...@@ -12,4 +12,4 @@
""" OSDU command line environment""" """ OSDU command line environment"""
__VERSION__ = "0.0.29" __VERSION__ = "0.0.30"
...@@ -20,8 +20,8 @@ import requests ...@@ -20,8 +20,8 @@ import requests
from osducli.click_cli import CustomClickCommand, State, command_with_output from osducli.click_cli import CustomClickCommand, State, command_with_output
from osducli.cliclient import CliOsduClient, handle_cli_exceptions from osducli.cliclient import CliOsduClient, handle_cli_exceptions
from osducli.commands.dataload.status import check_status
from osducli.commands.dataload.verify import batch_verify from osducli.commands.dataload.verify import batch_verify
from osducli.commands.workflow.status import check_status
from osducli.config import ( from osducli.config import (
CONFIG_ACL_OWNER, CONFIG_ACL_OWNER,
CONFIG_ACL_VIEWER, CONFIG_ACL_VIEWER,
......
...@@ -12,14 +12,11 @@ ...@@ -12,14 +12,11 @@
"""Dataload status command""" """Dataload status command"""
import sys
import time
import click import click
from osducli.click_cli import CustomClickCommand, State, command_with_output from osducli.click_cli import CustomClickCommand, State, command_with_output
from osducli.cliclient import CliOsduClient, handle_cli_exceptions from osducli.cliclient import handle_cli_exceptions
from osducli.config import CONFIG_WORKFLOW_URL, CLIConfig from osducli.commands.workflow.status import status
from osducli.log import get_logger from osducli.log import get_logger
START_TIME = "startTimeStamp" START_TIME = "startTimeStamp"
...@@ -50,91 +47,3 @@ logger = get_logger(__name__) ...@@ -50,91 +47,3 @@ logger = get_logger(__name__)
def _click_command(state: State, runid: str = None, runid_log: str = None, wait: bool = False): def _click_command(state: State, runid: str = None, runid_log: str = None, wait: bool = False):
"""Get status of workflow runs.""" """Get status of workflow runs."""
return status(state, runid, runid_log, wait) return status(state, runid, runid_log, wait)
def status(state: State, runid: str = None, runid_log: str = None, wait: bool = False) -> dict:
"""Get status of workflow runs
Args:
state (State): Global state
group (str): Email address of the group
Returns:
dict: Response from service
"""
runids = []
if runid is not None:
runids = [runid]
elif runid_log is not None:
with open(runid_log) as handle:
runids = [run_id.rstrip() for run_id in handle]
else:
logger.error("Specify either runid or runid_log")
sys.exit(1)
return check_status(state.config, runids, wait)
def check_status(config: CLIConfig, runids: list, wait: bool) -> list:
"""Check statis for a list of runids
Args:
config (CLIConfig): configuration
runids (list): list of runids
wait (bool): whether to wait for status to change out of running
Returns:
list: list containing runid and status.
"""
results = _check_status(config, runids)
if wait:
# parse the results to see if the ingestion is complete.
while True:
ingestion_complete = True
for result in results:
if result.get("status") == "running":
ingestion_complete = False
logger.debug(
"Not all records finished. Checking again in 30s. Tried upto %s and found running",
result.get("runId"),
)
break
if ingestion_complete:
break
print(results)
time.sleep(30) # 30 seconds sleep.
results = _check_status(config, runids) # recheck the status.
return results
def _check_status(config: CLIConfig, run_id_list: list):
logger.debug("list of run-ids: %s", run_id_list)
results = []
for run_id in run_id_list:
connection = CliOsduClient(config)
response_json = connection.cli_get_returning_json(
CONFIG_WORKFLOW_URL, "workflow/Osdu_ingest/workflowRun/" + run_id
)
if response_json is not None:
run_status = response_json.get(STATUS)
if run_status == "running":
results.append({RUN_ID: run_id, STATUS: run_status})
else:
time_taken = response_json.get(END_TIME) - response_json.get(START_TIME)
results.append(
{
RUN_ID: run_id,
END_TIME: response_json.get(END_TIME),
START_TIME: response_json.get(START_TIME),
STATUS: run_status,
TIME_TAKEN: time_taken / 1000,
}
)
else:
results.append({RUN_ID: run_id, STATUS: "Unable To fetch status"})
return results
...@@ -15,3 +15,4 @@ ...@@ -15,3 +15,4 @@
ENTITLEMENTS_SERVICE_NAME = "Entitlements Service" ENTITLEMENTS_SERVICE_NAME = "Entitlements Service"
ENTITLEMENTS_STATUS_PATH = "_ah/readiness_check" ENTITLEMENTS_STATUS_PATH = "_ah/readiness_check"
ENTITLEMENTS_SWAGGER_PATH = "swagger-ui.html" ENTITLEMENTS_SWAGGER_PATH = "swagger-ui.html"
ENTITLEMENTS_DOCUMENTATION = "https://community.opengroup.org/osdu/platform/security-and-compliance/entitlements/-/blob/master/docs/tutorial/Entitlements-Service.md" # noqa: E501 pylint: disable=line-too-long
...@@ -17,6 +17,7 @@ import click ...@@ -17,6 +17,7 @@ import click
from osducli.click_cli import CustomClickCommand, State, command_with_output from osducli.click_cli import CustomClickCommand, State, command_with_output
from osducli.cliclient import handle_cli_exceptions from osducli.cliclient import handle_cli_exceptions
from osducli.commands.entitlements._const import ( from osducli.commands.entitlements._const import (
ENTITLEMENTS_DOCUMENTATION,
ENTITLEMENTS_SERVICE_NAME, ENTITLEMENTS_SERVICE_NAME,
ENTITLEMENTS_STATUS_PATH, ENTITLEMENTS_STATUS_PATH,
ENTITLEMENTS_SWAGGER_PATH, ENTITLEMENTS_SWAGGER_PATH,
...@@ -37,4 +38,5 @@ def _click_command(state: State) -> dict: ...@@ -37,4 +38,5 @@ def _click_command(state: State) -> dict:
CONFIG_ENTITLEMENTS_URL, CONFIG_ENTITLEMENTS_URL,
ENTITLEMENTS_STATUS_PATH, ENTITLEMENTS_STATUS_PATH,
ENTITLEMENTS_SWAGGER_PATH, ENTITLEMENTS_SWAGGER_PATH,
ENTITLEMENTS_DOCUMENTATION,
) )
...@@ -15,3 +15,4 @@ ...@@ -15,3 +15,4 @@
LEGAL_SERVICE_NAME = "Legal service" LEGAL_SERVICE_NAME = "Legal service"
LEGAL_STATUS_PATH = "_ah/readiness_check" LEGAL_STATUS_PATH = "_ah/readiness_check"
LEGAL_SWAGGER_PATH = "swagger-ui.html" LEGAL_SWAGGER_PATH = "swagger-ui.html"
LEGAL_DOCUMENTATION = "https://community.opengroup.org/osdu/platform/security-and-compliance/legal/-/blob/master/docs/tutorial/ComplianceService.md" # noqa: E501 pylint: disable=line-too-long
...@@ -17,6 +17,7 @@ import click ...@@ -17,6 +17,7 @@ import click
from osducli.click_cli import CustomClickCommand, State, command_with_output from osducli.click_cli import CustomClickCommand, State, command_with_output
from osducli.cliclient import handle_cli_exceptions from osducli.cliclient import handle_cli_exceptions
from osducli.commands.legal._const import ( from osducli.commands.legal._const import (
LEGAL_DOCUMENTATION,
LEGAL_SERVICE_NAME, LEGAL_SERVICE_NAME,
LEGAL_STATUS_PATH, LEGAL_STATUS_PATH,
LEGAL_SWAGGER_PATH, LEGAL_SWAGGER_PATH,
...@@ -31,4 +32,11 @@ from osducli.util.service_info import info ...@@ -31,4 +32,11 @@ from osducli.util.service_info import info
@command_with_output(None) @command_with_output(None)
def _click_command(state: State) -> dict: def _click_command(state: State) -> dict:
"""Information about the service""" """Information about the service"""
return info(state, LEGAL_SERVICE_NAME, CONFIG_LEGAL_URL, LEGAL_STATUS_PATH, LEGAL_SWAGGER_PATH) return info(
state,
LEGAL_SERVICE_NAME,
CONFIG_LEGAL_URL,
LEGAL_STATUS_PATH,
LEGAL_SWAGGER_PATH,
LEGAL_DOCUMENTATION,
)
...@@ -15,3 +15,4 @@ ...@@ -15,3 +15,4 @@
SCHEMA_SERVICE_NAME = "Schema service" SCHEMA_SERVICE_NAME = "Schema service"
SCHEMA_STATUS_PATH = "schema?limit=1" SCHEMA_STATUS_PATH = "schema?limit=1"
SCHEMA_SWAGGER_PATH = "swagger-ui.html" SCHEMA_SWAGGER_PATH = "swagger-ui.html"
SCHEMA_DOCUMENTATION = "https://community.opengroup.org/osdu/platform/system/schema-service/-/blob/master/docs/SchemaService-OSDU.md" # noqa: E501 pylint: disable=line-too-long
...@@ -17,6 +17,7 @@ import click ...@@ -17,6 +17,7 @@ import click
from osducli.click_cli import CustomClickCommand, State, command_with_output from osducli.click_cli import CustomClickCommand, State, command_with_output
from osducli.cliclient import handle_cli_exceptions from osducli.cliclient import handle_cli_exceptions
from osducli.commands.schema._const import ( from osducli.commands.schema._const import (
SCHEMA_DOCUMENTATION,
SCHEMA_SERVICE_NAME, SCHEMA_SERVICE_NAME,
SCHEMA_STATUS_PATH, SCHEMA_STATUS_PATH,
SCHEMA_SWAGGER_PATH, SCHEMA_SWAGGER_PATH,
...@@ -32,5 +33,10 @@ from osducli.util.service_info import info ...@@ -32,5 +33,10 @@ from osducli.util.service_info import info
def _click_command(state: State) -> dict: def _click_command(state: State) -> dict:
"""Information about the service""" """Information about the service"""
return info( return info(
state, SCHEMA_SERVICE_NAME, CONFIG_SCHEMA_URL, SCHEMA_STATUS_PATH, SCHEMA_SWAGGER_PATH state,
SCHEMA_SERVICE_NAME,
CONFIG_SCHEMA_URL,
SCHEMA_STATUS_PATH,
SCHEMA_SWAGGER_PATH,
SCHEMA_DOCUMENTATION,
) )
...@@ -15,3 +15,4 @@ ...@@ -15,3 +15,4 @@
SEARCH_SERVICE_NAME = "Search service" SEARCH_SERVICE_NAME = "Search service"
SEARCH_STATUS_PATH = "health/readiness_check" SEARCH_STATUS_PATH = "health/readiness_check"
SEARCH_SWAGGER_PATH = "swagger-ui.html" SEARCH_SWAGGER_PATH = "swagger-ui.html"
SEARCH_DOCUMENTATION = "https://community.opengroup.org/osdu/platform/system/search-service/-/blob/master/docs/tutorial/SearchService.md" # noqa: E501 pylint: disable=line-too-long
...@@ -17,6 +17,7 @@ import click ...@@ -17,6 +17,7 @@ import click
from osducli.click_cli import CustomClickCommand, State, command_with_output from osducli.click_cli import CustomClickCommand, State, command_with_output
from osducli.cliclient import handle_cli_exceptions from osducli.cliclient import handle_cli_exceptions
from osducli.commands.search._const import ( from osducli.commands.search._const import (
SEARCH_DOCUMENTATION,
SEARCH_SERVICE_NAME, SEARCH_SERVICE_NAME,
SEARCH_STATUS_PATH, SEARCH_STATUS_PATH,
SEARCH_SWAGGER_PATH, SEARCH_SWAGGER_PATH,
...@@ -32,5 +33,10 @@ from osducli.util.service_info import info ...@@ -32,5 +33,10 @@ from osducli.util.service_info import info
def _click_command(state: State) -> dict: def _click_command(state: State) -> dict:
"""Information about the service""" """Information about the service"""
return info( return info(
state, SEARCH_SERVICE_NAME, CONFIG_SEARCH_URL, SEARCH_STATUS_PATH, SEARCH_SWAGGER_PATH state,
SEARCH_SERVICE_NAME,
CONFIG_SEARCH_URL,
SEARCH_STATUS_PATH,
SEARCH_SWAGGER_PATH,
SEARCH_DOCUMENTATION,
) )
...@@ -15,3 +15,4 @@ ...@@ -15,3 +15,4 @@
STORAGE_SERVICE_NAME = "Storage service" STORAGE_SERVICE_NAME = "Storage service"
STORAGE_STATUS_PATH = "health" STORAGE_STATUS_PATH = "health"
STORAGE_SWAGGER_PATH = "swagger-ui.html" STORAGE_SWAGGER_PATH = "swagger-ui.html"
STORAGE_DOCUMENTATION = "https://community.opengroup.org/osdu/platform/system/storage/-/blob/master/docs/tutorial/StorageService.md" # noqa: E501 pylint: disable=line-too-long
...@@ -17,6 +17,7 @@ import click ...@@ -17,6 +17,7 @@ import click
from osducli.click_cli import CustomClickCommand, State, command_with_output from osducli.click_cli import CustomClickCommand, State, command_with_output
from osducli.cliclient import handle_cli_exceptions from osducli.cliclient import handle_cli_exceptions
from osducli.commands.storage._const import ( from osducli.commands.storage._const import (
STORAGE_DOCUMENTATION,
STORAGE_SERVICE_NAME, STORAGE_SERVICE_NAME,
STORAGE_STATUS_PATH, STORAGE_STATUS_PATH,
STORAGE_SWAGGER_PATH, STORAGE_SWAGGER_PATH,
...@@ -32,5 +33,10 @@ from osducli.util.service_info import info ...@@ -32,5 +33,10 @@ from osducli.util.service_info import info
def _click_command(state: State) -> dict: def _click_command(state: State) -> dict:
"""Information about the service""" """Information about the service"""
return info( return info(
state, STORAGE_SERVICE_NAME, CONFIG_STORAGE_URL, STORAGE_STATUS_PATH, STORAGE_SWAGGER_PATH state,
STORAGE_SERVICE_NAME,
CONFIG_STORAGE_URL,
STORAGE_STATUS_PATH,
STORAGE_SWAGGER_PATH,
STORAGE_DOCUMENTATION,
) )
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Workflow get command"""
import click
from osducli.click_cli import CustomClickCommand, State, command_with_output
from osducli.cliclient import CliOsduClient, handle_cli_exceptions
from osducli.config import CONFIG_WORKFLOW_URL
# click entry point
@click.command(cls=CustomClickCommand)
@click.option("-n", "--name", help="Workflow (DAG) name", required=True)
@handle_cli_exceptions
@command_with_output(None)
def _click_command(state: State, name: str) -> dict:
"""Get information about a workflow"""
return get_workflow(state, name)
def get_workflow(state: State, name: str) -> dict:
"""Get a workflows
Args:
state (State): Global state
Returns:
dict: Response from service
"""
connection = CliOsduClient(state.config)
url = "workflow/" + name
response_json = connection.cli_get_returning_json(CONFIG_WORKFLOW_URL, url)
return response_json
...@@ -21,7 +21,7 @@ from osducli.config import CONFIG_WORKFLOW_URL ...@@ -21,7 +21,7 @@ from osducli.config import CONFIG_WORKFLOW_URL
# click entry point # click entry point
@click.command(cls=CustomClickCommand) @click.command(cls=CustomClickCommand)
@click.option("-n", "--name", help="DAG Name", required=True) @click.option("-n", "--name", help="Workflow (DAG) name", required=True)
@click.option("-d", "--description", help="Description", required=True) @click.option("-d", "--description", help="Description", required=True)
@handle_cli_exceptions @handle_cli_exceptions
@command_with_output(None) @command_with_output(None)
......
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Workflow runs command"""
import click
from osducli.click_cli import CustomClickCommand, State, command_with_output
from osducli.cliclient import CliOsduClient, handle_cli_exceptions
from osducli.config import CONFIG_WORKFLOW_URL
# click entry point
@click.command(cls=CustomClickCommand)
@click.option("-n", "--name", help="Workflow (DAG) name", required=True)
@handle_cli_exceptions
@command_with_output(None)
def _click_command(state: State, name: str) -> dict:
"""Get runs of a workflow"""
return runs(state, name)
def runs(state: State, name: str) -> dict:
"""Get runs
Args:
state (State): Global state
Returns:
dict: Response from service
"""
connection = CliOsduClient(state.config)
url = f"workflow/{name}/workflowRun"
response_json = connection.cli_get_returning_json(CONFIG_WORKFLOW_URL, url)
return response_json
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Dataload status command"""
import sys
import time
import click
from osducli.click_cli import CustomClickCommand, State, command_with_output
from osducli.cliclient import CliOsduClient, handle_cli_exceptions
from osducli.config import CONFIG_WORKFLOW_URL, CLIConfig
from osducli.log import get_logger
START_TIME = "startTimeStamp"
END_TIME = "endTimeStamp"
STATUS = "status"
RUN_ID = "runId"
TIME_TAKEN = "timeTaken"
FINISHED = "finished"
FAILED = "failed"
logger = get_logger(__name__)
# click entry point
@click.command(cls=CustomClickCommand)
@click.option("-r", "--runid", help="Runid to query status of.")
@click.option(
"-rl",
"--runid-log",
help="Path to a file containing run ids to get status of (see dataload ingest -h).",
type=click.Path(exists=True, file_okay=True, readable=True, resolve_path=True),
)
@click.option(
"-w", "--wait", help="Whether to wait for runs to complete.", is_flag=True, show_default=True
)
@handle_cli_exceptions
@command_with_output(None)
def _click_command(state: State, runid: str = None, runid_log: str = None, wait: bool = False):
"""Get status of workflow runs."""
return status(state, runid, runid_log, wait)
def status(state: State, runid: str = None, runid_log: str = None, wait: bool = False) -> dict:
"""Get status of workflow runs
Args:
state (State): Global state
group (str): Email address of the group
Returns:
dict: Response from service
"""
runids = []
if runid is not None:
runids = [runid]
elif runid_log is not None:
with open(runid_log) as handle:
runids = [run_id.rstrip() for run_id in handle]
else:
logger.error("Specify either runid or runid-log")
sys.exit(1)
return check_status(state.config, runids, wait)
def check_status(config: CLIConfig, runids: list, wait: bool) -> list:
"""Check statis for a list of runids
Args:
config (CLIConfig): configuration
runids (list): list of runids
wait (bool): whether to wait for status to change out of running
Returns:
list: list containing runid and status.
"""
results = _check_status(config, runids)
if wait:
# parse the results to see if the ingestion is complete.
while True:
ingestion_complete = True
for result in results:
if result.get("status") == "running":
ingestion_complete = False
logger.debug(
"Not all records finished. Checking again in 30s. Tried upto %s and found running",
result.get("runId"),
)
break
if ingestion_complete:
break
print(results)
time.sleep(30) # 30 seconds sleep.
results = _check_status(config, runids) # recheck the status.
return results
def _check_status(config: CLIConfig, run_id_list: list):
logger.debug("list of run-ids: %s", run_id_list)
results = []
for run_id in run_id_list:
connection = CliOsduClient(config)
response_json = connection.cli_get_returning_json(
CONFIG_WORKFLOW_URL, "workflow/Osdu_ingest/workflowRun/" + run_id
)
if response_json is not None:
run_status = response_json.get(STATUS)
if run_status == "running":
results.append({RUN_ID: run_id, STATUS: run_status})
else:
time_taken = response_json.get(END_TIME) - response_json.get(START_TIME)
results.append(
{
RUN_ID: run_id,
END_TIME: response_json.get(END_TIME),
START_TIME: response_json.get(START_TIME),
STATUS: run_status,
TIME_TAKEN: time_taken / 1000,
}
)
else:
results.append({RUN_ID: run_id, STATUS: "Unable To fetch status"})
return results
...@@ -21,7 +21,7 @@ from osducli.config import CONFIG_WORKFLOW_URL ...@@ -21,7 +21,7 @@ from osducli.config import CONFIG_WORKFLOW_URL
# click entry point # click entry point
@click.command(cls=CustomClickCommand) @click.command(cls=CustomClickCommand)
@click.option("-n", "--name", help="DAG Name", required=True) @click.option("-n", "--name", help="Workflow (DAG) name", required=True)
@handle_cli_exceptions @handle_cli_exceptions
@global_params @global_params
def _click_command(state: State, name: str) -> dict: def _click_command(state: State, name: str) -> dict:
......