Commit 68ae1bde authored by Mark Hewitt's avatar Mark Hewitt
Browse files

Merge branch 'feature/checkrefs-authority' into 'main'

Feature/checkrefs authority

Closes #7

See merge request !5
parents cab9e3fb a5903914
Pipeline #76399 passed with stages
in 2 minutes and 26 seconds
......@@ -28,6 +28,11 @@ For more information, specify the `-h` flag:
Change Log
==========
0.0.24
------
- checkrefs authority, acl and legal parameters for generated files
0.0.23
------
......
......@@ -12,4 +12,4 @@
""" OSDU command line environment"""
__VERSION__ = "0.0.23"
__VERSION__ = "0.0.24"
......@@ -25,7 +25,12 @@ import click
from osducli.click_cli import CustomClickCommand, State, command_with_output
from osducli.cliclient import handle_cli_exceptions
from osducli.commands.dataload.verify import batch_verify
from osducli.config import CLIConfig
from osducli.config import (
CONFIG_ACL_OWNER,
CONFIG_ACL_VIEWER,
CONFIG_LEGAL_TAG,
CLIConfig,
)
from osducli.log import get_logger
from osducli.util.file import get_files_from_path
......@@ -55,6 +60,29 @@ WORK_PRODUCT_COMPONENTS_SECTION = "WorkProductComponents"
type=click.Path(exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True),
required=False,
)
@click.option(
"-a",
"--authority",
help="Schema authority to use when generating manifest files.",
default="osdu",
show_default=True,
required=False,
)
@click.option(
"-aclo",
"--acl-owners",
help="Acl owners to use when generating manifest files. If not specified the global value is used.",
)
@click.option(
"-aclv",
"--acl-viewers",
help="Acl viewers to use when generating manifest files. If not specified the global value is used.",
)
@click.option(
"-l",
"--legal-tags",
help="Legal tag to use when generating manifest files. If not specified the global value is used.",
)
# @click.option("-b", "--batch", help="Batch size.", type=int, default=200, show_default=True)
# @click.option(
# "--batch-across-files",
......@@ -65,17 +93,41 @@ WORK_PRODUCT_COMPONENTS_SECTION = "WorkProductComponents"
# )
@handle_cli_exceptions
@command_with_output(None)
def _click_command(state: State, path: str, generated: str):
def _click_command(
state: State,
path: str,
generated: str,
authority: str,
acl_owners: str,
acl_viewers: str,
legal_tags: str,
):
"""Find any referenced id's missing in OSDU (work in progress).
This command will try and extract all referenced id's in the given manifests and then check
whether these exist in OSDU. You might typically use this command prior to ingestion to check
whether ingestion might fail due to the referential integrity check."""
return check_references(state, path, generated) # , path, batch, batch_across_files)
return check_references(
state,
path,
generated,
authority,
acl_owners,
acl_viewers,
legal_tags,
)
# region Create Manifests
def create_reference_data_manifest_json(data_partition: str, kind: str, code: str):
def create_reference_data_manifest_json(
data_partition: str,
kind: str,
code: str,
authority: str,
acl_owners: str,
acl_viewers: str,
legal_tags: str,
):
"""
Create a manifest json representation
......@@ -85,13 +137,13 @@ def create_reference_data_manifest_json(data_partition: str, kind: str, code: st
"ReferenceData": [
{
"id": f"{data_partition}:{kind}:{code}",
"kind": f"{data_partition}:wks:{kind}:1.0.0",
"kind": f"{authority}:wks:{kind}:1.0.0",
"acl": {
"owners": ["data.default.owners@opendes.contoso.com"],
"viewers": ["data.default.viewers@opendes.contoso.com"],
"owners": [acl_owners],
"viewers": [acl_viewers],
},
"legal": {
"legaltags": ["opendes-public-usa-dataset-7643990"],
"legaltags": [legal_tags],
"otherRelevantDataCountries": ["US"],
},
"data": {
......@@ -124,7 +176,15 @@ def create_reference_data_manifest_json(data_partition: str, kind: str, code: st
return data
def create_master_data_manifest_json(data_partition: str, kind: str, code: str):
def create_master_data_manifest_json(
data_partition: str,
kind: str,
code: str,
authority: str,
acl_owners: str,
acl_viewers: str,
legal_tags: str,
):
"""
Create a manifest json representation
......@@ -134,13 +194,13 @@ def create_master_data_manifest_json(data_partition: str, kind: str, code: str):
"MasterData": [
{
"id": f"{data_partition}:{kind}:{code}",
"kind": f"{data_partition}:wks:{kind}:1.0.0",
"kind": f"{authority}:wks:{kind}:1.0.0",
"acl": {
"owners": ["data.default.owners@opendes.contoso.com"],
"viewers": ["data.default.viewers@opendes.contoso.com"],
"owners": [acl_owners],
"viewers": [acl_viewers],
},
"legal": {
"legaltags": ["opendes-public-usa-dataset-7643990"],
"legaltags": [legal_tags],
"otherRelevantDataCountries": ["US"],
},
"data": {},
......@@ -183,7 +243,17 @@ def write_manifest(kind, name, output_dir, manifest):
json.dump(manifest, outfile, indent=2)
def create_reference_data_manifest(data_partition: str, kind: str, name: str, output_dir: str):
def create_reference_data_manifest(
data_partition: str,
kind: str,
name: str,
authority: str,
acl_owners: str,
acl_viewers: str,
legal_tags: str,
output_dir: str,
# , batch_size: int, batch_across_files: bool
):
"""Create a reference data manifest file
Args:
......@@ -192,11 +262,22 @@ def create_reference_data_manifest(data_partition: str, kind: str, name: str, ou
name (str): [description]
output_dir (str): [description]
"""
manifest = create_reference_data_manifest_json(data_partition, kind, name)
manifest = create_reference_data_manifest_json(
data_partition, kind, name, authority, acl_owners, acl_viewers, legal_tags
)
write_manifest(kind, name, output_dir, manifest)
def create_master_data_manifest(data_partition: str, kind: str, name: str, output_dir: str):
def create_master_data_manifest(
data_partition: str,
kind: str,
name: str,
authority: str,
acl_owners: str,
acl_viewers: str,
legal_tags: str,
output_dir: str,
):
"""Create a master data manifest file
Args:
......@@ -205,7 +286,9 @@ def create_master_data_manifest(data_partition: str, kind: str, name: str, outpu
name (str): [description]
output_dir (str): [description]
"""
manifest = create_master_data_manifest_json(data_partition, kind, name)
manifest = create_master_data_manifest_json(
data_partition, kind, name, authority, acl_owners, acl_viewers, legal_tags
)
write_manifest(kind, name, output_dir, manifest)
......@@ -748,9 +831,15 @@ def get_missing_ids_from_manifest(
return manifest, invalid_entities, missing_ids
def check_references(
state: State, path: str, manifest_output_dir: str # , batch_size: int, batch_across_files: bool
) -> dict: # noqa: C901 pylint: disable=R0912
def check_references( # noqa: C901 pylint: disable=R0912
state: State,
path: str,
manifest_output_dir: str = None,
authority: str = "osdu",
acl_owners: str = None,
acl_viewers: str = None,
legal_tags: str = None, # , batch_size: int, batch_across_files: bool
) -> dict:
"""Find any referenced id's that are missing in OSDU.
Args:
......@@ -760,6 +849,13 @@ def check_references(
Returns:
dict: Response from service
"""
if legal_tags is None:
legal_tags = state.config.get("core", CONFIG_LEGAL_TAG)
if acl_owners is None:
acl_owners = state.config.get("core", CONFIG_ACL_OWNER)
if acl_viewers is None:
acl_viewers = state.config.get("core", CONFIG_ACL_VIEWER)
files = get_files_from_path(path)
logger.debug("Files list: %s", files)
......@@ -795,9 +891,27 @@ def check_references(
# version = parts[3]
if kind.startswith("reference-data"):
create_reference_data_manifest(data_partition, kind, code, manifest_output_dir)
create_reference_data_manifest(
data_partition,
kind,
code,
authority,
acl_owners,
acl_viewers,
legal_tags,
manifest_output_dir,
)
elif kind.startswith("master-data"):
create_master_data_manifest(data_partition, kind, code, manifest_output_dir)
create_master_data_manifest(
data_partition,
kind,
code,
authority,
acl_owners,
acl_viewers,
legal_tags,
manifest_output_dir,
)
else:
logger.warning("Unknown type %s. Not created!", _id)
if manifest_output_dir:
......
......@@ -16,14 +16,13 @@ import click
from osducli.click_cli import CustomClickCommand, State, command_with_output
from osducli.cliclient import handle_cli_exceptions
from osducli.config import CONFIG_ENTITLEMENTS_URL
from osducli.util.service_info import info
from ._const import (
from osducli.commands.entitlements._const import (
ENTITLEMENTS_SERVICE_NAME,
ENTITLEMENTS_STATUS_PATH,
ENTITLEMENTS_SWAGGER_PATH,
)
from osducli.config import CONFIG_ENTITLEMENTS_URL
from osducli.util.service_info import info
# click entry point
......
......@@ -16,11 +16,14 @@ import click
from osducli.click_cli import CustomClickCommand, State, command_with_output
from osducli.cliclient import handle_cli_exceptions
from osducli.commands.legal._const import (
LEGAL_SERVICE_NAME,
LEGAL_STATUS_PATH,
LEGAL_SWAGGER_PATH,
)
from osducli.config import CONFIG_LEGAL_URL
from osducli.util.service_info import info
from ._const import LEGAL_SERVICE_NAME, LEGAL_STATUS_PATH, LEGAL_SWAGGER_PATH
# click entry point
@click.command(cls=CustomClickCommand)
......
......@@ -16,14 +16,13 @@ import click
from osducli.click_cli import CustomClickCommand, State, command_with_output
from osducli.cliclient import handle_cli_exceptions
from osducli.config import CONFIG_SCHEMA_URL
from osducli.util.service_info import info
from osducli.commands.schema._const import (
SCHEMA_SERVICE_NAME,
SCHEMA_STATUS_PATH,
SCHEMA_SWAGGER_PATH,
)
from osducli.config import CONFIG_SCHEMA_URL
from osducli.util.service_info import info
# click entry point
......
......@@ -16,11 +16,14 @@ import click
from osducli.click_cli import CustomClickCommand, State, command_with_output
from osducli.cliclient import handle_cli_exceptions
from osducli.commands.search._const import (
SEARCH_SERVICE_NAME,
SEARCH_STATUS_PATH,
SEARCH_SWAGGER_PATH,
)
from osducli.config import CONFIG_SEARCH_URL
from osducli.util.service_info import info
from ._const import SEARCH_SERVICE_NAME, SEARCH_STATUS_PATH, SEARCH_SWAGGER_PATH
# click entry point
@click.command(cls=CustomClickCommand)
......
......@@ -16,11 +16,14 @@ import click
from osducli.click_cli import CustomClickCommand, State, command_with_output
from osducli.cliclient import handle_cli_exceptions
from osducli.commands.unit._const import (
UNIT_SERVICE_NAME,
UNIT_STATUS_PATH,
UNIT_SWAGGER_PATH,
)
from osducli.config import CONFIG_UNIT_URL
from osducli.util.service_info import info
from ._const import UNIT_SERVICE_NAME, UNIT_STATUS_PATH, UNIT_SWAGGER_PATH
# click entry point
@click.command(cls=CustomClickCommand)
......
......@@ -16,11 +16,14 @@ import click
from osducli.click_cli import CustomClickCommand, State, command_with_output
from osducli.cliclient import handle_cli_exceptions
from osducli.commands.workflow._const import (
WORKFLOW_SERVICE_NAME,
WORKFLOW_STATUS_PATH,
WORKFLOW_SWAGGER_PATH,
)
from osducli.config import CONFIG_WORKFLOW_URL
from osducli.util.service_info import info
from ._const import WORKFLOW_SERVICE_NAME, WORKFLOW_STATUS_PATH, WORKFLOW_SWAGGER_PATH
# click entry point
@click.command(cls=CustomClickCommand)
......
Placeholder file to make sure git checks in this folder
\ No newline at end of file
......@@ -41,7 +41,7 @@ class ServiceInfoTests(unittest.TestCase):
LEGAL_STATUS_PATH,
LEGAL_SWAGGER_PATH,
)
assert result.exit_code == 0
self.assertEqual(result.exit_code, 0)
if __name__ == "__main__":
......
......@@ -41,7 +41,7 @@ class ServiceInfoTests(unittest.TestCase):
SCHEMA_STATUS_PATH,
SCHEMA_SWAGGER_PATH,
)
assert result.exit_code == 0
self.assertEqual(result.exit_code, 0)
if __name__ == "__main__":
......
......@@ -41,7 +41,7 @@ class ServiceInfoTests(unittest.TestCase):
SEARCH_STATUS_PATH,
SEARCH_SWAGGER_PATH,
)
assert result.exit_code == 0
self.assertEqual(result.exit_code, 0)
if __name__ == "__main__":
......
......@@ -37,7 +37,7 @@ class ServiceInfoTests(unittest.TestCase):
mock_info.assert_called_once_with(
mock.ANY, UNIT_SERVICE_NAME, CONFIG_UNIT_URL, UNIT_STATUS_PATH, UNIT_SWAGGER_PATH
)
assert result.exit_code == 0
self.assertEqual(result.exit_code, 0)
if __name__ == "__main__":
......
......@@ -41,7 +41,7 @@ class ServiceInfoTests(unittest.TestCase):
WORKFLOW_STATUS_PATH,
WORKFLOW_SWAGGER_PATH,
)
assert result.exit_code == 0
self.assertEqual(result.exit_code, 0)
if __name__ == "__main__":
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment