Merge branch 'feature/checkrefs-authority' into 'main'

Feature/checkrefs authority

Closes #7

See merge request !5

README.rst

@@ -28,6 +28,11 @@ For more information, specify the `-h` flag:
 Change Log
 ==========
 
+0.0.24
+------
+
+- checkrefs authority, acl and legal parameters for generated files
+
 0.0.23
 ------
 

src/osducli/__init__.py

@@ -12,4 +12,4 @@
 
 """ OSDU command line environment"""
 
-__VERSION__ = "0.0.23"
+__VERSION__ = "0.0.24"

src/osducli/commands/dataload/checkrefs.py

@@ -25,7 +25,12 @@ import click
 from osducli.click_cli import CustomClickCommand, State, command_with_output
 from osducli.cliclient import handle_cli_exceptions
 from osducli.commands.dataload.verify import batch_verify
-from osducli.config import CLIConfig
+from osducli.config import (
+    CONFIG_ACL_OWNER,
+    CONFIG_ACL_VIEWER,
+    CONFIG_LEGAL_TAG,
+    CLIConfig,
+)
 from osducli.log import get_logger
 from osducli.util.file import get_files_from_path
 
@@ -55,6 +60,29 @@ WORK_PRODUCT_COMPONENTS_SECTION = "WorkProductComponents"
     type=click.Path(exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True),
     required=False,
 )
+@click.option(
+    "-a",
+    "--authority",
+    help="Schema authority to use when generating manifest files.",
+    default="osdu",
+    show_default=True,
+    required=False,
+)
+@click.option(
+    "-aclo",
+    "--acl-owners",
+    help="Acl owners to use when generating manifest files. If not specified the global value is used.",
+)
+@click.option(
+    "-aclv",
+    "--acl-viewers",
+    help="Acl viewers to use when generating manifest files. If not specified the global value is used.",
+)
+@click.option(
+    "-l",
+    "--legal-tags",
+    help="Legal tag to use when generating manifest files. If not specified the global value is used.",
+)
 # @click.option("-b", "--batch", help="Batch size.", type=int, default=200, show_default=True)
 # @click.option(
 #     "--batch-across-files",
@@ -65,17 +93,41 @@ WORK_PRODUCT_COMPONENTS_SECTION = "WorkProductComponents"
 # )
 @handle_cli_exceptions
 @command_with_output(None)
-def _click_command(state: State, path: str, generated: str):
+def _click_command(
+    state: State,
+    path: str,
+    generated: str,
+    authority: str,
+    acl_owners: str,
+    acl_viewers: str,
+    legal_tags: str,
+):
     """Find any referenced id's missing in OSDU (work in progress).
 
     This command will try and extract all referenced id's in the given manifests and then check
     whether these exist in OSDU. You might typically use this command prior to ingestion to check
     whether ingestion might fail due to the referential integrity check."""
-    return check_references(state, path, generated)  # , path, batch, batch_across_files)
+    return check_references(
+        state,
+        path,
+        generated,
+        authority,
+        acl_owners,
+        acl_viewers,
+        legal_tags,
+    )
 
 
 # region Create Manifests
-def create_reference_data_manifest_json(data_partition: str, kind: str, code: str):
+def create_reference_data_manifest_json(
+    data_partition: str,
+    kind: str,
+    code: str,
+    authority: str,
+    acl_owners: str,
+    acl_viewers: str,
+    legal_tags: str,
+):
     """
     Create a manifest json representation
 
@@ -85,13 +137,13 @@ def create_reference_data_manifest_json(data_partition: str, kind: str, code: st
         "ReferenceData": [
             {
                 "id": f"{data_partition}:{kind}:{code}",
-                "kind": f"{data_partition}:wks:{kind}:1.0.0",
+                "kind": f"{authority}:wks:{kind}:1.0.0",
                 "acl": {
-                    "owners": ["data.default.owners@opendes.contoso.com"],
-                    "viewers": ["data.default.viewers@opendes.contoso.com"],
+                    "owners": [acl_owners],
+                    "viewers": [acl_viewers],
                 },
                 "legal": {
-                    "legaltags": ["opendes-public-usa-dataset-7643990"],
+                    "legaltags": [legal_tags],
                     "otherRelevantDataCountries": ["US"],
                 },
                 "data": {
@@ -124,7 +176,15 @@ def create_reference_data_manifest_json(data_partition: str, kind: str, code: st
     return data
 
 
-def create_master_data_manifest_json(data_partition: str, kind: str, code: str):
+def create_master_data_manifest_json(
+    data_partition: str,
+    kind: str,
+    code: str,
+    authority: str,
+    acl_owners: str,
+    acl_viewers: str,
+    legal_tags: str,
+):
     """
     Create a manifest json representation
 
@@ -134,13 +194,13 @@ def create_master_data_manifest_json(data_partition: str, kind: str, code: str):
         "MasterData": [
             {
                 "id": f"{data_partition}:{kind}:{code}",
-                "kind": f"{data_partition}:wks:{kind}:1.0.0",
+                "kind": f"{authority}:wks:{kind}:1.0.0",
                 "acl": {
-                    "owners": ["data.default.owners@opendes.contoso.com"],
-                    "viewers": ["data.default.viewers@opendes.contoso.com"],
+                    "owners": [acl_owners],
+                    "viewers": [acl_viewers],
                 },
                 "legal": {
-                    "legaltags": ["opendes-public-usa-dataset-7643990"],
+                    "legaltags": [legal_tags],
                     "otherRelevantDataCountries": ["US"],
                 },
                 "data": {},
@@ -183,7 +243,17 @@ def write_manifest(kind, name, output_dir, manifest):
         json.dump(manifest, outfile, indent=2)
 
 
-def create_reference_data_manifest(data_partition: str, kind: str, name: str, output_dir: str):
+def create_reference_data_manifest(
+    data_partition: str,
+    kind: str,
+    name: str,
+    authority: str,
+    acl_owners: str,
+    acl_viewers: str,
+    legal_tags: str,
+    output_dir: str,
+    # , batch_size: int, batch_across_files: bool
+):
     """Create a reference data manifest file
 
     Args:
@@ -192,11 +262,22 @@ def create_reference_data_manifest(data_partition: str, kind: str, name: str, ou
         name (str): [description]
         output_dir (str): [description]
     """
-    manifest = create_reference_data_manifest_json(data_partition, kind, name)
+    manifest = create_reference_data_manifest_json(
+        data_partition, kind, name, authority, acl_owners, acl_viewers, legal_tags
+    )
     write_manifest(kind, name, output_dir, manifest)
 
 
-def create_master_data_manifest(data_partition: str, kind: str, name: str, output_dir: str):
+def create_master_data_manifest(
+    data_partition: str,
+    kind: str,
+    name: str,
+    authority: str,
+    acl_owners: str,
+    acl_viewers: str,
+    legal_tags: str,
+    output_dir: str,
+):
     """Create a master data manifest file
 
     Args:
@@ -205,7 +286,9 @@ def create_master_data_manifest(data_partition: str, kind: str, name: str, outpu
         name (str): [description]
         output_dir (str): [description]
     """
-    manifest = create_master_data_manifest_json(data_partition, kind, name)
+    manifest = create_master_data_manifest_json(
+        data_partition, kind, name, authority, acl_owners, acl_viewers, legal_tags
+    )
     write_manifest(kind, name, output_dir, manifest)
 
 
@@ -748,9 +831,15 @@ def get_missing_ids_from_manifest(
     return manifest, invalid_entities, missing_ids
 
 
-def check_references(
-    state: State, path: str, manifest_output_dir: str  # , batch_size: int, batch_across_files: bool
-) -> dict:  # noqa: C901 pylint: disable=R0912
+def check_references(  # noqa: C901 pylint: disable=R0912
+    state: State,
+    path: str,
+    manifest_output_dir: str = None,
+    authority: str = "osdu",
+    acl_owners: str = None,
+    acl_viewers: str = None,
+    legal_tags: str = None,  # , batch_size: int, batch_across_files: bool
+) -> dict:
     """Find any referenced id's that are missing in OSDU.
 
     Args:
@@ -760,6 +849,13 @@ def check_references(
     Returns:
         dict: Response from service
     """
+    if legal_tags is None:
+        legal_tags = state.config.get("core", CONFIG_LEGAL_TAG)
+    if acl_owners is None:
+        acl_owners = state.config.get("core", CONFIG_ACL_OWNER)
+    if acl_viewers is None:
+        acl_viewers = state.config.get("core", CONFIG_ACL_VIEWER)
+
     files = get_files_from_path(path)
     logger.debug("Files list: %s", files)
 
@@ -795,9 +891,27 @@ def check_references(
                 # version = parts[3]
 
                 if kind.startswith("reference-data"):
-                    create_reference_data_manifest(data_partition, kind, code, manifest_output_dir)
+                    create_reference_data_manifest(
+                        data_partition,
+                        kind,
+                        code,
+                        authority,
+                        acl_owners,
+                        acl_viewers,
+                        legal_tags,
+                        manifest_output_dir,
+                    )
                 elif kind.startswith("master-data"):
-                    create_master_data_manifest(data_partition, kind, code, manifest_output_dir)
+                    create_master_data_manifest(
+                        data_partition,
+                        kind,
+                        code,
+                        authority,
+                        acl_owners,
+                        acl_viewers,
+                        legal_tags,
+                        manifest_output_dir,
+                    )
                 else:
                     logger.warning("Unknown type %s. Not created!", _id)
         if manifest_output_dir:

src/osducli/commands/entitlements/info.py

@@ -16,14 +16,13 @@ import click
 
 from osducli.click_cli import CustomClickCommand, State, command_with_output
 from osducli.cliclient import handle_cli_exceptions
-from osducli.config import CONFIG_ENTITLEMENTS_URL
-from osducli.util.service_info import info
-
-from ._const import (
+from osducli.commands.entitlements._const import (
     ENTITLEMENTS_SERVICE_NAME,
     ENTITLEMENTS_STATUS_PATH,
     ENTITLEMENTS_SWAGGER_PATH,
 )
+from osducli.config import CONFIG_ENTITLEMENTS_URL
+from osducli.util.service_info import info
 
 
 # click entry point

src/osducli/commands/legal/info.py

@@ -16,11 +16,14 @@ import click
 
 from osducli.click_cli import CustomClickCommand, State, command_with_output
 from osducli.cliclient import handle_cli_exceptions
+from osducli.commands.legal._const import (
+    LEGAL_SERVICE_NAME,
+    LEGAL_STATUS_PATH,
+    LEGAL_SWAGGER_PATH,
+)
 from osducli.config import CONFIG_LEGAL_URL
 from osducli.util.service_info import info
 
-from ._const import LEGAL_SERVICE_NAME, LEGAL_STATUS_PATH, LEGAL_SWAGGER_PATH
-
 
 # click entry point
 @click.command(cls=CustomClickCommand)

src/osducli/commands/schema/info.py

@@ -16,14 +16,13 @@ import click
 
 from osducli.click_cli import CustomClickCommand, State, command_with_output
 from osducli.cliclient import handle_cli_exceptions
-from osducli.config import CONFIG_SCHEMA_URL
-from osducli.util.service_info import info
-
 from osducli.commands.schema._const import (
     SCHEMA_SERVICE_NAME,
     SCHEMA_STATUS_PATH,
     SCHEMA_SWAGGER_PATH,
 )
+from osducli.config import CONFIG_SCHEMA_URL
+from osducli.util.service_info import info
 
 
 # click entry point

src/osducli/commands/search/info.py

@@ -16,11 +16,14 @@ import click
 
 from osducli.click_cli import CustomClickCommand, State, command_with_output
 from osducli.cliclient import handle_cli_exceptions
+from osducli.commands.search._const import (
+    SEARCH_SERVICE_NAME,
+    SEARCH_STATUS_PATH,
+    SEARCH_SWAGGER_PATH,
+)
 from osducli.config import CONFIG_SEARCH_URL
 from osducli.util.service_info import info
 
-from ._const import SEARCH_SERVICE_NAME, SEARCH_STATUS_PATH, SEARCH_SWAGGER_PATH
-
 
 # click entry point
 @click.command(cls=CustomClickCommand)

src/osducli/commands/unit/info.py

@@ -16,11 +16,14 @@ import click
 
 from osducli.click_cli import CustomClickCommand, State, command_with_output
 from osducli.cliclient import handle_cli_exceptions
+from osducli.commands.unit._const import (
+    UNIT_SERVICE_NAME,
+    UNIT_STATUS_PATH,
+    UNIT_SWAGGER_PATH,
+)
 from osducli.config import CONFIG_UNIT_URL
 from osducli.util.service_info import info
 
-from ._const import UNIT_SERVICE_NAME, UNIT_STATUS_PATH, UNIT_SWAGGER_PATH
-
 
 # click entry point
 @click.command(cls=CustomClickCommand)

src/osducli/commands/workflow/info.py

@@ -16,11 +16,14 @@ import click
 
 from osducli.click_cli import CustomClickCommand, State, command_with_output
 from osducli.cliclient import handle_cli_exceptions
+from osducli.commands.workflow._const import (
+    WORKFLOW_SERVICE_NAME,
+    WORKFLOW_STATUS_PATH,
+    WORKFLOW_SWAGGER_PATH,
+)
 from osducli.config import CONFIG_WORKFLOW_URL
 from osducli.util.service_info import info
 
-from ._const import WORKFLOW_SERVICE_NAME, WORKFLOW_STATUS_PATH, WORKFLOW_SWAGGER_PATH
-
 
 # click entry point
 @click.command(cls=CustomClickCommand)

tests/commands/dataload/generated/.placeholder

+Placeholder file to make sure git checks in this folder
\ No newline at end of file

tests/commands/legal/test_info.py

@@ -41,7 +41,7 @@ class ServiceInfoTests(unittest.TestCase):
                 LEGAL_STATUS_PATH,
                 LEGAL_SWAGGER_PATH,
             )
-            assert result.exit_code == 0
+            self.assertEqual(result.exit_code, 0)
 
 
 if __name__ == "__main__":

tests/commands/schema/test_info.py

@@ -41,7 +41,7 @@ class ServiceInfoTests(unittest.TestCase):
                 SCHEMA_STATUS_PATH,
                 SCHEMA_SWAGGER_PATH,
             )
-            assert result.exit_code == 0
+            self.assertEqual(result.exit_code, 0)
 
 
 if __name__ == "__main__":

tests/commands/search/test_info.py

@@ -41,7 +41,7 @@ class ServiceInfoTests(unittest.TestCase):
                 SEARCH_STATUS_PATH,
                 SEARCH_SWAGGER_PATH,
             )
-            assert result.exit_code == 0
+            self.assertEqual(result.exit_code, 0)
 
 
 if __name__ == "__main__":

tests/commands/unit/test_info.py

@@ -37,7 +37,7 @@ class ServiceInfoTests(unittest.TestCase):
             mock_info.assert_called_once_with(
                 mock.ANY, UNIT_SERVICE_NAME, CONFIG_UNIT_URL, UNIT_STATUS_PATH, UNIT_SWAGGER_PATH
             )
-            assert result.exit_code == 0
+            self.assertEqual(result.exit_code, 0)
 
 
 if __name__ == "__main__":

tests/commands/workflow/test_info.py

@@ -41,7 +41,7 @@ class ServiceInfoTests(unittest.TestCase):
                 WORKFLOW_STATUS_PATH,
                 WORKFLOW_SWAGGER_PATH,
             )
-            assert result.exit_code == 0
+            self.assertEqual(result.exit_code, 0)
 
 
 if __name__ == "__main__":