dataload verify supports reference-data {{NAMESPACE}} replacement

README.rst

@@ -28,6 +28,11 @@ For more information, specify the `-h` flag:
 Change Log
 ==========
 
+0.0.21
+------
+
+- dataload verify supports reference-data {{NAMESPACE}} replacement
+
 0.0.20
 ------
 
@@ -40,6 +45,7 @@ Change Log
 - merge dataload checkrefs code (wip)
 - user friendly output mode
 - improved dataload helper text
+- support for python 3.10
 
 0.0.18
 ------

src/osducli/__init__.py

@@ -12,4 +12,4 @@
 
 """ OSDU command line environment"""
 
-__VERSION__ = "0.0.20"
+__VERSION__ = "0.0.21"

src/osducli/commands/dataload/verify.py

@@ -18,7 +18,7 @@ import click
 
 from osducli.click_cli import CustomClickCommand, State, command_with_output
 from osducli.cliclient import CliOsduClient, handle_cli_exceptions
-from osducli.config import CONFIG_SEARCH_URL, CLIConfig
+from osducli.config import CONFIG_DATA_PARTITION_ID, CONFIG_SEARCH_URL, CLIConfig
 from osducli.log import get_logger
 from osducli.util.file import get_files_from_path
 
@@ -139,10 +139,16 @@ def verify(
     for filepath in files:
         if filepath.endswith(".json"):
             with open(filepath) as file:
-                data_object = json.load(file)
+                json_string = file.read()
+                # for reference data do replacements (acl, legal happens later)
+                json_string = json_string.replace(
+                    "{{NAMESPACE}}", state.config.get("core", CONFIG_DATA_PARTITION_ID)
+                )
+                data_object = json.loads(json_string)
 
                 logger.info("Processing file %s.", filepath)
 
+                ingested_data = None
                 if not data_object:
                     logger.error("Error with file %s. File is empty.", filepath)
 
@@ -152,9 +158,10 @@ def verify(
                 elif "MasterData" in data_object:
                     ingested_data = data_object["MasterData"]
 
-                for ingested_datum in ingested_data:
-                    if "id" in ingested_datum:
-                        ids_to_verify.append(ingested_datum.get("id"))
+                if ingested_data:
+                    for ingested_datum in ingested_data:
+                        if "id" in ingested_datum:
+                            ids_to_verify.append(ingested_datum.get("id"))
 
                 batch_verify(
                     state.config, batch_size, ids_to_verify, success, failed, not batch_across_files