Commit ba7192f5 authored by Mark Hewitt's avatar Mark Hewitt
Browse files

dataload verify supports reference-data {{NAMESPACE}} replacement

parent b28f45e0
Pipeline #74297 passed with stages
in 3 minutes and 12 seconds
...@@ -28,6 +28,11 @@ For more information, specify the `-h` flag: ...@@ -28,6 +28,11 @@ For more information, specify the `-h` flag:
Change Log Change Log
========== ==========
0.0.21
------
- dataload verify supports reference-data {{NAMESPACE}} replacement
0.0.20 0.0.20
------ ------
...@@ -40,6 +45,7 @@ Change Log ...@@ -40,6 +45,7 @@ Change Log
- merge dataload checkrefs code (wip) - merge dataload checkrefs code (wip)
- user friendly output mode - user friendly output mode
- improved dataload helper text - improved dataload helper text
- support for python 3.10
0.0.18 0.0.18
------ ------
......
...@@ -12,4 +12,4 @@ ...@@ -12,4 +12,4 @@
""" OSDU command line environment""" """ OSDU command line environment"""
__VERSION__ = "0.0.20" __VERSION__ = "0.0.21"
...@@ -18,7 +18,7 @@ import click ...@@ -18,7 +18,7 @@ import click
from osducli.click_cli import CustomClickCommand, State, command_with_output from osducli.click_cli import CustomClickCommand, State, command_with_output
from osducli.cliclient import CliOsduClient, handle_cli_exceptions from osducli.cliclient import CliOsduClient, handle_cli_exceptions
from osducli.config import CONFIG_SEARCH_URL, CLIConfig from osducli.config import CONFIG_DATA_PARTITION_ID, CONFIG_SEARCH_URL, CLIConfig
from osducli.log import get_logger from osducli.log import get_logger
from osducli.util.file import get_files_from_path from osducli.util.file import get_files_from_path
...@@ -139,10 +139,16 @@ def verify( ...@@ -139,10 +139,16 @@ def verify(
for filepath in files: for filepath in files:
if filepath.endswith(".json"): if filepath.endswith(".json"):
with open(filepath) as file: with open(filepath) as file:
data_object = json.load(file) json_string = file.read()
# for reference data do replacements (acl, legal happens later)
json_string = json_string.replace(
"{{NAMESPACE}}", state.config.get("core", CONFIG_DATA_PARTITION_ID)
)
data_object = json.loads(json_string)
logger.info("Processing file %s.", filepath) logger.info("Processing file %s.", filepath)
ingested_data = None
if not data_object: if not data_object:
logger.error("Error with file %s. File is empty.", filepath) logger.error("Error with file %s. File is empty.", filepath)
...@@ -152,6 +158,7 @@ def verify( ...@@ -152,6 +158,7 @@ def verify(
elif "MasterData" in data_object: elif "MasterData" in data_object:
ingested_data = data_object["MasterData"] ingested_data = data_object["MasterData"]
if ingested_data:
for ingested_datum in ingested_data: for ingested_datum in ingested_data:
if "id" in ingested_datum: if "id" in ingested_datum:
ids_to_verify.append(ingested_datum.get("id")) ids_to_verify.append(ingested_datum.get("id"))
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment