Commit ba7192f5 authored by Mark Hewitt's avatar Mark Hewitt
Browse files

dataload verify supports reference-data {{NAMESPACE}} replacement

parent b28f45e0
Pipeline #74297 passed with stages
in 3 minutes and 12 seconds
......@@ -28,6 +28,11 @@ For more information, specify the `-h` flag:
Change Log
==========
0.0.21
------
- dataload verify supports reference-data {{NAMESPACE}} replacement
0.0.20
------
......@@ -40,6 +45,7 @@ Change Log
- merge dataload checkrefs code (wip)
- user friendly output mode
- improved dataload helper text
- support for python 3.10
0.0.18
------
......
......@@ -12,4 +12,4 @@
""" OSDU command line environment"""
__VERSION__ = "0.0.20"
__VERSION__ = "0.0.21"
......@@ -18,7 +18,7 @@ import click
from osducli.click_cli import CustomClickCommand, State, command_with_output
from osducli.cliclient import CliOsduClient, handle_cli_exceptions
from osducli.config import CONFIG_SEARCH_URL, CLIConfig
from osducli.config import CONFIG_DATA_PARTITION_ID, CONFIG_SEARCH_URL, CLIConfig
from osducli.log import get_logger
from osducli.util.file import get_files_from_path
......@@ -139,10 +139,16 @@ def verify(
for filepath in files:
if filepath.endswith(".json"):
with open(filepath) as file:
data_object = json.load(file)
json_string = file.read()
# for reference data do replacements (acl, legal happens later)
json_string = json_string.replace(
"{{NAMESPACE}}", state.config.get("core", CONFIG_DATA_PARTITION_ID)
)
data_object = json.loads(json_string)
logger.info("Processing file %s.", filepath)
ingested_data = None
if not data_object:
logger.error("Error with file %s. File is empty.", filepath)
......@@ -152,6 +158,7 @@ def verify(
elif "MasterData" in data_object:
ingested_data = data_object["MasterData"]
if ingested_data:
for ingested_datum in ingested_data:
if "id" in ingested_datum:
ids_to_verify.append(ingested_datum.get("id"))
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment