Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in / Register
Toggle navigation
Menu
Open sidebar
Open Subsurface Data Universe Software
OSDU Community Utilities
Data Loading
osdu-cli
Commits
b5985575
Commit
b5985575
authored
Oct 13, 2021
by
Mark Hewitt
Browse files
improved dataload helper text
parent
0f10ffbe
Pipeline
#71626
passed with stages
in 2 minutes and 28 seconds
Changes
4
Pipelines
2
Hide whitespace changes
Inline
Side-by-side
README.rst
View file @
b5985575
...
...
@@ -33,6 +33,8 @@ Change Log
- schema add --overwrite-existing option
- merge dataload checkrefs code (wip)
- user friendly output mode
- updated dataload helper text
0.0.18
------
...
...
src/osducli/commands/dataload/checkrefs.py
View file @
b5985575
...
...
@@ -44,7 +44,7 @@ WORK_PRODUCT_COMPONENTS_SECTION = "WorkProductComponents"
@
click
.
option
(
"-p"
,
"--path"
,
help
=
"Path to a manifest
or manifest
s to check."
,
help
=
"Path to a manifest
file or file
s to check."
,
type
=
click
.
Path
(
exists
=
True
,
file_okay
=
True
,
dir_okay
=
True
,
readable
=
True
,
resolve_path
=
True
),
required
=
True
,
)
...
...
@@ -66,7 +66,11 @@ WORK_PRODUCT_COMPONENTS_SECTION = "WorkProductComponents"
@
handle_cli_exceptions
@
command_with_output
(
None
)
def
_click_command
(
state
:
State
,
path
:
str
,
generated
:
str
):
"""Find any referenced id's that are missing in OSDU (work in progress)."""
"""Find any referenced id's missing in OSDU (work in progress).
This command will try and extract all referenced id's in the given manifests and then check
whether these exist in OSDU. You might typically use this command prior to ingestion to check
whether ingestion might fail due to the referential integrity check."""
return
check_references
(
state
,
path
,
generated
)
# , path, batch, batch_across_files)
...
...
src/osducli/commands/dataload/ingest.py
View file @
b5985575
...
...
@@ -44,7 +44,7 @@ VERIFY_BATCH_SIZE = 200
@
click
.
option
(
"-p"
,
"--path"
,
help
=
"Path to a file or files to ingest."
,
help
=
"Path to a
manifest
file or files to ingest."
,
type
=
click
.
Path
(
exists
=
True
,
file_okay
=
True
,
dir_okay
=
True
,
readable
=
True
,
resolve_path
=
True
),
required
=
True
,
)
...
...
@@ -88,7 +88,11 @@ def _click_command(
skip_existing
:
str
=
False
,
simulate
:
bool
=
False
,
):
"""Ingest files into OSDU."""
"""Ingest manifest files into OSDU.
This command will take existing manfiest files and load them into OSDU via the workflow
service and Airflow. 'legal' and 'acl' tags will be updated based upon the current
configuration"""
return
ingest
(
state
,
path
,
files
,
batch
,
runid_log
,
wait
,
skip_existing
,
simulate
)
...
...
@@ -102,7 +106,7 @@ def ingest(
skip_existing
:
bool
=
False
,
simulate
:
bool
=
False
,
)
->
dict
:
"""Ingest files into OSDU
"""Ingest
manifest
files into OSDU
Args:
state (State): Global state
...
...
src/osducli/commands/dataload/verify.py
View file @
b5985575
...
...
@@ -30,7 +30,7 @@ logger = get_logger(__name__)
@
click
.
option
(
"-p"
,
"--path"
,
help
=
"Path to a file or files to check."
,
help
=
"Path to a
manifest
file or files to check."
,
type
=
click
.
Path
(
exists
=
True
,
file_okay
=
True
,
dir_okay
=
True
,
readable
=
True
,
resolve_path
=
True
),
required
=
True
,
)
...
...
@@ -47,9 +47,11 @@ logger = get_logger(__name__)
def
_click_command
(
state
:
State
,
path
:
str
,
batch
:
int
=
200
,
batch_across_files
=
True
):
"""Verify if records exist in OSDU.
Note that this doesn't support versioning - success indicates that
a record is found, although there is no check of the contents so it could be an older version if you have
done multiple uploads of the same item with different content."""
This command will check whether id's in the specified manifest files exist in OSDU.
Note that this doesn't support versioning - success indicates that a record's id
is found, however there is no check of the contents so it could be an older version
if you have done multiple uploads of the same item with different content."""
return
verify
(
state
,
path
,
batch
,
batch_across_files
)
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment