From 6574658f6dd71d64eb7d9c41866748e014bc11b4 Mon Sep 17 00:00:00 2001
From: Dmitrii Valuiskii <dmitrii_valuiskii@epam.com>
Date: Mon, 21 Dec 2020 14:12:44 +0000
Subject: [PATCH] GONRG-782: add support custom acl and legal tag. update
tests.
---
src/plugins/operators/process_manifest_r2.py | 8 +++-----
.../mock-data/osdu-ingest-r2-invalid.json | 8 +++++++-
.../mock-data/osdu-ingest-r2-valid.json | 18 ++++++++++++------
.../data/process_manifest_r2_op.py | 18 +++++++++++++++++-
.../test_process_manifest_r2.py | 12 ++++++++++++
5 files changed, 51 insertions(+), 13 deletions(-)
diff --git a/src/plugins/operators/process_manifest_r2.py b/src/plugins/operators/process_manifest_r2.py
index 9a56bdc..d77987c 100644
--- a/src/plugins/operators/process_manifest_r2.py
+++ b/src/plugins/operators/process_manifest_r2.py
@@ -30,8 +30,6 @@ import tenacity
from airflow.models import BaseOperator, Variable
from libs.refresh_token import AirflowTokenRefresher, refresh_token
-ACL_DICT = eval(Variable.get("acl"))
-LEGAL_DICT = eval(Variable.get("legal"))
config = configparser.RawConfigParser()
config.read(Variable.get("dataload_config_path"))
@@ -65,12 +63,12 @@ class FileType(enum.Enum):
def dataload(**kwargs):
data_conf = kwargs['dag_run'].conf
+ conf_payload = kwargs["dag_run"].conf["Payload"]
loaded_conf = {
- "acl": ACL_DICT,
- "legal_tag": LEGAL_DICT,
+ "acl": conf_payload["acl"],
+ "legal_tag": conf_payload["legal"],
"data_object": data_conf
}
- conf_payload = kwargs["dag_run"].conf["Payload"]
return loaded_conf, conf_payload
diff --git a/tests/end-to-end-tests/mock-data/osdu-ingest-r2-invalid.json b/tests/end-to-end-tests/mock-data/osdu-ingest-r2-invalid.json
index 00712d2..9073081 100644
--- a/tests/end-to-end-tests/mock-data/osdu-ingest-r2-invalid.json
+++ b/tests/end-to-end-tests/mock-data/osdu-ingest-r2-invalid.json
@@ -108,7 +108,13 @@
"Payload": {
"authorization": "Bearer foo",
"data-partition-id": "common",
- "AppKey": "test"
+ "AppKey": "test",
+ "acl": {
+ "viewers": ["data.default.viewers"],
+ "owners": ["data.default.owners"]},
+ "legal": {
+ "legaltags": ["odes-demo-legaltag"],
+ "otherRelevantDataCountries": ["FR", "US", "CA"]}
},
"WorkflowID": "foo"
}
diff --git a/tests/end-to-end-tests/mock-data/osdu-ingest-r2-valid.json b/tests/end-to-end-tests/mock-data/osdu-ingest-r2-valid.json
index f8d280c..65cdbeb 100644
--- a/tests/end-to-end-tests/mock-data/osdu-ingest-r2-valid.json
+++ b/tests/end-to-end-tests/mock-data/osdu-ingest-r2-valid.json
@@ -15,6 +15,18 @@
{
+ "Payload": {
+ "authorization": "Bearer test",
+ "data-partition-id": "test",
+ "AppKey": "test",
+ "kind_version": "3.0.0",
+ "acl": {
+ "viewers": ["data.default.viewers"],
+ "owners": ["data.default.owners"]},
+ "legal": {
+ "legaltags": ["odes-demo-legaltag"],
+ "otherRelevantDataCountries": ["FR", "US", "CA"]}
+ },
"WorkProduct": {
"ResourceTypeID": "srn:type:work-product/WellLog:",
"ResourceSecurityClassification": "srn:reference-data/ResourceSecurityClassification:RESTRICTED:",
@@ -106,12 +118,6 @@
]
}
],
- "Payload": {
- "authorization": "Bearer test",
- "data-partition-id": "test",
- "AppKey": "test",
- "kind_version": "3.0.0"
- },
"Files": [
{
"ResourceTypeID": "srn:type:file/las2:",
diff --git a/tests/plugin-unit-tests/data/process_manifest_r2_op.py b/tests/plugin-unit-tests/data/process_manifest_r2_op.py
index 575f5dd..fa248ae 100644
--- a/tests/plugin-unit-tests/data/process_manifest_r2_op.py
+++ b/tests/plugin-unit-tests/data/process_manifest_r2_op.py
@@ -55,7 +55,13 @@ CONF = {
"authorization": "Bearer test",
"data-partition-id": "test",
"AppKey": "test",
- "kind_version": "3.0.0"
+ "kind_version": "3.0.0",
+ "acl": {
+ "viewers": ["data.default.viewers@odes.osdu.joonix.net"],
+ "owners": ["data.default.owners@odes.osdu.joonix.net"]},
+ "legal": {
+ "legaltags": ["odes-demo-legaltag"],
+ "otherRelevantDataCountries": ["FR", "US", "CA"]}
},
"Files": [
{
@@ -104,3 +110,13 @@ LOADED_CONF = {
}
CONF_PAYLOAD = CONF["Payload"]
+
+
+class DAG_RUN:
+ def __init__(self):
+ self.conf = CONF
+
+
+DAG_RUN_CONF = {
+ "dag_run": DAG_RUN()
+}
diff --git a/tests/plugin-unit-tests/test_process_manifest_r2.py b/tests/plugin-unit-tests/test_process_manifest_r2.py
index e794084..690baa5 100644
--- a/tests/plugin-unit-tests/test_process_manifest_r2.py
+++ b/tests/plugin-unit-tests/test_process_manifest_r2.py
@@ -58,3 +58,15 @@ def test_process_file_items(data_type, loaded_conf, conf_payload, expected_file_
assert file_id_regex.match(i[0]["data"]["ResourceID"])
i[0]["data"]["ResourceID"] = ""
assert file_list == expected_file_list
+
+
+@pytest.mark.parametrize(
+ "dag_run",
+ [
+ (test_data.DAG_RUN_CONF)
+ ]
+)
+def test_dataload_conf(dag_run):
+ loaded_conf, conf_payload = process_manifest_r2.dataload(**dag_run)
+ assert loaded_conf["acl"] == test_data.CONF["Payload"]["acl"]
+ assert loaded_conf["legal_tag"] == test_data.CONF["Payload"]["legal"]
--
GitLab