GONRG-782: add support custom acl and legal tag. update tests.

src/plugins/operators/process_manifest_r2.py

@@ -30,8 +30,6 @@ import tenacity
 from airflow.models import BaseOperator, Variable
 from libs.refresh_token import AirflowTokenRefresher, refresh_token
 
-ACL_DICT = eval(Variable.get("acl"))
-LEGAL_DICT = eval(Variable.get("legal"))
 
 config = configparser.RawConfigParser()
 config.read(Variable.get("dataload_config_path"))
@@ -65,12 +63,12 @@ class FileType(enum.Enum):
 
 def dataload(**kwargs):
     data_conf = kwargs['dag_run'].conf
+    conf_payload = kwargs["dag_run"].conf["Payload"]
     loaded_conf = {
-        "acl": ACL_DICT,
-        "legal_tag": LEGAL_DICT,
+        "acl": conf_payload["acl"],
+        "legal_tag": conf_payload["legal"],
         "data_object": data_conf
     }
-    conf_payload = kwargs["dag_run"].conf["Payload"]
     return loaded_conf, conf_payload
 
 

tests/end-to-end-tests/mock-data/osdu-ingest-r2-invalid.json

@@ -108,7 +108,13 @@
     "Payload": {
         "authorization": "Bearer foo",
         "data-partition-id": "common",
-        "AppKey": "test"
+        "AppKey": "test",
+        "acl": {
+            "viewers": ["data.default.viewers"],
+            "owners": ["data.default.owners"]},
+        "legal": {
+            "legaltags": ["odes-demo-legaltag"],
+            "otherRelevantDataCountries": ["FR", "US", "CA"]}
     },
     "WorkflowID": "foo"
 }

tests/end-to-end-tests/mock-data/osdu-ingest-r2-valid.json

@@ -15,6 +15,18 @@
 
 
 {
+    "Payload": {
+        "authorization": "Bearer test",
+        "data-partition-id": "test",
+        "AppKey": "test",
+        "kind_version": "3.0.0",
+        "acl": {
+            "viewers": ["data.default.viewers"],
+            "owners": ["data.default.owners"]},
+        "legal": {
+            "legaltags": ["odes-demo-legaltag"],
+            "otherRelevantDataCountries": ["FR", "US", "CA"]}
+    },
     "WorkProduct": {
         "ResourceTypeID": "srn:type:work-product/WellLog:",
         "ResourceSecurityClassification": "srn:reference-data/ResourceSecurityClassification:RESTRICTED:",
@@ -106,12 +118,6 @@
             ]
         }
     ],
-    "Payload": {
-        "authorization": "Bearer test",
-        "data-partition-id": "test",
-        "AppKey": "test",
-        "kind_version": "3.0.0"
-    },
     "Files": [
         {
             "ResourceTypeID": "srn:type:file/las2:",

tests/plugin-unit-tests/data/process_manifest_r2_op.py

@@ -55,7 +55,13 @@ CONF = {
         "authorization": "Bearer test",
         "data-partition-id": "test",
         "AppKey": "test",
-        "kind_version": "3.0.0"
+        "kind_version": "3.0.0",
+        "acl": {
+            "viewers": ["data.default.viewers@odes.osdu.joonix.net"],
+            "owners": ["data.default.owners@odes.osdu.joonix.net"]},
+        "legal": {
+            "legaltags": ["odes-demo-legaltag"],
+            "otherRelevantDataCountries": ["FR", "US", "CA"]}
     },
     "Files": [
         {
@@ -104,3 +110,13 @@ LOADED_CONF = {
     }
 
 CONF_PAYLOAD = CONF["Payload"]
+
+
+class DAG_RUN:
+    def __init__(self):
+        self.conf = CONF
+
+
+DAG_RUN_CONF = {
+    "dag_run": DAG_RUN()
+}

tests/plugin-unit-tests/test_process_manifest_r2.py

@@ -58,3 +58,15 @@ def test_process_file_items(data_type, loaded_conf, conf_payload, expected_file_
       assert file_id_regex.match(i[0]["data"]["ResourceID"])
       i[0]["data"]["ResourceID"] = ""
    assert file_list == expected_file_list
+
+
+@pytest.mark.parametrize(
+    "dag_run",
+    [
+        (test_data.DAG_RUN_CONF)
+    ]
+)
+def test_dataload_conf(dag_run):
+    loaded_conf, conf_payload = process_manifest_r2.dataload(**dag_run)
+    assert loaded_conf["acl"] == test_data.CONF["Payload"]["acl"]
+    assert loaded_conf["legal_tag"] == test_data.CONF["Payload"]["legal"]