Merge branch 'trusted-fix/pre-defined-ids-and-autogen-id' into 'master'

Fix for pre-defined Ids and autogenerated Ids

See merge request !11

src/dags/libs/process_manifest_r3.py

@@ -49,7 +49,6 @@ class ManifestProcessor(HeadersMixin):
         "legal": {},
         "acl": {},
         "kind": "",
-        "id": "",
         "data": {}
     }
 
@@ -107,20 +106,6 @@ class ManifestProcessor(HeadersMixin):
             logger.error(f"Unhandled exception while uploading {file_path}: {e}")
         return file_record
 
-    def generate_id(self, manifest_fragment: dict) -> str:
-        """Generate id to use it in Storage.
-
-        :param manifest_fragment: The manifest to update
-        :type manifest_fragment: dict
-        :return: The generated id
-        :rtype: str
-        """
-        group_type = manifest_fragment.get("groupType", "doc")
-        kind = manifest_fragment.get("kind")
-        kind_name = self._get_kind_name(kind)
-        _id = f"{self.context.data_partition_id}:{group_type}_{kind_name}:{str(uuid.uuid4())}"
-        return _id
-
     def populate_manifest_storage_record(self, manifest: dict) -> dict:
         """Create a record from manifest to store it in Storage service.
 
@@ -130,7 +115,8 @@ class ManifestProcessor(HeadersMixin):
         :rtype: dict
         """
         record = copy.deepcopy(self.RECORD_TEMPLATE)
-        record["id"] = manifest["id"] if manifest.get("id") else self.generate_id(manifest)
+        if manifest.get("id"):
+            record["id"] = manifest["id"]
         record["kind"] = manifest.pop("kind")
         record["legal"] = manifest.pop("legal")
         record["acl"] = manifest.pop("acl")
@@ -146,7 +132,8 @@ class ManifestProcessor(HeadersMixin):
         :rtype: dict
         """
         record = copy.deepcopy(self.RECORD_TEMPLATE)
-        record["id"] = manifest["id"] if manifest.get("id") else self.generate_id(manifest)
+        if manifest.get("id"):
+            record["id"] = manifest["id"]
         record["kind"] = manifest.pop("kind")
         record["legal"] = manifest.pop("legal")
         record["acl"] = manifest.pop("acl")

src/dags/libs/traverse_manifest.py

@@ -39,8 +39,10 @@ class ManifestTraversal(object):
         :param schema: corresponding generic schema (for future schema validation)
         :return:
         """
+        extracted_schema = schema.split("/")[-1]
+        logger.debug(f"Extracted schema kind: {extracted_schema}")
         return {
-            "schema": schema,
+            "schema": extracted_schema,
             "entity": entity
         }
 

src/dags/libs/validate_schema.py

@@ -158,18 +158,12 @@ class SchemaValidator(HeadersMixin):
             raise e
         return response
 
-    def _delete_id_pattern(self, schema: dict) -> dict:
-        if schema.get("properties") and schema["properties"].get("id"):
-            schema["properties"]["id"].pop("pattern", None)
-        return schema
-
     def _validate_entity(self, entity: dict, schema: dict = None):
         """
         Validate the 'data' field of any entity against a schema got by entity's kind.
         """
         if not schema:
             schema = self.get_schema(entity["kind"])
-        schema = self._delete_id_pattern(schema)
         data = entity["data"]
         try:
             self._validate_against_schema(schema, data)
@@ -258,7 +252,7 @@ class SchemaValidator(HeadersMixin):
             return True
         except exceptions.ValidationError as exc:
             logger.error("Schema validation error.")
-            logger.error(f"Manifest kind: {schema['kind']}")
+            logger.error(f"Manifest kind: {entity['kind']}")
             logger.error(f"Manifest: {entity}")
             logger.error(f"Error: {exc}")
             return False

tests/plugin-unit-tests/test_manifest_processor_r3.py

@@ -97,7 +97,6 @@ class TestManifestProcessor:
             source_file_checker=source_file_checker,
         )
 
-        monkeypatch.setattr(manifest_processor, "generate_id", lambda manifest: "test_id")
         monkeypatch.setattr(file_handler, "upload_file",
                             lambda *args, **kwargs: "/test/source_file")
         monkeypatch.setattr(file_handler, "save_file_record",
@@ -233,36 +232,3 @@ class TestManifestProcessor:
         for manifest_part in manifest_processor.manifest_records:
             kind = manifest_part["entity"]["kind"]
             assert expected_kind_name == manifest_processor._get_kind_name(kind)
-
-    @pytest.mark.parametrize(
-        "conf_path,expected_kind_name,traversal_manifest_file",
-        [
-            pytest.param(MANIFEST_WELLBORE_VALID_PATH, "Wellbore", TRAVERSAL_WELLBORE_VALID_PATH, id="Valid Wellbore"),
-        ]
-    )
-    def test_generate_id(
-        self,
-        monkeypatch,
-        conf_path: str,
-        traversal_manifest_file: str,
-        expected_kind_name: str
-    ):
-        with open(conf_path) as f:
-            conf = json.load(f)
-        with open(traversal_manifest_file) as f:
-            manifest_file = json.load(f)
-        manifest_records = manifest_file
-        context = process_manifest_r3.Context.populate(conf)
-        manifest_processor = process_manifest_r3.ManifestProcessor(
-            storage_url="",
-            manifest_records=manifest_records,
-            token_refresher=AirflowTokenRefresher(),
-            context=context,
-            file_handler=FileHandler("test", AirflowTokenRefresher(), context),
-            source_file_checker=SourceFileChecker()
-        )
-        for manifest_part in manifest_processor.manifest_records:
-            group_type = manifest_part["entity"]["groupType"]
-            kind_name = manifest_processor._get_kind_name(manifest_part["entity"]["kind"])
-            generated_id = manifest_processor.generate_id(manifest_part["entity"])
-            assert generated_id.startswith(f"{TENANT}:{group_type}_{kind_name}:")