Merge branch 'fix/GONRG-1783_Validate_entire_manifest_entity' into 'integration-master'

GONRG-1783: Valdiate entire manifest entity

See merge request go3-nrg/platform/data-flow/ingestion/ingestion-dags!55

src/dags/libs/validate_schema.py

@@ -110,7 +110,7 @@ class SchemaValidator(HeadersMixin):
         response = requests.get(uri, headers=headers, timeout=60)
         return response
 
-    def __delete_refs(self, schema_part: Union[dict, list]):
+    def _delete_refs(self, schema_part: Union[dict, list]):
         """
         Recursively clear a schema's object parts containing "$ref".
         This method is used by generic manifest validation, deleting these fields make such a
@@ -122,10 +122,10 @@ class SchemaValidator(HeadersMixin):
                 schema_part.clear()
             else:
                 for k in schema_part:
-                    self.__delete_refs(schema_part[k])
+                    self._delete_refs(schema_part[k])
         elif isinstance(schema_part, list):
             for i in schema_part:
-                self.__delete_refs(i)
+                self._delete_refs(i)
 
     def get_schema_request(self, uri: str) -> dict:
         """Get schema from Schema service. Change $id field to url.
@@ -164,9 +164,8 @@ class SchemaValidator(HeadersMixin):
         """
         if not schema:
             schema = self.get_schema(entity["kind"])
-        data = entity["data"]
         try:
-            self._validate_against_schema(schema, data)
+            self._validate_against_schema(schema, entity)
             logger.debug(f"Record successfully validated")
             return True
         except exceptions.ValidationError as exc:
@@ -175,17 +174,6 @@ class SchemaValidator(HeadersMixin):
             logger.error(f"Error: {exc}")
             return False
 
-    def _validate_work_product(self, work_product: dict):
-        """
-        Validate WP manifest. Raise error if manifest is not valid.
-        """
-        for key, value in work_product.items():
-            if key != "WorkProduct":
-                for component in value:
-                    self._validate_entity(component)
-            else:
-                self._validate_entity(value)
-
     def _validate_against_schema(self, schema: dict, data: Any):
         """
         Validate any data against schema.
@@ -203,27 +191,6 @@ class SchemaValidator(HeadersMixin):
         validator = jsonschema.Draft7Validator(schema=schema, resolver=resolver)
         validator.validate(data)
 
-    def _validate_data_group(self, entities: list):
-        """
-        Validate each entity from a list of entities.
-        :param entities:
-        :return:
-        """
-        if isinstance(entities, list):
-            for entity in entities:
-                self._validate_entity(entity)
-
-    def _validate_whole_manifest(self, manifest: dict):
-        """
-        Validate any manifest in general.
-        Also at this step verify that MasterData, ReferenceData, WorkProduct, WorkProductComponents,
-        Files entities correspond their generic schemas, because references to those schemas are in
-        a Manifest schema.
-        """
-        schema = self.get_schema(manifest["kind"])
-        logger.debug(f"Validating kind {manifest['kind']}")
-        self._validate_against_schema(schema, manifest)
-
     def validate_common_schema(self, manifest: dict) -> dict:
         """
         This is a preliminary validation of a manifest that verifies that a manifest corresponds
@@ -237,29 +204,19 @@ class SchemaValidator(HeadersMixin):
         schema = self.get_schema(manifest["manifest"]["kind"])
         schema_without_refs = copy.deepcopy(schema)
         if schema_without_refs.get("properties"):
-            self.__delete_refs(schema_without_refs["properties"])
+            self._delete_refs(schema_without_refs["properties"])
         else:
-            self.__delete_refs(schema_without_refs)
+            self._delete_refs(schema_without_refs)
         logger.debug("Schema without refs")
         logger.debug(f"{schema_without_refs}")
         self._validate_against_schema(schema, manifest)
         return schema
 
-    def _validate_against_generic_schema(self, schema: str, entity: Any) -> bool:
-        try:
-            self._validate_against_schema(schema, entity)
-            logger.debug(f"Record successfully validated against generic schema.")
-            return True
-        except exceptions.ValidationError as exc:
-            logger.error("Schema validation error.")
-            logger.error(f"Manifest kind: {entity['kind']}")
-            logger.error(f"Manifest: {entity}")
-            logger.error(f"Error: {exc}")
-            return False
-
     def validate_manifest(self, manifest_records: List[dict]) -> List[dict]:
         """
-        Validate manifest. Raise error if manifest is not valid.
+        Validate manifest's entities one-by-one. Return list of
+        :param manifest_records: List of manifest's records
+        :return: List of entities passed the validation
         """
         validated_records = []
         if not manifest_records:
@@ -267,9 +224,7 @@ class SchemaValidator(HeadersMixin):
         for manifest_record in manifest_records:
             manifest = manifest_record.get("entity")
             if isinstance(manifest, dict) and manifest.get("kind"):
-                generic_schema = self.get_schema(manifest_record.get("schema"))
-                validation_result = self._validate_against_generic_schema(generic_schema, manifest) \
-                                    and self._validate_entity(manifest)
+                validation_result = self._validate_entity(manifest)
                 if validation_result:
                     validated_records.append(manifest_record)
             else:

tests/set_airflow_env.sh

@@ -19,6 +19,9 @@ pip install jsonschema
 pip install google
 pip install google-cloud-storage
 pip install deepdiff
+pip install azure-identity
+pip install azure-keyvault-secrets
+pip install msal
 export ACL='{"viewers": ["foo"],"owners": ["foo"]}'
 export LEGAL='{"legaltags": ["foo"], "otherRelevantDataCountries": ["FR", "US", "CA"],"status": "compliant"}'
 export WORKFLOW_URL="http://127.0.0.1:5000"