GONRG-2291: Skipped reason truncated

src/dags/libs/processors/single_manifest_processor.py

@@ -72,7 +72,7 @@ class SingleManifestProcessor:
             )[FIRST_STORED_RECORD_INDEX]
             entity_node.system_srn = record_id
         except Exception as e:
-            raise ProcessRecordError(entity_node.entity_info.entity, f"{e}")
+            raise ProcessRecordError(entity_node.entity_info.entity, f"{e}"[:128])
         return record_id
 
     def _process_records(self, manifest_analyzer: ManifestAnalyzer) -> Tuple[List[str], List[dict]]:

src/dags/libs/utils.py

@@ -63,7 +63,7 @@ def create_skipped_entity_info(entity: Any, reason: str) -> dict:
         skipped_entity_info = {
             "id": entity.get("id", ""),
             "kind": entity.get("kind", ""),
-            "reason": reason[:128]
+            "reason": reason
         }
     else:
         skipped_entity_info = {

src/dags/libs/validation/validate_file_source.py

@@ -88,16 +88,17 @@ class FileSourceValidator:
                 logger.error(
                     f"Rejecting invalid dataset: {dataset.get('id')}, invalid structure. KeyError: {exc}"
                 )
-                raise DatasetValidationError(dataset, f"Invalid structure. KeyError: {exc}")
+                raise DatasetValidationError(dataset, f"Invalid structure. KeyError: {exc}"[:128])
 
         elif is_file_collection:
             try:
                 is_valid_dataset = self._validate_file_collection_record(dataset)
             except KeyError as exc:
                 logger.error(
-                    f"Rejecting invalid dataset: {dataset.get('id')}, invalid structure. KeyError: {exc}"
+                    f"Rejecting invalid dataset: {dataset.get('id')}, "
+                    f"invalid structure. KeyError: {exc}"[:128]
                 )
-                raise DatasetValidationError(dataset, f"Invalid structure. KeyError: {exc}")
+                raise DatasetValidationError(dataset, f"Invalid structure. KeyError: {exc}"[:128])
         else:
             logger.error(f"Unknown file type: {dataset.get('kind')}.")
 

src/dags/libs/validation/validate_referential_integrity.py

@@ -240,7 +240,8 @@ class ManifestIntegrity:
                 f"the WPC's 'Datasets': {duplicated_ids}."
             )
             raise ValidationIntegrityError(wpc,
-                                           reason=f"It has duplicated Datasets: {duplicated_ids}.")
+                                           reason=f"It has duplicated "
+                                                  f"Datasets and Artefacts: {duplicated_ids}.")
 
     def _ensure_artefacts_integrity(self, work_product_components: list) -> Tuple[
         List[dict], List[dict]]:

src/plugins/operators/validate_manifest_schema.py

@@ -23,7 +23,7 @@ from airflow.utils import apply_defaults
 from airflow.models import BaseOperator, Variable
 from libs.constants import DATA_TYPES_WITH_SURROGATE_KEYS, SURROGATE_KEYS_PATHS
 from libs.context import Context
-from libs.exceptions import EmptyManifestError
+from libs.exceptions import EmptyManifestError, GenericManifestSchemaError
 from libs.refresh_token import AirflowTokenRefresher
 from libs.validation.validate_schema import SchemaValidator
 from operators.mixins.ReceivingContextMixin import ReceivingContextMixin
@@ -78,7 +78,13 @@ class ValidateManifestSchemaOperator(BaseOperator, ReceivingContextMixin):
                 f"Data {context['dag_run'].conf} doesn't contain 'manifest field'")
 
         _ = schema_validator.validate_common_schema(manifest_data)
-        valid_manifest_file, skipped_entities = schema_validator.ensure_manifest_validity(manifest_data)
+        try:
+            valid_manifest_file, skipped_entities = schema_validator.ensure_manifest_validity(
+                manifest_data
+            )
+        except GenericManifestSchemaError as err:
+            context["ti"].xcom_push(key="skipped_ids", value=str(err))
+            raise err
         if self._show_skipped_ids:
             context["ti"].xcom_push(key="skipped_ids", value=skipped_entities)
         return {"manifest": valid_manifest_file}