Merge branch 'bugfix/GONRG-2292_Find_references_with_no_version_in_Search' into 'master'

GONRG-2292: Find references with no version in Search

See merge request !48

src/dags/libs/validation/validate_referential_integrity.py

@@ -161,7 +161,10 @@ class ManifestIntegrity:
         found_ids = search_handler.search_records()
 
         for entity_id in external_references:
-            if entity_id.srn not in found_ids:
+            # As found_ids contains ids with versions and bare ids, and if entity_id is an id
+            # with no version (refers to the last version), we use just the bare id.
+            entity_srn = entity_id.srn if entity_id.version else entity_id.id
+            if entity_srn not in found_ids:
                 missing_ids.add(entity_id.srn)
 
         return missing_ids

tests/plugin-unit-tests/test_manifest_integrity.py

@@ -17,6 +17,7 @@ import copy
 import json
 import os
 import sys
+from typing import List, Set
 
 sys.path.append(f"{os.getenv('AIRFLOW_SRC_DIR')}/plugins")
 sys.path.append(f"{os.getenv('AIRFLOW_SRC_DIR')}/dags")
@@ -26,18 +27,31 @@ from mock_providers import get_test_credentials
 from file_paths import MANIFEST_WELL_PATH, REF_RESULT_WELL_PATH, MANIFEST_WELLLOG_PATH, \
     REF_RESULT_WELLLOG_PATH, MANIFEST_GENERIC_PATH
 from libs.exceptions import ValidationIntegrityError
-from libs.refresh_token import AirflowTokenRefresher
+from libs.refresh_token import BaseTokenRefresher
 from libs.context import Context
 from libs.search_record_ids import ExtendedSearchId
 from libs.validation.validate_referential_integrity import ManifestIntegrity
+from libs.utils import EntityId, split_id
 
 
 class TestIntegrityProvider:
 
+    @staticmethod
+    def mock_valid_extended_search(monkeypatch, entity_references: List[EntityId]):
+        search_response = set()
+        for entity in entity_references:
+            search_response.add(entity.id)
+            search_response.add(entity.srn)
+        monkeypatch.setattr(
+            ExtendedSearchId,
+            "search_records",
+            lambda *args, **kwargs: search_response
+        )
+
     @pytest.fixture
     def manifest_integrity(self) -> ManifestIntegrity:
         context = Context(app_key="", data_partition_id="test")
-        manifest_integrity = ManifestIntegrity("", AirflowTokenRefresher(get_test_credentials()),
+        manifest_integrity = ManifestIntegrity("", BaseTokenRefresher(get_test_credentials()),
                                                context)
         return manifest_integrity
 
@@ -121,11 +135,7 @@ class TestIntegrityProvider:
         references = manifest_integrity._extract_references(work_product_component)
         manifest_records = manifest_integrity._extract_external_references(
             work_product_component, references)
-        monkeypatch.setattr(
-            ExtendedSearchId,
-            "search_records",
-            lambda *args, **kwargs: [e.srn for e in manifest_records]
-        )
+        self.mock_valid_extended_search(monkeypatch, manifest_records)
         expected_wpc_list = copy.deepcopy(manifest["Data"]["WorkProductComponents"])
         manifest_integrity.ensure_integrity(manifest)
         assert expected_wpc_list == manifest["Data"]["WorkProductComponents"]
@@ -148,11 +158,7 @@ class TestIntegrityProvider:
         references = manifest_integrity._extract_references(work_product_component)
         manifest_records = manifest_integrity._extract_external_references(
             work_product_component, references)
-        monkeypatch.setattr(
-            ExtendedSearchId,
-            "search_records",
-            lambda *args, **kwargs: [mr.srn for mr in manifest_records]
-        )
+        self.mock_valid_extended_search(monkeypatch, manifest_records)
         expected_wpc_list = copy.deepcopy(manifest["Data"]["WorkProductComponents"])
         manifest_integrity.ensure_integrity(manifest)
         assert expected_wpc_list == manifest["Data"]["WorkProductComponents"]
@@ -200,6 +206,71 @@ class TestIntegrityProvider:
             manifest["Data"]["WorkProductComponents"])
         assert wrong_wpc not in valid_wpcs
 
+    @pytest.mark.parametrize(
+        "external_references,search_response,expected_missing_references",
+        [
+            pytest.param(
+                [
+                    "osdu:reference-data--ResourceSecurityClassification:Public:",
+                    "osdu:master-data--Organisation:HESS:",
+                ],
+                set(),
+                [
+                    "osdu:reference-data--ResourceSecurityClassification:Public:",
+                    "osdu:master-data--Organisation:HESS:",
+                ],
+                id="Empty search return"
+            ),
+            pytest.param(
+                [
+                    "osdu:reference-data--ResourceSecurityClassification:Public:",
+                    "osdu:master-data--Organisation:HESS:123",
+                ],
+                {
+                    "osdu:reference-data--ResourceSecurityClassification:Public",
+                    "osdu:reference-data--ResourceSecurityClassification:Public:123",
+                    "osdu:master-data--Organisation:HESS",
+                    "osdu:master-data--Organisation:HESS:123",
+                },
+                set(),
+                id="Full search return"
+            ),
+            pytest.param(
+                [
+                    "osdu:reference-data--ResourceSecurityClassification:Public:",
+                    "osdu:master-data--Organisation:HESS:",
+                ],
+                {
+                    "osdu:reference-data--ResourceSecurityClassification:Public:111",
+                    "osdu:reference-data--ResourceSecurityClassification:Public",
+                },
+                [
+                    "osdu:master-data--Organisation:HESS:",
+                ],
+                id="Partial search return."
+            )
+        ]
+    )
+    def test_find_missing_external_ids(
+        self,
+        monkeypatch,
+        manifest_integrity,
+        external_references: List[str],
+        search_response: set,
+        expected_missing_references: set
+    ):
+        entity_ids = [split_id(r) for r in external_references]
+        monkeypatch.setattr(
+            ExtendedSearchId,
+            "search_records",
+            lambda *args, **kwargs: search_response
+        )
+        missing_ids = manifest_integrity._find_missing_external_ids(entity_ids)
+        assert not missing_ids.symmetric_difference(expected_missing_references), \
+            f'External references {external_references}\n' \
+            f'Search response {search_response}\n' \
+            f'Expected missing ids {expected_missing_references}'
+
     @pytest.mark.parametrize(
         "entity,search_return",
         [
@@ -245,7 +316,7 @@ class TestIntegrityProvider:
                 },
                 {
                     "osdu:master-data--Organisation:HESS:1",
-                    "osdu:master-data--Organisation:HESS:",
+                    "osdu:master-data--Organisation:HESS",
 
                 },
                 id="Search last version"),
@@ -256,7 +327,10 @@ class TestIntegrityProvider:
                         "osdu:master-data--Organisation:HESS:1",
                     ]
                 },
-                {"osdu:master-data--Organisation:HESS:1"},
+                {
+                    "osdu:master-data--Organisation:HESS:1",
+                    "osdu:master-data--Organisation:HESS"
+                },
                 id="Search returns all ids"),
             pytest.param(
                 {
@@ -265,8 +339,10 @@ class TestIntegrityProvider:
                         "osdu:master-data--Organisation:HESS:1:",
                     ]
                 },
-                {"osdu:master-data--Organisation:HESS:1:1",
-                 "osdu:master-data--Organisation:HESS:1:"},
+                {
+                    "osdu:master-data--Organisation:HESS:1:1",
+                    "osdu:master-data--Organisation:HESS:1"
+                },
                 id="Integer part of id of reference is not considered as a version")
         ]
     )