Merge branch 'feature/add_lru_cache_for_schemas' into 'master'

GONRG-2142: Add lru cache

See merge request !46

src/dags/libs/validation/validate_schema.py

@@ -17,18 +17,20 @@
 
 import copy
 import logging
+from functools import lru_cache
 from typing import Any, List, Union, Tuple, Union
 
 import jsonschema
 import requests
 import tenacity
 from jsonschema import exceptions
+
 from libs.constants import DATA_SECTION, DATASETS_SECTION, MASTER_DATA_SECTION, \
     REFERENCE_DATA_SECTION, WORK_PRODUCT_SECTION, WORK_PRODUCT_COMPONENTS_SECTION
 from libs.context import Context
 from libs.exceptions import GenericManifestSchemaError, EntitySchemaValidationError
-from libs.traverse_manifest import ManifestEntity
 from libs.mixins import HeadersMixin
+from libs.traverse_manifest import ManifestEntity
 from osdu_api.libs.auth.authorization import TokenRefresher, authorize
 
 logger = logging.getLogger()
@@ -70,7 +72,8 @@ class OSDURefResolver(jsonschema.RefResolver):
         :rtype: dict
         """
         document = super().resolve_fragment(document, fragment)
-        fragment_parts = fragment.split("/")  # /definitions/<OsduID> -> [..., <OsduID>]
+        # /definitions/<OsduID> -> [..., <OsduID>]
+        fragment_parts = fragment.split("/")
         if len(fragment_parts) > 1:
             document.pop("$id", None)
         return document
@@ -146,9 +149,15 @@ class SchemaValidator(HeadersMixin):
         response["$id"] = uri
         return response
 
+    @lru_cache()
     def get_schema(self, kind: str) -> Union[dict, None]:
         """Fetch schema from Schema service.
 
+        Implies that the cache is a one-off.
+
+        @TODO lru_cache should be replaced by generic cache mechanism to work with providers
+        solutions
+
         :param kind: The kind of the scheema to fetch
         :type kind: str
         :raises e: Generic exception

tests/plugin-unit-tests/test_operators_r3.py

@@ -29,6 +29,7 @@ sys.path.append(f"{os.getenv('AIRFLOW_SRC_DIR')}/dags")
 from libs.exceptions import PipelineFailedError
 import pytest
 import requests
+from functools import lru_cache
 
 from file_paths import (
     MANIFEST_WELLBORE_VALID_PATH,
@@ -80,6 +81,7 @@ class TestOperators(object):
 
     def test_process_manifest_r3_operator(self, monkeypatch):
 
+        @lru_cache()
         def _get_common_schema(*args, **kwargs):
             with open(MANIFEST_GENERIC_SCHEMA_PATH) as f:
                 manifest_schema = json.load(f)