Merge branch 'cherry-pick-2c4a39e8' into 'release/0.12'

Merge branch 'fix/Date-time-validation' into 'master'

See merge request !7

osdu_ingestion/libs/constants.py

@@ -34,7 +34,6 @@ SURROGATE_KEYS_PATHS = [
 SEARCH_ID_BATCH_SIZE = 25
 SAVE_RECORDS_BATCH_SIZE = 500
 
-
 DATA_SECTION = "Data"
 DATASETS_SECTION = "Datasets"
 MASTER_DATA_SECTION ="MasterData"

osdu_ingestion/libs/validation/validate_schema.py

@@ -17,12 +17,14 @@
 
 import copy
 import logging
+import re
 from functools import lru_cache
 from typing import Any, List, Tuple, Union
 
 import jsonschema
 import requests
 import tenacity
+from dateutil import parser as date_parser
 from jsonschema import FormatChecker, exceptions
 from osdu_api.auth.authorization import TokenRefresher, authorize
 
@@ -51,6 +53,29 @@ TIMEOUT = 1
 SurrogateKeysPaths = List[Tuple[Union[str, int]]]
 
 
+@FormatChecker.cls_checks("date-time", raises=jsonschema.FormatError)
+def validate_date_time(value: str) -> bool:
+    """
+    Check if a date-time value follows formats specified in indexer:
+    https://community.opengroup.org/osdu/platform/system/indexer-service/-/blob/master/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/parser/DateTimeParserTest.java#L31
+
+    :param value: date-time value
+    :type value: str
+    :return: does date-time value follow Indexer's format.
+    :rtype: bool
+    """
+    try:
+        # Check if a date-time value's format is "EEE MMM dd HH:mm:ss zzz yyyy".
+        # If it is not, assume that value follows ISO 8601 standart. 
+        if re.fullmatch(r"\w{3} \w{3} \d{2} \d{2}:\d{2}:\d{2} \w{3} \d{4}", value):
+            date_parser.parse(value)
+        else:
+            date_parser.isoparse(value)
+        return True
+    except (date_parser.ParserError, ValueError):
+        return False
+
+
 class OSDURefResolver(jsonschema.RefResolver):
     """Extends base jsonschema resolver for OSDU."""
 
@@ -313,9 +338,7 @@ class SchemaValidator(HeadersMixin):
             schema=schema,
             instance=data,
             resolver=resolver,
-            format_checker=FormatChecker(
-                formats=("date-time", "time", "date")
-            )
+            format_checker=FormatChecker()
         )
 
     @staticmethod

osdu_ingestion/tests/libs-unit-tests/test_schema_validator_r3.py

@@ -401,3 +401,83 @@ class TestSchemaValidator:
         with pytest.raises(jsonschema.exceptions.ValidationError) as err:
             schema_validator._validate_against_schema(schema, data)
         assert "is not a \'date-time\'" in str(err)
+
+
+    @pytest.mark.parametrize(
+        "datetime_value",
+        [
+            pytest.param(
+                "2000-01-02 10:10:44",
+                id="yyyy-MM-dd HH:mm:ss"
+            ),
+            pytest.param(
+                "2000-01-02T10:10:44",
+                id="yyyy-MM-ddTHH:mm:ss"
+            ),
+            pytest.param(
+                "2000-01-02 10:10:44.123",
+                id="yyyy-MM-dd HH:mm:ss.SSS"
+            ),
+            pytest.param(
+                "2000-01-02 10:10:44.123000",
+                id="yyyy-MM-dd HH:mm:ss.SSSSSS"
+            ),
+            pytest.param(
+                "20000102",
+                id="yyyyMMdd"
+            ),
+            pytest.param(
+                "2018-11-06T19:37:11.128Z",
+                id="Zulu indicator yyyy-MM-dd'T'HH:mm:ssz"
+            ),
+            pytest.param(
+                "1968-11-01T00:00:00+00:01",
+                id="yyyy-MM-dd HH:mm:ss.SSS+HH:MM"
+            ),
+            pytest.param(
+                "2000-01-02T10:10:44-08:30",
+                id="yyyy-MM-dd'T'HH:mm:ssXXX"
+            ),
+            pytest.param(
+                "Wed Jul 18 10:10:44 PST 2018",
+                id="EEE MMM dd HH:mm:ss zzz yyyy"
+            )
+        ]
+    )
+    def test_validate_valid_date_time_format(self, datetime_value: str):
+        schema = {"properties": {"date-time-data": {"format": "date-time"}}}
+        date_time_data = {"date-time-data": datetime_value}
+        context = Context(app_key="", data_partition_id="")
+        schema_validator = SchemaValidator(
+            "",
+            BaseTokenRefresher(get_test_credentials()),
+            context
+        )
+        schema_validator._validate_against_schema(schema, date_time_data)
+
+
+    @pytest.mark.parametrize(
+        "datetime_value",
+        [
+            pytest.param(
+                "2000-01-aa02 10:10:44",
+            ),
+            pytest.param(
+                "mar 11",
+            ), 
+            pytest.param(
+                "Wd Jul 18 10:10:44 PST 20181",
+            )
+        ]
+    )
+    def test_invalid_date_time_format(self, datetime_value: str):
+        schema = {"properties": {"date-time-data": {"format": "date-time"}}}
+        date_time_data = {"date-time-data": datetime_value}
+        context = Context(app_key="", data_partition_id="")
+        schema_validator = SchemaValidator(
+            "",
+            BaseTokenRefresher(get_test_credentials()),
+            context
+        )
+        with pytest.raises(jsonschema.exceptions.ValidationError) as err:
+            schema_validator._validate_against_schema(schema, date_time_data)

requirements.txt

 jsonschema==3.2.0
 requests==2.25.1
-strict-rfc3339==0.7
 tenacity==6.2.0
 toposort==1.6
+python-dateutil>=2.3, <3
 
 dataclasses==0.8;python_version<"3.7"

setup.py

@@ -52,8 +52,8 @@ setuptools.setup(
     ],
     install_requires=[
         "jsonschema==3.2.0",
+        "python-dateutil>=2.3, <3",
         "pyyaml==5.4.1",
-        "strict-rfc3339==0.7",
         "toposort==1.6",
         "osdu-api>=0.11.0",
         "dataclasses==0.8;python_version<'3.7'"