Skip to content
Snippets Groups Projects
Commit 204084d1 authored by Yan Sushchynski (EPAM)'s avatar Yan Sushchynski (EPAM)
Browse files

Fix datetime

parent 0c8eadfe
No related tags found
1 merge request!6Fix datetime validation
Pipeline #74800 passed
......@@ -34,7 +34,6 @@ SURROGATE_KEYS_PATHS = [
SEARCH_ID_BATCH_SIZE = 25
SAVE_RECORDS_BATCH_SIZE = 500
DATA_SECTION = "Data"
DATASETS_SECTION = "Datasets"
MASTER_DATA_SECTION ="MasterData"
......
......@@ -17,12 +17,14 @@
import copy
import logging
import re
from functools import lru_cache
from typing import Any, List, Tuple, Union
import jsonschema
import requests
import tenacity
from dateutil import parser as date_parser
from jsonschema import FormatChecker, exceptions
from osdu_api.auth.authorization import TokenRefresher, authorize
......@@ -51,6 +53,29 @@ TIMEOUT = 1
SurrogateKeysPaths = List[Tuple[Union[str, int]]]
@FormatChecker.cls_checks("date-time", raises=jsonschema.FormatError)
def validate_date_time(value: str) -> bool:
"""
Check if a date-time value follows formats specified in indexer:
https://community.opengroup.org/osdu/platform/system/indexer-service/-/blob/master/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/parser/DateTimeParserTest.java#L31
:param value: date-time value
:type value: str
:return: does date-time value follow Indexer's format.
:rtype: bool
"""
try:
# Check if a date-time value's format is "EEE MMM dd HH:mm:ss zzz yyyy".
# If it is not, assume that value follows ISO 8601 standart.
if re.fullmatch(r"\w{3} \w{3} \d{2} \d{2}:\d{2}:\d{2} \w{3} \d{4}", value):
date_parser.parse(value)
else:
date_parser.isoparse(value)
return True
except (date_parser.ParserError, ValueError):
return False
class OSDURefResolver(jsonschema.RefResolver):
"""Extends base jsonschema resolver for OSDU."""
......@@ -313,9 +338,7 @@ class SchemaValidator(HeadersMixin):
schema=schema,
instance=data,
resolver=resolver,
format_checker=FormatChecker(
formats=("date-time", "time", "date")
)
format_checker=FormatChecker()
)
@staticmethod
......
......@@ -401,3 +401,83 @@ class TestSchemaValidator:
with pytest.raises(jsonschema.exceptions.ValidationError) as err:
schema_validator._validate_against_schema(schema, data)
assert "is not a \'date-time\'" in str(err)
@pytest.mark.parametrize(
"datetime_value",
[
pytest.param(
"2000-01-02 10:10:44",
id="yyyy-MM-dd HH:mm:ss"
),
pytest.param(
"2000-01-02T10:10:44",
id="yyyy-MM-ddTHH:mm:ss"
),
pytest.param(
"2000-01-02 10:10:44.123",
id="yyyy-MM-dd HH:mm:ss.SSS"
),
pytest.param(
"2000-01-02 10:10:44.123000",
id="yyyy-MM-dd HH:mm:ss.SSSSSS"
),
pytest.param(
"20000102",
id="yyyyMMdd"
),
pytest.param(
"2018-11-06T19:37:11.128Z",
id="Zulu indicator yyyy-MM-dd'T'HH:mm:ssz"
),
pytest.param(
"1968-11-01T00:00:00+00:01",
id="yyyy-MM-dd HH:mm:ss.SSS+HH:MM"
),
pytest.param(
"2000-01-02T10:10:44-08:30",
id="yyyy-MM-dd'T'HH:mm:ssXXX"
),
pytest.param(
"Wed Jul 18 10:10:44 PST 2018",
id="EEE MMM dd HH:mm:ss zzz yyyy"
)
]
)
def test_validate_valid_date_time_format(self, datetime_value: str):
schema = {"properties": {"date-time-data": {"format": "date-time"}}}
date_time_data = {"date-time-data": datetime_value}
context = Context(app_key="", data_partition_id="")
schema_validator = SchemaValidator(
"",
BaseTokenRefresher(get_test_credentials()),
context
)
schema_validator._validate_against_schema(schema, date_time_data)
@pytest.mark.parametrize(
"datetime_value",
[
pytest.param(
"2000-01-aa02 10:10:44",
),
pytest.param(
"mar 11",
),
pytest.param(
"Wd Jul 18 10:10:44 PST 20181",
)
]
)
def test_invalid_date_time_format(self, datetime_value: str):
schema = {"properties": {"date-time-data": {"format": "date-time"}}}
date_time_data = {"date-time-data": datetime_value}
context = Context(app_key="", data_partition_id="")
schema_validator = SchemaValidator(
"",
BaseTokenRefresher(get_test_credentials()),
context
)
with pytest.raises(jsonschema.exceptions.ValidationError) as err:
schema_validator._validate_against_schema(schema, date_time_data)
jsonschema==3.2.0
requests==2.25.1
strict-rfc3339==0.7
tenacity==6.2.0
toposort==1.6
python-dateutil>=2.3, <3
dataclasses==0.8;python_version<"3.7"
......@@ -52,8 +52,8 @@ setuptools.setup(
],
install_requires=[
"jsonschema==3.2.0",
"python-dateutil>=2.3, <3",
"pyyaml==5.4.1",
"strict-rfc3339==0.7",
"toposort==1.6",
"osdu-api>=0.11.0",
"dataclasses==0.8;python_version<'3.7'"
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment