Commit ec88a530 authored by Luc Yriarte's avatar Luc Yriarte
Browse files

Merge branch 'master' into bulk-api-v2

parents 3a436429 1a0814d2
......@@ -260,6 +260,7 @@ The following software have components provided under the terms of this license:
- pytest-httpx (from )
- pytest-mock (from https://github.com/pytest-dev/pytest-mock/)
- python-rapidjson (from https://github.com/python-rapidjson/python-rapidjson)
- python-ulid (from https://github.com/mdomke/python-ulid)
- pytz (from http://pythonhosted.org/pytz)
- requests-oauthlib (from https://github.com/requests/requests-oauthlib)
- six (from http://pypi.python.org/pypi/six/)
......
......@@ -13,7 +13,7 @@
# limitations under the License.
import uuid
from typing import Tuple
from typing import Tuple, Optional
class BulkId:
......@@ -30,7 +30,9 @@ class BulkId:
# Returns a tuple (<uuid> : str, <prefix> : str)
@classmethod
def bulk_urn_decode(cls, urn: str) -> Tuple[str, str]:
def bulk_urn_decode(cls, urn: str) -> Tuple[str, Optional[str]]:
if urn is None:
raise ValueError('attempted to decode empty urn')
parts = urn.split(":")
if len(parts) < 4:
return str(uuid.UUID(urn)), None
......
......@@ -31,7 +31,7 @@ class WithLifeTime:
:param builder_fn: input factory function/callable that builds an object.
:return: returns a callable that returns an object
"""
raise NotImplemented()
raise NotImplementedError()
class Transient(Base):
""" Transient lifetime: a new object is created each time """
......
This diff is collapsed.
......@@ -20,7 +20,7 @@ from .search_v3 import (
SearchQuery,
query_type,
basic_query_request,
added_query,
added_relationships_query,
query_request_with_specific_attribute,
OSDU_WELLBORE_KIND,
OSDU_WELLLOG_KIND,
......@@ -52,7 +52,7 @@ async def fastquery_wellbores(body: SearchQuery = None, ctx: Context = Depends(g
response_model=CursorQueryResponse)
async def fastquery_welllogs_bywellbore(wellbore_id: str, body: SearchQuery = None,
ctx: Context = Depends(get_ctx)):
query = added_query(wellbore_id, WELLBORE_RELATIONSHIP, body.query)
query = added_relationships_query(wellbore_id, WELLBORE_RELATIONSHIP, body.query)
return await basic_query_request(query_type, OSDU_WELLLOG_KIND, ctx, query)
......@@ -77,5 +77,5 @@ async def fastquery_welllogs_bywellboreattribute(wellbore_attribute: str, body:
response_model=CursorQueryResponse)
async def fastquery_markers_bywellbore(wellbore_id: str, body: SearchQuery = None,
ctx: Context = Depends(get_ctx)):
query = added_query(wellbore_id, WELLBORE_RELATIONSHIP, body.query)
query = added_relationships_query(wellbore_id, WELLBORE_RELATIONSHIP, body.query)
return await basic_query_request(query_type, OSDU_WELLBOREMARKERSET_KIND, ctx, query)
This diff is collapsed.
......@@ -361,6 +361,19 @@ def build_request_create_osdu_welllog(b_use_fixed_id=True) -> RequestRunner:
"DrillingFluidProperty": "Example DrillingFluidProperty",
"HoleTypeLogging": "OPENHOLE",
"VerticalMeasurementID": "Example VerticalMeasurementID",
"VerticalMeasurement": {
"EffectiveDateTime": "2020-02-13T09:13:15.550000+00:00",
"VerticalMeasurement": 12345.6,
"TerminationDateTime": "2020-02-13T09:13:15.550000+00:00",
"VerticalMeasurementTypeID": "namespace:reference-data--VerticalMeasurementType:Plug%20Back%20depth:",
"VerticalMeasurementPathID": "namespace:reference-data--VerticalMeasurementPath:SomeUniqueVerticalMeasurementPathID:",
"VerticalMeasurementSourceID": "namespace:reference-data--VerticalMeasurementSource:SomeUniqueVerticalMeasurementSourceID:",
"WellboreTVDTrajectoryID": "namespace:work-product-component--WellboreTrajectory:WellboreTrajectory-911bb71f-06ab-4deb-8e68-b8c9229dc76b:",
"VerticalMeasurementUnitOfMeasureID": "namespace:reference-data--UnitOfMeasure:m:",
"VerticalCRSID": "namespace:reference-data--CoordinateReferenceSystem:SomeUniqueCoordinateReferenceSystemID:",
"VerticalReferenceID": "Example VerticalReferenceID",
"VerticalMeasurementDescription": "Example VerticalMeasurementDescription"
},
"Curves": [
{
"CurveID": "Example CurveID",
......@@ -380,20 +393,7 @@ def build_request_create_osdu_welllog(b_use_fixed_id=True) -> RequestRunner:
"LogCurveTypeID": "namespace:reference-data--LogCurveType:SomeUniqueLogCurveTypeID:",
"LogCurveBusinessValueID": "namespace:reference-data--LogCurveBusinessValue:SomeUniqueLogCurveBusinessValueID:",
"LogCurveMainFamilyID": "namespace:reference-data--LogCurveMainFamily:SomeUniqueLogCurveMainFamilyID:",
"LogCurveFamilyID": "namespace:reference-data--LogCurveFamily:SomeUniqueLogCurveFamilyID:",
"VerticalMeasurement": {
"EffectiveDateTime": "2020-02-13T09:13:15.550000+00:00",
"VerticalMeasurement": 12345.6,
"TerminationDateTime": "2020-02-13T09:13:15.550000+00:00",
"VerticalMeasurementTypeID": "namespace:reference-data--VerticalMeasurementType:Plug%20Back%20depth:",
"VerticalMeasurementPathID": "namespace:reference-data--VerticalMeasurementPath:SomeUniqueVerticalMeasurementPathID:",
"VerticalMeasurementSourceID": "namespace:reference-data--VerticalMeasurementSource:SomeUniqueVerticalMeasurementSourceID:",
"WellboreTVDTrajectoryID": "namespace:work-product-component--WellboreTrajectory:WellboreTrajectory-911bb71f-06ab-4deb-8e68-b8c9229dc76b:",
"VerticalMeasurementUnitOfMeasureID": "namespace:reference-data--UnitOfMeasure:m:",
"VerticalCRSID": "namespace:reference-data--CoordinateReferenceSystem:SomeUniqueCoordinateReferenceSystemID:",
"VerticalReferenceID": "Example VerticalReferenceID",
"VerticalMeasurementDescription": "Example VerticalMeasurementDescription"
}
"LogCurveFamilyID": "namespace:reference-data--LogCurveFamily:SomeUniqueLogCurveFamilyID:"
}
],
"ExtensionProperties": {}
......@@ -689,6 +689,19 @@ def get_cleaned_ref_and_res(res: dict) -> (dict, dict):
"DrillingFluidProperty": "Example DrillingFluidProperty",
"HoleTypeLogging": "OPENHOLE",
"VerticalMeasurementID": "Example VerticalMeasurementID",
"VerticalMeasurement": {
"EffectiveDateTime": "2020-02-13T09:13:15.550000+00:00",
"VerticalMeasurement": 12345.6,
"TerminationDateTime": "2020-02-13T09:13:15.550000+00:00",
"VerticalMeasurementTypeID": "namespace:reference-data--VerticalMeasurementType:Plug%20Back%20depth:",
"VerticalMeasurementPathID": "namespace:reference-data--VerticalMeasurementPath:SomeUniqueVerticalMeasurementPathID:",
"VerticalMeasurementSourceID": "namespace:reference-data--VerticalMeasurementSource:SomeUniqueVerticalMeasurementSourceID:",
"WellboreTVDTrajectoryID": "namespace:work-product-component--WellboreTrajectory:WellboreTrajectory-911bb71f-06ab-4deb-8e68-b8c9229dc76b:",
"VerticalMeasurementUnitOfMeasureID": "namespace:reference-data--UnitOfMeasure:m:",
"VerticalCRSID": "namespace:reference-data--CoordinateReferenceSystem:SomeUniqueCoordinateReferenceSystemID:",
"VerticalReferenceID": "Example VerticalReferenceID",
"VerticalMeasurementDescription": "Example VerticalMeasurementDescription"
},
"Curves": [
{
"CurveID": "Example CurveID",
......@@ -708,20 +721,7 @@ def get_cleaned_ref_and_res(res: dict) -> (dict, dict):
"LogCurveTypeID": "namespace:reference-data--LogCurveType:SomeUniqueLogCurveTypeID:",
"LogCurveBusinessValueID": "namespace:reference-data--LogCurveBusinessValue:SomeUniqueLogCurveBusinessValueID:",
"LogCurveMainFamilyID": "namespace:reference-data--LogCurveMainFamily:SomeUniqueLogCurveMainFamilyID:",
"LogCurveFamilyID": "namespace:reference-data--LogCurveFamily:SomeUniqueLogCurveFamilyID:",
"VerticalMeasurement": {
"EffectiveDateTime": "2020-02-13T09:13:15.550000+00:00",
"VerticalMeasurement": 12345.6,
"TerminationDateTime": "2020-02-13T09:13:15.550000+00:00",
"VerticalMeasurementTypeID": "namespace:reference-data--VerticalMeasurementType:Plug%20Back%20depth:",
"VerticalMeasurementPathID": "namespace:reference-data--VerticalMeasurementPath:SomeUniqueVerticalMeasurementPathID:",
"VerticalMeasurementSourceID": "namespace:reference-data--VerticalMeasurementSource:SomeUniqueVerticalMeasurementSourceID:",
"WellboreTVDTrajectoryID": "namespace:work-product-component--WellboreTrajectory:WellboreTrajectory-911bb71f-06ab-4deb-8e68-b8c9229dc76b:",
"VerticalMeasurementUnitOfMeasureID": "namespace:reference-data--UnitOfMeasure:m:",
"VerticalCRSID": "namespace:reference-data--CoordinateReferenceSystem:SomeUniqueCoordinateReferenceSystemID:",
"VerticalReferenceID": "Example VerticalReferenceID",
"VerticalMeasurementDescription": "Example VerticalMeasurementDescription"
}
"LogCurveFamilyID": "namespace:reference-data--LogCurveFamily:SomeUniqueLogCurveFamilyID:"
}
],
"ExtensionProperties": {}
......
......@@ -17,7 +17,6 @@ from enum import Enum
from contextlib import contextmanager
import random
import httpx
import numpy as np
import numpy.testing as npt
import pandas as pd
......@@ -381,3 +380,83 @@ def test_get_data_with_limit_and_offset_filter(with_wdms_env, entity_type):
if r.ok:
pd.testing.assert_frame_equal(expected_data, serializer.read(r.response.content))
@pytest.mark.tag('chunking', 'smoke')
@pytest.mark.parametrize('entity_type', [EntityType.well_log, EntityType.wellbore_trajectory])
@pytest.mark.parametrize('serializer', [ParquetSerializer(), JsonSerializer()])
def test_multiple_overwrite_sessions_in_parallel_then_commit(with_wdms_env, entity_type, serializer):
with create_record(with_wdms_env, entity_type) as record_id:
# create session
sessions = [{
'id': create_session(with_wdms_env, entity_type, record_id, True),
'df': generate_df(['MD', 'X'], range(8))
} for _i in range(5)] # mode overwrite
for session in sessions:
session_id = session['id']
expected = session['df']
# post chunk
build_request_post_chunk(entity_type, record_id, session_id, serializer.dump(expected)).call(
with_wdms_env, headers={'Content-Type': serializer.mime_type},).assert_ok()
random.shuffle(sessions)
for session in sessions:
session_id = session['id']
expected = session['df']
# commit session
complete_session(with_wdms_env, entity_type, record_id, session_id, True) # commit
# then read and check expected
result = build_request_get_data(entity_type, record_id).call(
with_wdms_env, headers={'Accept': serializer.mime_type}, assert_status=200)
pd.testing.assert_frame_equal(
expected, serializer.read(result.response.content), check_dtype=False)
# check type set to false since in Json dType is lost so int32 can become int64
@pytest.mark.tag('chunking', 'smoke')
@pytest.mark.parametrize('entity_type', [EntityType.well_log, EntityType.wellbore_trajectory])
@pytest.mark.parametrize('serializer', [ParquetSerializer(), JsonSerializer()])
def test_multiple_update_sessions_in_parallel_then_commit(with_wdms_env, entity_type, serializer):
with create_record(with_wdms_env, entity_type) as record_id:
# post data
data = generate_df(['MD', 'X'], range(10))
data_to_send = serializer.dump(data)
build_request_post_data(entity_type, record_id, data_to_send).call(
with_wdms_env, headers={'Content-Type': serializer.mime_type}).assert_ok()
# create session
sessions = [{
'id': create_session(with_wdms_env, entity_type, record_id, False),
'df': generate_df(['MD', 'X'], range(10, 20))
} for _i in range(5)] # mode overwrite
for session in sessions:
session_id = session['id']
expected = session['df']
# post chunk
build_request_post_chunk(entity_type, record_id, session_id, serializer.dump(expected)).call(
with_wdms_env, headers={'Content-Type': serializer.mime_type},).assert_ok()
random.shuffle(sessions)
for session in sessions:
session_id = session['id']
expected = pd.concat([data, session['df']])
# commit session
complete_session(with_wdms_env, entity_type, record_id, session_id, True) # commit
# then read and check expected
result = build_request_get_data(entity_type, record_id).call(
with_wdms_env, headers={'Accept': serializer.mime_type}, assert_status=200)
pd.testing.assert_frame_equal(
expected, serializer.read(result.response.content), check_dtype=False)
# check type set to false since in Json dType is lost so int32 can become int64
......@@ -11,6 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import pytest
from app.bulk_persistence import BulkId
import uuid
......@@ -19,3 +20,17 @@ import uuid
def test_bulk_id_is_an_uuid():
uuid.UUID(BulkId.new_bulk_id())
# urn decode test
def test_decode_urn_no_prefix():
uuid, prefix = BulkId.bulk_urn_decode("urn:uuid:489768d2-eee1-4a8f-ae95-7b0c30b0dcd8")
assert uuid == "489768d2-eee1-4a8f-ae95-7b0c30b0dcd8"
assert prefix is None
def test_decode_urn_with_prefix():
uuid, prefix = BulkId.bulk_urn_decode("urn:myprefix:uuid:489768d2-eee1-4a8f-ae95-7b0c30b0dcd8")
assert uuid == "489768d2-eee1-4a8f-ae95-7b0c30b0dcd8"
assert prefix == 'myprefix'
def test_decode_urn_none():
with pytest.raises(ValueError):
uuid, prefix = BulkId.bulk_urn_decode(None)
......@@ -22,6 +22,7 @@ from fastapi.testclient import TestClient
from fastapi import Header, status
from odes_storage.models import CreateUpdateRecordsResponse, Record
from app.model.osdu_model import Wellbore, Well
from app.clients import SearchServiceClient, StorageRecordServiceClient
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment