Commit 55ba01b4 authored by fabian serin's avatar fabian serin
Browse files

work in progress

parent 7cc2168e
Pipeline #57976 failed with stages
in 4 minutes and 7 seconds
......@@ -12,12 +12,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import List
from fastapi import APIRouter, Depends
from odes_search.models import (
QueryRequest,
CursorQueryResponse,
)
CursorQueryRequest,
BaseModel,
Field,
Optional,
)
from app.clients.search_service_client import get_search_service
from ..common_parameters import REQUIRED_ROLES_READ
from app.utils import Context
......@@ -29,14 +32,16 @@ from .search import (
query_type_returned_fields,
basic_query_request,
basic_query_request_with_cursor)
router = APIRouter()
#osdu kind
# osdu kind
OSDU_WELLBORE_KIND = '*:wks:master-data--Wellbore:*'
OSDU_WELLLOG_KIND = '*:wks:work-product-component--WellLog:*'
OSDU_WELLBOREMARKERSET_KIND = '*:wks:work-product-component--WellboreMarkerSet:*'
WELLBORE_RELATIONSHIP = "WellboreID"
def create_relationships_id_str(data_type: str, id: str):
return f'data.{data_type}:\"{id}\"'
......@@ -49,6 +54,23 @@ def added_query(generated_query: str, user_query: str = None):
return query
def update_query_with_names_based_search(names: str = None, user_query: str = None) -> str:
generated_query = f"data.FacilityName:{names}"
return added_query(generated_query, user_query)
def escape_forbidden_characters_for_search(input_str: str) -> str:
# ? and * are allowed for wildcard search
reserved_char_list = ['+', '-', '=', '>', '<', '!', '(', ')', '{', '}', '[', ']', '^', '"', '~',
':', '\\', '/']
def escape_char(input_char: str, reserved_char_list: [str]) -> str:
return input_char if input_char not in reserved_char_list else f"\\{input_char}"
result_str = ''.join([escape_char(char, reserved_char_list) for char in input_str])
return result_str
def added_relationships_query(id: str, data_type: str, query: str = None):
relationships_id = create_relationships_id_str(data_type, id)
return added_query(relationships_id, query)
......@@ -73,7 +95,7 @@ async def query_request_with_specific_attribute(query_type: str, attribute: str,
return query_result
relationships_ids = [create_relationships_id_str(data_type, r["id"]) for r in response.results]
id_list = ' OR '.join(relationships_ids) # [a, b, c] => 'a OR b OR c'
id_list = ' OR '.join(relationships_ids) # [a, b, c] => 'a OR b OR c'
query = added_query(id_list, query)
......@@ -87,6 +109,108 @@ async def query_request_with_specific_attribute(query_type: str, attribute: str,
query_request=query_request)
class SearchQueryRequest(BaseModel):
# Used by as input, w/o kind, etc...
limit: "Optional[int]" = Field(None, alias="limit")
query: "Optional[str]" = Field(None, alias="query")
cursor: "Optional[str]" = Field(None, alias="cursor")
offset: "Optional[int]" = Field(None, alias="offset")
SearchQueryRequest.update_forward_refs()
DEFAULT_SEARCHQUERYREQUEST = SearchQueryRequest(limit=None, query=None, cursor=None, offset=None)
class SimpleCursorQueryRequest(BaseModel):
# Used by as input, w/o kind, etc...
limit: "Optional[int]" = Field(None, alias="limit")
query: "Optional[str]" = Field(None, alias="query")
cursor: "Optional[str]" = Field(None, alias="cursor")
SimpleCursorQueryRequest.update_forward_refs()
DEFAULT_CURSORQUERYREQUEST = SimpleCursorQueryRequest(limit=None, query=None, cursor=None)
class SimpleOffsetQueryRequest(BaseModel):
limit: "Optional[int]" = Field(None, alias="limit")
query: "Optional[str]" = Field(None, alias="query")
offset: "Optional[int]" = Field(None, alias="offset")
SimpleOffsetQueryRequest.update_forward_refs()
DEFAULT_QUERYREQUEST = SimpleOffsetQueryRequest(limit=None, query=None, offset=None)
async def query_request_with_cursor(query_type: str, kind: str, ctx: Context, query: SimpleCursorQueryRequest = None):
returned_fields = query_type_returned_fields(query_type)
query_request = CursorQueryRequest(kind=kind,
limit=query.limit or 1000,
query=query.query,
returnedFields=[returned_fields],
cursor=query.cursor)
client = await get_search_service(ctx)
return await client.query_with_cursor(
data_partition_id=ctx.partition_id,
cursor_query_request=query_request)
async def query_request_with_offset(query_type: str, kind: str, ctx: Context, query: SimpleOffsetQueryRequest = None):
returned_fields = query_type_returned_fields(query_type)
query_request = QueryRequest(kind=kind,
limit=query.limit or 1000,
query=query.query,
returnedFields=[returned_fields],
offset=query.offset)
client = await get_search_service(ctx)
return await client.query(
data_partition_id=ctx.partition_id,
query_request=query_request)
async def query_request(query_type: str, kind: str, ctx: Context, query: SearchQueryRequest = None):
# use offset if not not none else use cursor
query_as_dict = query.dict(exclude_none=True, exclude_unset=True)
if query.offset is not None:
cursor_query = SimpleOffsetQueryRequest(**query_as_dict)
return await query_request_with_offset(query_type, kind, ctx, cursor_query)
cursor_query = SimpleCursorQueryRequest(**query_as_dict)
return await query_request_with_cursor(query_type, kind, ctx, cursor_query)
@router.post('/query/wellbores/byname', summary='Query with cursor or offset, get wellbores',
description=f"""Get Wellbores object by name. <p>The wellbore kind is {OSDU_WELLBORE_KIND}
returns all records directly based on existing schemas. The query is done on data.FacilityName field</p>{REQUIRED_ROLES_READ}""",
response_model=CursorQueryResponse)
async def query_wellbores_by_name(names: str, body: SearchQueryRequest = DEFAULT_QUERYREQUEST,
ctx: Context = Depends(get_ctx)):
names = escape_forbidden_characters_for_search(names)
body.query = update_query_with_names_based_search(names=names, user_query=body.query)
return await query_request(query_type, OSDU_WELLBORE_KIND, ctx, body)
# @router.post('/querywithcursor/wellbores/byname', summary='Query with cursor, get wellbores',
# description=f"""Get Wellbores object by name. <p>The wellbore kind is {OSDU_WELLBORE_KIND}
# returns all records directly based on existing schemas. The query is done on data.FacilityName field</p>{REQUIRED_ROLES_READ}""",
# response_model=CursorQueryResponse)
# async def query_cursor_wellbores_by_name(names: str, body: SimpleCursorQueryRequest = DEFAULT_CURSORQUERYREQUEST, ctx: Context = Depends(get_ctx)):
# names = escape_forbidden_characters_for_search(names)
# body.query = update_query_with_names_based_search(names=names, user_query=body.query)
# return await query_request_with_cursor(query_type, OSDU_WELLBORE_KIND, ctx, body)
#
#
# @router.post('/querywithoffset/wellbores/byname', summary='Query with offset, get wellbores',
# description=f"""Get Wellbores object by name. <p>The wellbore kind is {OSDU_WELLBORE_KIND}
# returns all records directly based on existing schemas. The query is done on data.FacilityName field</p>{REQUIRED_ROLES_READ}""",
# response_model=CcsQueryResponse)
# async def query_offset_wellbores_by_name(names: str, body: SimpleOffsetQueryRequest = DEFAULT_QUERYREQUEST, ctx: Context = Depends(get_ctx)):
# names = escape_forbidden_characters_for_search(names)
# body.query = update_query_with_names_based_search(names=names, user_query=body.query)
# return await query_request_with_offset(query_type, OSDU_WELLBORE_KIND, ctx, body)
@router.post('/query/wellbores', summary='Query with cursor, get wellbores',
description=f"""Get all Wellbores object. <p>The wellbore kind is {OSDU_WELLBORE_KIND}
returns all records directly based on existing schemas</p>{REQUIRED_ROLES_READ}""",
......@@ -94,13 +218,14 @@ async def query_request_with_specific_attribute(query_type: str, attribute: str,
async def query_wellbores(body: SearchQuery = None, ctx: Context = Depends(get_ctx)):
return await basic_query_request_with_cursor(query_type, OSDU_WELLBORE_KIND, ctx, body.query)
@router.post('/query/wellbores/{wellboreId}/welllogs', summary='Query with cursor, search WellLogs by wellbore ID',
description=f"""Get all WellLogs object using its relationship Wellbore ID. <p>All WellLogs linked to this
specific ID will be returned</p>
<p>The WellLogs kind is {OSDU_WELLLOG_KIND} returns all records directly based on existing schemas</p>{REQUIRED_ROLES_READ}""",
response_model=CursorQueryResponse)
async def query_welllogs_bywellbore(wellboreId: str, body: SearchQuery = None,
ctx: Context = Depends(get_ctx)):
ctx: Context = Depends(get_ctx)):
query = added_relationships_query(wellboreId, WELLBORE_RELATIONSHIP, body.query)
return await basic_query_request(query_type, OSDU_WELLLOG_KIND, ctx, query)
......@@ -112,13 +237,15 @@ async def query_welllogs_bywellbore(wellboreId: str, body: SearchQuery = None,
<p>The WellLogs kind is {OSDU_WELLLOG_KIND} returns all records directly based on existing schemas</p>{REQUIRED_ROLES_READ}""",
response_model=CursorQueryResponse)
async def query_welllogs_bywellboreattribute(wellboreAttribute: str, body: SearchQuery = None,
ctx: Context = Depends(get_ctx)):
return await query_request_with_specific_attribute(query_type, wellboreAttribute, OSDU_WELLBORE_KIND, OSDU_WELLLOG_KIND,
ctx: Context = Depends(get_ctx)):
return await query_request_with_specific_attribute(query_type, wellboreAttribute, OSDU_WELLBORE_KIND,
OSDU_WELLLOG_KIND,
WELLBORE_RELATIONSHIP, ctx,
body.query)
@router.post('/query/wellbores/{wellboreId}/wellboremarkersets', summary='Query with cursor, search wellbore markersets by wellbore ID',
@router.post('/query/wellbores/{wellboreId}/wellboremarkersets',
summary='Query with cursor, search wellbore markersets by wellbore ID',
description=f"""Get all Wellbore Markersets objects using its relationship Wellbore ID. <p>All Markers linked to this
specific ID will be returned</p>
<p>The Wellbore Markerset kind is {OSDU_WELLBOREMARKERSET_KIND} returns all records directly based on existing schemas</p>{REQUIRED_ROLES_READ}""",
......
......@@ -200,8 +200,8 @@ for v3_api, tag in ddms_v3_routes_groups:
wdms_app.include_router(search.router, prefix='/ddms', tags=['search'], dependencies=basic_dependencies)
wdms_app.include_router(fast_search.router, prefix='/ddms', tags=['fast-search'], dependencies=basic_dependencies)
wdms_app.include_router(search_v3.router, prefix='/ddms/v3', tags=['search v3'], dependencies=basic_dependencies)
wdms_app.include_router(fast_search_v3.router, prefix='/ddms/v3', tags=['fast-search v3'],
wdms_app.include_router(search_v3.router, prefix=ALPHA_APIS_PREFIX + DDMS_V3_PATH, tags=['search v3'], dependencies=basic_dependencies)
wdms_app.include_router(fast_search_v3.router, prefix=ALPHA_APIS_PREFIX + DDMS_V3_PATH, tags=['fast-search v3'],
dependencies=basic_dependencies)
alpha_tags = ['ALPHA feature: bulk data chunking']
......
......@@ -51,7 +51,7 @@ def build_request_search_markersets_by_wellbore_id() -> RequestRunner:
rq_proto = Request(
name='search markersets by wellbore id',
method='POST',
url='{{base_url}}/ddms/v3/{{search_query_type}}/wellbores/{{setup_search_osdu_wellbore_id}}/wellboremarkersets',
url='{{base_url}}/alpha/ddms/v3/{{search_query_type}}/wellbores/{{setup_search_osdu_wellbore_id}}/wellboremarkersets',
headers={
'accept': 'application/json',
'data-partition-id': '{{data_partition}}',
......@@ -103,7 +103,7 @@ def build_request_search_osdu_wellbores_by_geo_polygon() -> RequestRunner:
rq_proto = Request(
name='search wellbores by geo polygon',
method='POST',
url='{{base_url}}/ddms/v3/{{search_query_type}}/wellbores/bygeopolygon',
url='{{base_url}}/alpha/ddms/v3/{{search_query_type}}/wellbores/bygeopolygon',
headers={
'accept': 'application/json',
'data-partition-id': '{{data_partition}}',
......@@ -176,7 +176,7 @@ def build_request_search_welllogs_by_wellbores_attribute() -> RequestRunner:
rq_proto = Request(
name='search logset by wellbores attribute',
method='POST',
url='{{base_url}}/ddms/v3/{{search_query_type}}/wellbore/data.DefaultVerticalMeasurementID:"KB"/welllogs',
url='{{base_url}}/alpha/ddms/v3/{{search_query_type}}/wellbore/data.DefaultVerticalMeasurementID:"KB"/welllogs',
headers={
'accept': 'application/json',
'data-partition-id': '{{data_partition}}',
......@@ -224,7 +224,7 @@ def build_request_search_osdu_wellbores_by_bounding_box() -> RequestRunner:
rq_proto = Request(
name='search wellbores by bounding box',
method='POST',
url='{{base_url}}/ddms/v3/{{search_query_type}}/wellbores/byboundingbox?latitude_top_left=-20&longitude_top_left=71&latitude_bottom_right=-22&longitude_bottom_right=73',
url='{{base_url}}/alpha/ddms/v3/{{search_query_type}}/wellbores/byboundingbox?latitude_top_left=-20&longitude_top_left=71&latitude_bottom_right=-22&longitude_bottom_right=73',
headers={
'accept': 'application/json',
'data-partition-id': '{{data_partition}}',
......@@ -256,7 +256,7 @@ def build_request_search_osdu_wellbores_by_distance() -> RequestRunner:
rq_proto = Request(
name='search wellbores by distance',
method='POST',
url='{{base_url}}/ddms/v3/{{search_query_type}}/wellbores/bydistance?latitude=-21.5399&longitude=72.4635&distance=100',
url='{{base_url}}/alpha/ddms/v3/{{search_query_type}}/wellbores/bydistance?latitude=-21.5399&longitude=72.4635&distance=100',
headers={
'accept': 'application/json',
'data-partition-id': '{{data_partition}}',
......@@ -304,7 +304,7 @@ def build_request_search_welllogs_by_wellbore_id() -> RequestRunner:
rq_proto = Request(
name='search logset by wellbore id',
method='POST',
url='{{base_url}}/ddms/v3/{{search_query_type}}/wellbores/{{setup_search_osdu_wellbore_id}}/welllogs',
url='{{base_url}}/alpha/ddms/v3/{{search_query_type}}/wellbores/{{setup_search_osdu_wellbore_id}}/welllogs',
headers={
'accept': 'application/json',
'data-partition-id': '{{data_partition}}',
......@@ -346,4 +346,20 @@ def build_request_search_logs() -> RequestRunner:
},
payload='{ "query": "" }'
)
return RequestRunner(rq_proto)
\ No newline at end of file
return RequestRunner(rq_proto)
def build_request_search_wellbore_by_name() -> RequestRunner:
rq_proto = Request(
name='search wellbore by name',
method='POST',
url='{{base_url}}/alpha/ddms/v3/{{search_query_type}}/wellbores/byname?names=wdms_e2e_search_refs_v%2A',
headers={
'accept': 'application/json',
'data-partition-id': '{{data_partition}}',
'Connection': '{{header_connection}}',
'Authorization': 'Bearer {{token}}',
},
payload='{}'
)
return RequestRunner(rq_proto)
......@@ -94,3 +94,11 @@ def test_search_logset_by_wellbore_attribute(with_wdms_env, set_search_query_typ
assert resobj.totalCount >= 2
@pytest.mark.tag('search')
@pytest.mark.dependency(depends=["test_setup_for_search"])
def test_search_wellbore_by_name(with_wdms_env):
#Only search and no fast search
env = with_wdms_env
env.set('search_query_type', 'query')
resobj = build_request_search_wellbore_by_name().call(with_wdms_env, assert_status=200).get_response_obj()
assert resobj.totalCount >= 1
......@@ -31,3 +31,28 @@ RELATIONSHIPS_QUERY_PARAMS = [
@pytest.mark.parametrize("id, user_query, expected_query", RELATIONSHIPS_QUERY_PARAMS)
def test_added_relationships_query(id, user_query, expected_query):
assert search_v3.added_relationships_query(id, 'WellboreID', user_query) == expected_query
NAMES_QUERY_PARAMS = [
(None, None, 'data.FacilityName:None'),
('Fab OR Fred', None, 'data.FacilityName:Fab OR Fred'),
('Fab', 'data.AnyField:\\"any value\\"', 'data.FacilityName:Fab AND (data.AnyField:\\"any value\\")'),
]
@pytest.mark.parametrize("names, user_query, expected_query", NAMES_QUERY_PARAMS)
def test_update_query_with_names_based_search(names, user_query, expected_query):
assert search_v3.update_query_with_names_based_search(names, user_query) == expected_query
ESCAPE_CHAR_PARAMS = [
('', ''),
('not char to escape', 'not char to escape'),
('wildcard * ? not to escape', 'wildcard * ? not to escape'),
(r'all other to escape +-=><!(){}[]^"~:\ /', r'all other to escape \+\-\=\>\<\!\(\)\{\}\[\]\^\"\~\:\\ \/'),
]
@pytest.mark.parametrize("input_str, expected_str", ESCAPE_CHAR_PARAMS)
def test_escape_forbidden_characters_for_search(input_str, expected_str):
assert search_v3.escape_forbidden_characters_for_search(input_str) == expected_str
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment