From fe569bcfa799e9793e00793948471d445be06897 Mon Sep 17 00:00:00 2001 From: "DIR\\FSerin" Date: Wed, 18 Aug 2021 17:23:38 +0200 Subject: [PATCH 1/6] Search wellLogs in progress --- app/routers/search/search_v3.py | 4 +- ...arch_v3_wellbore.py => search_v3_alpha.py} | 15 ++++ app/wdms_app.py | 4 +- spec/generated/openapi.json | 84 +++++++++++++++++++ .../wdms/search_apis/search.py | 34 ++++++++ .../functional/tests/test_search_v3.py | 20 +++++ 6 files changed, 157 insertions(+), 4 deletions(-) rename app/routers/search/{search_v3_wellbore.py => search_v3_alpha.py} (72%) diff --git a/app/routers/search/search_v3.py b/app/routers/search/search_v3.py index 45732242..4c4dc106 100644 --- a/app/routers/search/search_v3.py +++ b/app/routers/search/search_v3.py @@ -127,8 +127,8 @@ async def query_request_with_specific_attribute(query_type: str, attribute: str, -def update_query_with_names_based_search(names: str = None, user_query: str = None) -> str: - generated_query = f"data.FacilityName:{names}" +def update_query_with_names_based_search(names: str = None, user_query: str = None, name_field = "data.FacilityName") -> str: + generated_query = f"{name_field}:{names}" return added_query(generated_query, user_query) diff --git a/app/routers/search/search_v3_wellbore.py b/app/routers/search/search_v3_alpha.py similarity index 72% rename from app/routers/search/search_v3_wellbore.py rename to app/routers/search/search_v3_alpha.py index 9bc7d7c0..31d6c870 100644 --- a/app/routers/search/search_v3_wellbore.py +++ b/app/routers/search/search_v3_alpha.py @@ -20,6 +20,7 @@ from .search_v3 import ( DEFAULT_QUERYREQUEST, OSDU_WELLBORE_KIND, OSDU_WELLBORETRAJECTORY_KIND, + OSDU_WELLLOG_KIND, escape_forbidden_characters_for_search, update_query_with_names_based_search, query_request, @@ -53,3 +54,17 @@ async def query_trajectories_bywellbore(wellboreId: str, body: SearchQueryReques ctx: Context = Depends(get_ctx)): body.query = added_relationships_query(wellboreId, WELLBORE_RELATIONSHIP, body.query) return await query_request(query_type, OSDU_WELLBORETRAJECTORY_KIND, ctx, body) + + +@router.post('/query/welllogs/byname', + summary='Query with cursor, search WellLogs by name and optionally by wellbore ID', + description=f"""Get all WellLogs objects using its name and optionally relationship Wellbore ID. +

The WellLogs kind is {OSDU_WELLLOG_KIND} returns all records directly based on existing schemas. The query is done on data.Name field

{REQUIRED_ROLES_READ}""", + response_model=CursorQueryResponse) +async def query_trajectories_bywellbore(names: str, wellbore_id: str = None, body: SearchQueryRequest = DEFAULT_QUERYREQUEST, + ctx: Context = Depends(get_ctx)): + if wellbore_id is not None: + body.query = added_relationships_query(wellbore_id, WELLBORE_RELATIONSHIP, body.query) + names = escape_forbidden_characters_for_search(names) + body.query = update_query_with_names_based_search(names=names, user_query=body.query, name_field="data.Name") + return await query_request(query_type, OSDU_WELLLOG_KIND, ctx, body) diff --git a/app/wdms_app.py b/app/wdms_app.py index c3a43539..f9c91076 100644 --- a/app/wdms_app.py +++ b/app/wdms_app.py @@ -48,7 +48,7 @@ from app.routers.ddms_v3 import ( from app.routers.bulk import bulk_routes from app.routers.trajectory import trajectory_ddms_v2 from app.routers.dipset import dipset_ddms_v2, dip_ddms_v2 -from app.routers.search import search, fast_search, search_v3, fast_search_v3, search_v3_wellbore +from app.routers.search import search, fast_search, search_v3, fast_search_v3, search_v3_alpha from app.clients import StorageRecordServiceClient, SearchServiceClient from app.utils import ( get_http_client_session, @@ -206,7 +206,7 @@ wdms_app.include_router(fast_search.router, prefix='/ddms', tags=['fast-search'] wdms_app.include_router(search_v3.router, prefix=DDMS_V3_PATH, tags=['search v3'], dependencies=basic_dependencies) wdms_app.include_router(fast_search_v3.router, prefix=DDMS_V3_PATH, tags=['fast-search v3'], dependencies=basic_dependencies) -wdms_app.include_router(search_v3_wellbore.router, prefix=ALPHA_APIS_PREFIX+DDMS_V3_PATH, tags=['ALPHA feature: search v3'], +wdms_app.include_router(search_v3_alpha.router, prefix=ALPHA_APIS_PREFIX + DDMS_V3_PATH, tags=['ALPHA feature: search v3'], dependencies=basic_dependencies) diff --git a/spec/generated/openapi.json b/spec/generated/openapi.json index 75cad1f7..1e40d0e9 100644 --- a/spec/generated/openapi.json +++ b/spec/generated/openapi.json @@ -9354,6 +9354,90 @@ ] } }, + "/alpha/ddms/v3/query/welllogs/byname": { + "post": { + "description": "Get all WellLogs objects using its name and optionally relationship Wellbore ID. \n

The WellLogs kind is *:wks:work-product-component--WellLog:* returns all records directly based on existing schemas. The query is done on data.Name field

\n

Required roles: 'users.datalake.viewers' or 'users.datalake.editors' or 'users.datalake.admins'.\n\"In addition, users must be a member of data groups to access the data.

\n", + "operationId": "query_trajectories_bywellbore_alpha_ddms_v3_query_welllogs_byname_post", + "parameters": [ + { + "in": "query", + "name": "names", + "required": true, + "schema": { + "title": "Names", + "type": "string" + } + }, + { + "in": "query", + "name": "wellbore_id", + "required": false, + "schema": { + "title": "Wellbore Id", + "type": "string" + } + }, + { + "description": "identifier of the data partition to query", + "in": "header", + "name": "data-partition-id", + "required": false, + "schema": { + "description": "identifier of the data partition to query", + "minLength": 1, + "title": "data partition id", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/SearchQueryRequest" + } + ], + "default": {}, + "title": "Body" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CursorQueryResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "HTTPBearer": [] + } + ], + "summary": "Query with cursor, search WellLogs by name and optionally by wellbore ID", + "tags": [ + "ALPHA feature: search v3" + ] + } + }, "/ddms/fastquery/logs": { "post": { "description": "Get all Logs object.

The Logs kind is\n *:wks:log:* returns all records IDs directly based on existing schemas

\n

Required roles: 'users.datalake.viewers' or 'users.datalake.editors' or 'users.datalake.admins'.\n\"In addition, users must be a member of data groups to access the data.

\n", diff --git a/tests/integration/functional/request_builders/wdms/search_apis/search.py b/tests/integration/functional/request_builders/wdms/search_apis/search.py index c14b5a84..1b477682 100644 --- a/tests/integration/functional/request_builders/wdms/search_apis/search.py +++ b/tests/integration/functional/request_builders/wdms/search_apis/search.py @@ -379,3 +379,37 @@ def build_request_search_trajectory_by_wellbore_id() -> RequestRunner: payload='{}' ) return RequestRunner(rq_proto) + + +def build_request_search_welllog_by_name() -> RequestRunner: + rq_proto = Request( + name='search wellbore by name', + method='POST', + url='{{base_url}}/alpha/ddms/v3/{{search_query_type}}/welllogs/byname?names=wdms_e2e_search_record_%2A', + headers={ + 'accept': 'application/json', + 'data-partition-id': '{{data_partition}}', + 'Connection': '{{header_connection}}', + 'Authorization': 'Bearer {{token}}', + }, + payload='{}' + ) + return RequestRunner(rq_proto) + + + +def build_request_search_welllog_by_name_and_wellbore() -> RequestRunner: + rq_proto = Request( + name='search wellbore by name', + method='POST', + url='{{base_url}}/alpha/ddms/v3/{{search_query_type}}/welllogs/byname?names=wdms_e2e_search_record_%2A' + '&wellbore_id={{setup_search_osdu_wellbore_id}}', + headers={ + 'accept': 'application/json', + 'data-partition-id': '{{data_partition}}', + 'Connection': '{{header_connection}}', + 'Authorization': 'Bearer {{token}}', + }, + payload='{}' + ) + return RequestRunner(rq_proto) diff --git a/tests/integration/functional/tests/test_search_v3.py b/tests/integration/functional/tests/test_search_v3.py index 6f5983b6..78c66304 100644 --- a/tests/integration/functional/tests/test_search_v3.py +++ b/tests/integration/functional/tests/test_search_v3.py @@ -114,3 +114,23 @@ def test_search_trajectory_by_wellbore_id(with_wdms_env): env.set('search_query_type', 'query') resobj = build_request_search_trajectory_by_wellbore_id().call(with_wdms_env, assert_status=200).get_response_obj() assert resobj.totalCount >= 1 + + +@pytest.mark.tag('search') +@pytest.mark.dependency(depends=["test_setup_for_search"]) +def test_search_wellLog_by_name(with_wdms_env): + #Only search and no fast search + env = with_wdms_env + env.set('search_query_type', 'query') + resobj = build_request_search_welllog_by_name().call(with_wdms_env, assert_status=200).get_response_obj() + assert resobj.totalCount >= 1 + + +@pytest.mark.tag('search') +@pytest.mark.dependency(depends=["test_setup_for_search"]) +def test_search_wellLog_by_name_and_wellbore(with_wdms_env): + #Only search and no fast search + env = with_wdms_env + env.set('search_query_type', 'query') + resobj = build_request_search_welllog_by_name_and_wellbore().call(with_wdms_env, assert_status=200).get_response_obj() + assert resobj.totalCount >= 1 -- GitLab From 39943eb10f75c92cffdc1252f98c33996f8d8351 Mon Sep 17 00:00:00 2001 From: "DIR\\FSerin" Date: Fri, 20 Aug 2021 11:42:36 +0200 Subject: [PATCH 2/6] Search wellLogs --- tests/integration/functional/request_builders/wdms_variables.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/functional/request_builders/wdms_variables.py b/tests/integration/functional/request_builders/wdms_variables.py index 187f6370..28f883c5 100644 --- a/tests/integration/functional/request_builders/wdms_variables.py +++ b/tests/integration/functional/request_builders/wdms_variables.py @@ -30,7 +30,7 @@ variables_dict = { "authorityKind": "{{data_partition}}", "osduWellboreKind": "osdu:wks:master-data--Wellbore:1.0.0", "osduWellKind": "osdu:wks:master-data--Well:1.0.0", - "osduWellLogKind": "osdu:wks:work-product-component--WellLog:1.0.0", + "osduWellLogKind": "osdu:wks:work-product-component--WellLog:1.1.0", "osduWellboreTrajectoryKind": "osdu:wks:work-product-component--WellboreTrajectory:1.1.0", "osduWellboreMarkerSetKind": "osdu:wks:work-product-component--WellboreMarkerSet:1.0.0", "acl_domain": "p4d.cloud.slb-ds.com", -- GitLab From 14d4dede81631a56419cea68cc090aab3b23b173 Mon Sep 17 00:00:00 2001 From: "DIR\\FSerin" Date: Fri, 20 Aug 2021 13:43:43 +0200 Subject: [PATCH 3/6] Updating NOTICE (minor change) --- NOTICE | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/NOTICE b/NOTICE index 2891afec..7112b62a 100644 --- a/NOTICE +++ b/NOTICE @@ -14,7 +14,7 @@ The following software have components provided under the terms of this license: - coverage (from https://coverage.readthedocs.io) - cryptography (from https://github.com/pyca/cryptography) - google-api-core (from https://github.com/GoogleCloudPlatform/google-cloud-python) -- google-auth (from https://github.com/GoogleCloudPlatform/google-auth-library-python) +- google-auth (from https://github.com/googleapis/google-auth-library-python) - google-auth-oauthlib (from https://github.com/GoogleCloudPlatform/google-auth-library-python-oauthlib) - google-cloud-core (from https://github.com/GoogleCloudPlatform/google-cloud-python) - google-cloud-monitoring (from https://github.com/GoogleCloudPlatform/google-cloud-python) @@ -314,7 +314,7 @@ The following software have components provided under the terms of this license: - async-timeout (from https://github.com/aio-libs/async_timeout/) - coverage (from https://coverage.readthedocs.io) - distributed (from https://distributed.readthedocs.io/en/latest/) -- google-auth (from https://github.com/GoogleCloudPlatform/google-auth-library-python) +- google-auth (from https://github.com/googleapis/google-auth-library-python) - google-auth-oauthlib (from https://github.com/GoogleCloudPlatform/google-auth-library-python-oauthlib) - numpy (from http://www.numpy.org) - pandas (from http://pandas.pydata.org) -- GitLab From 9dec8612da142ac5b33839398ae94fb4c3977336 Mon Sep 17 00:00:00 2001 From: "DIR\\FSerin" Date: Fri, 20 Aug 2021 15:30:40 +0200 Subject: [PATCH 4/6] fix name mismatch --- app/routers/search/search_v3_alpha.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/routers/search/search_v3_alpha.py b/app/routers/search/search_v3_alpha.py index 31d6c870..4fe4d387 100644 --- a/app/routers/search/search_v3_alpha.py +++ b/app/routers/search/search_v3_alpha.py @@ -61,7 +61,7 @@ async def query_trajectories_bywellbore(wellboreId: str, body: SearchQueryReques description=f"""Get all WellLogs objects using its name and optionally relationship Wellbore ID.

The WellLogs kind is {OSDU_WELLLOG_KIND} returns all records directly based on existing schemas. The query is done on data.Name field

{REQUIRED_ROLES_READ}""", response_model=CursorQueryResponse) -async def query_trajectories_bywellbore(names: str, wellbore_id: str = None, body: SearchQueryRequest = DEFAULT_QUERYREQUEST, +async def query_welllogs_byname(names: str, wellbore_id: str = None, body: SearchQueryRequest = DEFAULT_QUERYREQUEST, ctx: Context = Depends(get_ctx)): if wellbore_id is not None: body.query = added_relationships_query(wellbore_id, WELLBORE_RELATIONSHIP, body.query) -- GitLab From eebfa5b89cbf2d81de0d94282ebfb2219891b088 Mon Sep 17 00:00:00 2001 From: "DIR\\FSerin" Date: Fri, 20 Aug 2021 15:32:06 +0200 Subject: [PATCH 5/6] updating openapi.json --- spec/generated/openapi.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/generated/openapi.json b/spec/generated/openapi.json index 1e40d0e9..a4778b2c 100644 --- a/spec/generated/openapi.json +++ b/spec/generated/openapi.json @@ -9357,7 +9357,7 @@ "/alpha/ddms/v3/query/welllogs/byname": { "post": { "description": "Get all WellLogs objects using its name and optionally relationship Wellbore ID. \n

The WellLogs kind is *:wks:work-product-component--WellLog:* returns all records directly based on existing schemas. The query is done on data.Name field

\n

Required roles: 'users.datalake.viewers' or 'users.datalake.editors' or 'users.datalake.admins'.\n\"In addition, users must be a member of data groups to access the data.

\n", - "operationId": "query_trajectories_bywellbore_alpha_ddms_v3_query_welllogs_byname_post", + "operationId": "query_welllogs_byname_alpha_ddms_v3_query_welllogs_byname_post", "parameters": [ { "in": "query", -- GitLab From 3eada8d274154f24cb8c4d3f11404b02d6e70379 Mon Sep 17 00:00:00 2001 From: "DIR\\FSerin" Date: Mon, 23 Aug 2021 11:46:59 +0200 Subject: [PATCH 6/6] MR comments : remove the byname in the url --- app/routers/search/search_v3.py | 4 ++++ app/routers/search/search_v3_alpha.py | 8 ++++---- spec/generated/openapi.json | 12 ++++++------ .../request_builders/wdms/search_apis/search.py | 6 +++--- tests/unit/routers/search/search_v3_test.py | 2 +- 5 files changed, 18 insertions(+), 14 deletions(-) diff --git a/app/routers/search/search_v3.py b/app/routers/search/search_v3.py index 4c4dc106..9afda9f0 100644 --- a/app/routers/search/search_v3.py +++ b/app/routers/search/search_v3.py @@ -128,6 +128,8 @@ async def query_request_with_specific_attribute(query_type: str, attribute: str, def update_query_with_names_based_search(names: str = None, user_query: str = None, name_field = "data.FacilityName") -> str: + if names is None: + return user_query generated_query = f"{name_field}:{names}" return added_query(generated_query, user_query) @@ -135,6 +137,8 @@ def update_query_with_names_based_search(names: str = None, user_query: str = No def escape_forbidden_characters_for_search(input_str: str) -> str: # Reserved character are listed here https://community.opengroup.org/osdu/documentation/-/blob/master/platform/tutorials/core-services/SearchService.md # ? and * are allowed for wildcard search + if input_str is None: + return None reserved_char_list = ['+', '-', '=', '>', '<', '!', '(', ')', '{', '}', '[', ']', '^', '"', '~', ':', '\\', '/'] diff --git a/app/routers/search/search_v3_alpha.py b/app/routers/search/search_v3_alpha.py index 4fe4d387..8877b82d 100644 --- a/app/routers/search/search_v3_alpha.py +++ b/app/routers/search/search_v3_alpha.py @@ -33,11 +33,11 @@ from ..common_parameters import REQUIRED_ROLES_READ router = APIRouter() -@router.post('/query/wellbores/byname', summary='Query with cursor or offset, get wellbores', +@router.post('/query/wellbores', summary='Query with cursor or offset, get wellbores', description=f"""Get Wellbores object by name.

The wellbore kind is {OSDU_WELLBORE_KIND} returns all records directly based on existing schemas. The query is done on data.FacilityName field

{REQUIRED_ROLES_READ}""", response_model=CursorQueryResponse) -async def query_wellbores_by_name(names: str, body: SearchQueryRequest = DEFAULT_QUERYREQUEST, +async def query_wellbores_by_name(names: str = None, body: SearchQueryRequest = DEFAULT_QUERYREQUEST, ctx: Context = Depends(get_ctx)): names = escape_forbidden_characters_for_search(names) body.query = update_query_with_names_based_search(names=names, user_query=body.query) @@ -56,12 +56,12 @@ async def query_trajectories_bywellbore(wellboreId: str, body: SearchQueryReques return await query_request(query_type, OSDU_WELLBORETRAJECTORY_KIND, ctx, body) -@router.post('/query/welllogs/byname', +@router.post('/query/welllogs', summary='Query with cursor, search WellLogs by name and optionally by wellbore ID', description=f"""Get all WellLogs objects using its name and optionally relationship Wellbore ID.

The WellLogs kind is {OSDU_WELLLOG_KIND} returns all records directly based on existing schemas. The query is done on data.Name field

{REQUIRED_ROLES_READ}""", response_model=CursorQueryResponse) -async def query_welllogs_byname(names: str, wellbore_id: str = None, body: SearchQueryRequest = DEFAULT_QUERYREQUEST, +async def query_welllogs_by_name(names: str = None, wellbore_id: str = None, body: SearchQueryRequest = DEFAULT_QUERYREQUEST, ctx: Context = Depends(get_ctx)): if wellbore_id is not None: body.query = added_relationships_query(wellbore_id, WELLBORE_RELATIONSHIP, body.query) diff --git a/spec/generated/openapi.json b/spec/generated/openapi.json index a4778b2c..7675e468 100644 --- a/spec/generated/openapi.json +++ b/spec/generated/openapi.json @@ -9204,15 +9204,15 @@ ] } }, - "/alpha/ddms/v3/query/wellbores/byname": { + "/alpha/ddms/v3/query/wellbores": { "post": { "description": "Get Wellbores object by name.

The wellbore kind is *:wks:master-data--Wellbore:*\n returns all records directly based on existing schemas. The query is done on data.FacilityName field

\n

Required roles: 'users.datalake.viewers' or 'users.datalake.editors' or 'users.datalake.admins'.\n\"In addition, users must be a member of data groups to access the data.

\n", - "operationId": "query_wellbores_by_name_alpha_ddms_v3_query_wellbores_byname_post", + "operationId": "query_wellbores_by_name_alpha_ddms_v3_query_wellbores_post", "parameters": [ { "in": "query", "name": "names", - "required": true, + "required": false, "schema": { "title": "Names", "type": "string" @@ -9354,15 +9354,15 @@ ] } }, - "/alpha/ddms/v3/query/welllogs/byname": { + "/alpha/ddms/v3/query/welllogs": { "post": { "description": "Get all WellLogs objects using its name and optionally relationship Wellbore ID. \n

The WellLogs kind is *:wks:work-product-component--WellLog:* returns all records directly based on existing schemas. The query is done on data.Name field

\n

Required roles: 'users.datalake.viewers' or 'users.datalake.editors' or 'users.datalake.admins'.\n\"In addition, users must be a member of data groups to access the data.

\n", - "operationId": "query_welllogs_byname_alpha_ddms_v3_query_welllogs_byname_post", + "operationId": "query_welllogs_by_name_alpha_ddms_v3_query_welllogs_post", "parameters": [ { "in": "query", "name": "names", - "required": true, + "required": false, "schema": { "title": "Names", "type": "string" diff --git a/tests/integration/functional/request_builders/wdms/search_apis/search.py b/tests/integration/functional/request_builders/wdms/search_apis/search.py index 1b477682..2b67842a 100644 --- a/tests/integration/functional/request_builders/wdms/search_apis/search.py +++ b/tests/integration/functional/request_builders/wdms/search_apis/search.py @@ -353,7 +353,7 @@ def build_request_search_wellbore_by_name() -> RequestRunner: rq_proto = Request( name='search wellbore by name', method='POST', - url='{{base_url}}/alpha/ddms/v3/{{search_query_type}}/wellbores/byname?names=wdms_e2e_search_refs_v%2A', + url='{{base_url}}/alpha/ddms/v3/{{search_query_type}}/wellbores?names=wdms_e2e_search_refs_v%2A', headers={ 'accept': 'application/json', 'data-partition-id': '{{data_partition}}', @@ -385,7 +385,7 @@ def build_request_search_welllog_by_name() -> RequestRunner: rq_proto = Request( name='search wellbore by name', method='POST', - url='{{base_url}}/alpha/ddms/v3/{{search_query_type}}/welllogs/byname?names=wdms_e2e_search_record_%2A', + url='{{base_url}}/alpha/ddms/v3/{{search_query_type}}/welllogs?names=wdms_e2e_search_record_%2A', headers={ 'accept': 'application/json', 'data-partition-id': '{{data_partition}}', @@ -402,7 +402,7 @@ def build_request_search_welllog_by_name_and_wellbore() -> RequestRunner: rq_proto = Request( name='search wellbore by name', method='POST', - url='{{base_url}}/alpha/ddms/v3/{{search_query_type}}/welllogs/byname?names=wdms_e2e_search_record_%2A' + url='{{base_url}}/alpha/ddms/v3/{{search_query_type}}/welllogs?names=wdms_e2e_search_record_%2A' '&wellbore_id={{setup_search_osdu_wellbore_id}}', headers={ 'accept': 'application/json', diff --git a/tests/unit/routers/search/search_v3_test.py b/tests/unit/routers/search/search_v3_test.py index 421deb92..d9b0d724 100644 --- a/tests/unit/routers/search/search_v3_test.py +++ b/tests/unit/routers/search/search_v3_test.py @@ -34,7 +34,7 @@ def test_added_relationships_query(id, user_query, expected_query): NAMES_QUERY_PARAMS = [ - (None, None, 'data.FacilityName:None'), + (None, None, None), ('Fab OR Fred', None, 'data.FacilityName:Fab OR Fred'), ('Fab', 'data.AnyField:\\"any value\\"', 'data.FacilityName:Fab AND (data.AnyField:\\"any value\\")'), ] -- GitLab