Commit 886d99eb authored by Luc Yriarte's avatar Luc Yriarte
Browse files

Merge branch 'master' into basic-apis-v2

parents ad4c763d 0933cca7
Pipeline #58884 passed with stages
in 46 minutes and 39 seconds
......@@ -162,7 +162,6 @@ osdu-gcp-test-python:
stage: integration
image: gcr.io/google.com/cloudsdktool/cloud-sdk
needs: ['osdu-gcp-deploy']
allow_failure: true
only:
variables:
- $OSDU_GCP == 'true' && $OSDU_GCP_INT_TEST_TYPE == 'python'
......@@ -193,9 +192,6 @@ osdu-gcp-test-python:
# Allow failure on deployments
osdu-gcp-deploy:
allow_failure: true
ibm-deploy:
allow_failure: true
......@@ -203,6 +199,3 @@ ibm-deploy:
ibm-test:
allow_failure: true
aws-test-python:
allow_failure: true
......@@ -12,12 +12,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import List
from fastapi import APIRouter, Depends
from odes_search.models import (
QueryRequest,
CursorQueryResponse,
)
CursorQueryResponse)
from app.clients.search_service_client import get_search_service
from ..common_parameters import REQUIRED_ROLES_READ
from app.utils import Context
......@@ -29,14 +27,16 @@ from .search import (
query_type_returned_fields,
basic_query_request,
basic_query_request_with_cursor)
router = APIRouter()
#osdu kind
# osdu kind
OSDU_WELLBORE_KIND = '*:wks:master-data--Wellbore:*'
OSDU_WELLLOG_KIND = '*:wks:work-product-component--WellLog:*'
OSDU_WELLBOREMARKERSET_KIND = '*:wks:work-product-component--WellboreMarkerSet:*'
WELLBORE_RELATIONSHIP = "WellboreID"
def create_relationships_id_str(data_type: str, id: str):
return f'data.{data_type}:\"{id}\"'
......@@ -73,7 +73,7 @@ async def query_request_with_specific_attribute(query_type: str, attribute: str,
return query_result
relationships_ids = [create_relationships_id_str(data_type, r["id"]) for r in response.results]
id_list = ' OR '.join(relationships_ids) # [a, b, c] => 'a OR b OR c'
id_list = ' OR '.join(relationships_ids) # [a, b, c] => 'a OR b OR c'
query = added_query(id_list, query)
......@@ -94,13 +94,14 @@ async def query_request_with_specific_attribute(query_type: str, attribute: str,
async def query_wellbores(body: SearchQuery = None, ctx: Context = Depends(get_ctx)):
return await basic_query_request_with_cursor(query_type, OSDU_WELLBORE_KIND, ctx, body.query)
@router.post('/query/wellbores/{wellboreId}/welllogs', summary='Query with cursor, search WellLogs by wellbore ID',
description=f"""Get all WellLogs object using its relationship Wellbore ID. <p>All WellLogs linked to this
specific ID will be returned</p>
<p>The WellLogs kind is {OSDU_WELLLOG_KIND} returns all records directly based on existing schemas</p>{REQUIRED_ROLES_READ}""",
response_model=CursorQueryResponse)
async def query_welllogs_bywellbore(wellboreId: str, body: SearchQuery = None,
ctx: Context = Depends(get_ctx)):
ctx: Context = Depends(get_ctx)):
query = added_relationships_query(wellboreId, WELLBORE_RELATIONSHIP, body.query)
return await basic_query_request(query_type, OSDU_WELLLOG_KIND, ctx, query)
......@@ -112,13 +113,15 @@ async def query_welllogs_bywellbore(wellboreId: str, body: SearchQuery = None,
<p>The WellLogs kind is {OSDU_WELLLOG_KIND} returns all records directly based on existing schemas</p>{REQUIRED_ROLES_READ}""",
response_model=CursorQueryResponse)
async def query_welllogs_bywellboreattribute(wellboreAttribute: str, body: SearchQuery = None,
ctx: Context = Depends(get_ctx)):
return await query_request_with_specific_attribute(query_type, wellboreAttribute, OSDU_WELLBORE_KIND, OSDU_WELLLOG_KIND,
ctx: Context = Depends(get_ctx)):
return await query_request_with_specific_attribute(query_type, wellboreAttribute, OSDU_WELLBORE_KIND,
OSDU_WELLLOG_KIND,
WELLBORE_RELATIONSHIP, ctx,
body.query)
@router.post('/query/wellbores/{wellboreId}/wellboremarkersets', summary='Query with cursor, search wellbore markersets by wellbore ID',
@router.post('/query/wellbores/{wellboreId}/wellboremarkersets',
summary='Query with cursor, search wellbore markersets by wellbore ID',
description=f"""Get all Wellbore Markersets objects using its relationship Wellbore ID. <p>All Markers linked to this
specific ID will be returned</p>
<p>The Wellbore Markerset kind is {OSDU_WELLBOREMARKERSET_KIND} returns all records directly based on existing schemas</p>{REQUIRED_ROLES_READ}""",
......
# Copyright 2021 Schlumberger
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from fastapi import APIRouter, Depends
from odes_search.models import (
QueryRequest,
CursorQueryResponse,
CursorQueryRequest,
BaseModel,
Field,
Optional)
from app.clients.search_service_client import get_search_service
from app.utils import Context
from .search_v3 import (
added_query,
create_relationships_id_str,
query_type_returned_fields,
OSDU_WELLBORE_KIND,
get_ctx,
query_type)
from ..common_parameters import REQUIRED_ROLES_READ
router = APIRouter()
def update_query_with_names_based_search(names: str = None, user_query: str = None) -> str:
generated_query = f"data.FacilityName:{names}"
return added_query(generated_query, user_query)
def escape_forbidden_characters_for_search(input_str: str) -> str:
# Reserved character are listed here https://community.opengroup.org/osdu/documentation/-/blob/master/platform/tutorials/core-services/SearchService.md
# ? and * are allowed for wildcard search
reserved_char_list = ['+', '-', '=', '>', '<', '!', '(', ')', '{', '}', '[', ']', '^', '"', '~',
':', '\\', '/']
def escape_char(input_char: str, reserved_char_list: [str]) -> str:
return input_char if input_char not in reserved_char_list else f"\\{input_char}"
result_str = ''.join([escape_char(char, reserved_char_list) for char in input_str])
return result_str
def added_relationships_query(id: str, data_type: str, query: str = None):
relationships_id = create_relationships_id_str(data_type, id)
return added_query(relationships_id, query)
class SearchQueryRequest(BaseModel):
# Used by as input, w/o kind, etc...
limit: "Optional[int]" = Field(None, alias="limit")
query: "Optional[str]" = Field(None, alias="query")
cursor: "Optional[str]" = Field(None, alias="cursor")
offset: "Optional[int]" = Field(None, alias="offset")
SearchQueryRequest.update_forward_refs()
DEFAULT_SEARCHQUERYREQUEST = SearchQueryRequest(limit=None, query=None, cursor=None, offset=None)
class SimpleCursorQueryRequest(BaseModel):
# Used by as input, w/o kind, etc...
limit: "Optional[int]" = Field(None, alias="limit")
query: "Optional[str]" = Field(None, alias="query")
cursor: "Optional[str]" = Field(None, alias="cursor")
SimpleCursorQueryRequest.update_forward_refs()
DEFAULT_CURSORQUERYREQUEST = SimpleCursorQueryRequest(limit=None, query=None, cursor=None)
class SimpleOffsetQueryRequest(BaseModel):
limit: "Optional[int]" = Field(None, alias="limit")
query: "Optional[str]" = Field(None, alias="query")
offset: "Optional[int]" = Field(None, alias="offset")
SimpleOffsetQueryRequest.update_forward_refs()
DEFAULT_QUERYREQUEST = SimpleOffsetQueryRequest(limit=None, query=None, offset=None)
async def query_request_with_cursor(query_type: str, kind: str, ctx: Context, query: SimpleCursorQueryRequest = None):
returned_fields = query_type_returned_fields(query_type)
query_request = CursorQueryRequest(kind=kind,
limit=query.limit or 1000,
query=query.query,
returnedFields=[returned_fields],
cursor=query.cursor)
client = await get_search_service(ctx)
return await client.query_with_cursor(
data_partition_id=ctx.partition_id,
cursor_query_request=query_request)
async def query_request_with_offset(query_type: str, kind: str, ctx: Context, query: SimpleOffsetQueryRequest = None):
returned_fields = query_type_returned_fields(query_type)
query_request = QueryRequest(kind=kind,
limit=query.limit or 1000,
query=query.query,
returnedFields=[returned_fields],
offset=query.offset)
client = await get_search_service(ctx)
return await client.query(
data_partition_id=ctx.partition_id,
query_request=query_request)
async def query_request(query_type: str, kind: str, ctx: Context, query: SearchQueryRequest = None):
# use offset if not not none else use cursor
query_as_dict = query.dict(exclude_none=True, exclude_unset=True)
if query.offset is not None:
cursor_query = SimpleOffsetQueryRequest(**query_as_dict)
return await query_request_with_offset(query_type, kind, ctx, cursor_query)
cursor_query = SimpleCursorQueryRequest(**query_as_dict)
return await query_request_with_cursor(query_type, kind, ctx, cursor_query)
@router.post('/query/wellbores/byname', summary='Query with cursor or offset, get wellbores',
description=f"""Get Wellbores object by name. <p>The wellbore kind is {OSDU_WELLBORE_KIND}
returns all records directly based on existing schemas. The query is done on data.FacilityName field</p>{REQUIRED_ROLES_READ}""",
response_model=CursorQueryResponse)
async def query_wellbores_by_name(names: str, body: SearchQueryRequest = DEFAULT_QUERYREQUEST,
ctx: Context = Depends(get_ctx)):
names = escape_forbidden_characters_for_search(names)
body.query = update_query_with_names_based_search(names=names, user_query=body.query)
return await query_request(query_type, OSDU_WELLBORE_KIND, ctx, body)
......@@ -48,7 +48,7 @@ from app.routers.ddms_v3 import (
from app.routers.bulk import bulk_routes
from app.routers.trajectory import trajectory_ddms_v2
from app.routers.dipset import dipset_ddms_v2, dip_ddms_v2
from app.routers.search import search, fast_search, search_v3, fast_search_v3
from app.routers.search import search, fast_search, search_v3, fast_search_v3, search_v3_wellbore
from app.clients import StorageRecordServiceClient, SearchServiceClient
from app.utils import (
get_http_client_session,
......@@ -203,9 +203,12 @@ for v3_api, tag in ddms_v3_routes_groups:
wdms_app.include_router(search.router, prefix='/ddms', tags=['search'], dependencies=basic_dependencies)
wdms_app.include_router(fast_search.router, prefix='/ddms', tags=['fast-search'], dependencies=basic_dependencies)
wdms_app.include_router(search_v3.router, prefix='/ddms/v3', tags=['search v3'], dependencies=basic_dependencies)
wdms_app.include_router(fast_search_v3.router, prefix='/ddms/v3', tags=['fast-search v3'],
wdms_app.include_router(search_v3.router, prefix=DDMS_V3_PATH, tags=['search v3'], dependencies=basic_dependencies)
wdms_app.include_router(fast_search_v3.router, prefix=DDMS_V3_PATH, tags=['fast-search v3'],
dependencies=basic_dependencies)
wdms_app.include_router(search_v3_wellbore.router, prefix=ALPHA_APIS_PREFIX+DDMS_V3_PATH, tags=['ALPHA feature: search v3'],
dependencies=basic_dependencies)
alpha_tags = ['ALPHA feature: bulk data chunking']
v3_bulk_dependencies = [*basic_dependencies, Depends(set_v3_input_dataframe_check), Depends(set_osdu_bulk_id_access)]
......
......@@ -135,7 +135,6 @@ azure_test_py:
- source venv/bin/activate
- pip install --upgrade pip
- pip install wheel pytest pytest-cov
- pip install -r frozenrequirements.txt
- pip install -r requirements.txt
- pip install -r requirements_dev.txt
- svctoken=$(python devops/scripts/azure_jwt_client.py)
......
......@@ -19,10 +19,9 @@ kind: Ingress
metadata:
annotations:
{{ include "os-wellbore-ddms.commonAnnotations" . | indent 4}}
appgw.ingress.kubernetes.io/ssl-redirect: "true"
appgw.ingress.kubernetes.io/connection-draining: "true"
appgw.ingress.kubernetes.io/connection-draining-timeout: "30"
kubernetes.io/ingress.class: azure/application-gateway
{{- if .Values.ingress.extraAnnotations }}
{{ toYaml .Values.ingress.extraAnnotations | indent 4 }}
{{- end }}
{{ if .Values.ingress.hosts.host }}
cert-manager.io/acme-challenge-type: http01
cert-manager.io/cluster-issuer: letsencrypt
......@@ -49,4 +48,4 @@ spec:
hosts:
- {{ .Values.ingress.hosts.host }}
{{ end }}
{{- end -}}
\ No newline at end of file
{{- end -}}
......@@ -28,9 +28,9 @@ replicaCount: 2
resources:
limits:
memory: 3Gi
memory: 2Gi
requests:
memory: 3Gi
memory: 2Gi
annotations:
buildNumber: []
......@@ -65,10 +65,16 @@ configMap:
ingress:
enabled: false
extraAnnotations:
appgw.ingress.kubernetes.io/ssl-redirect: "true"
appgw.ingress.kubernetes.io/request-timeout: "300"
appgw.ingress.kubernetes.io/connection-draining: "true"
appgw.ingress.kubernetes.io/connection-draining-timeout: "30"
kubernetes.io/ingress.class: azure/application-gateway
hosts:
host: # leave it empty for our dev cluster
pathPrefix: /api/os-wellbore-ddms
tlsSecret: osdu-certificate
authorizationPolicy:
name: wellbore-jwt-authz
\ No newline at end of file
name: wellbore-jwt-authz
......@@ -41,7 +41,6 @@ steps:
pip install virtualenv
virtualenv venv
source venv/bin/activate
pip install -r frozenrequirements.txt
pip install -r ./requirements.txt
pip install -r ./requirements_dev.txt
......
......@@ -44,7 +44,6 @@ phases:
- chmod +x ./tests/aws-test/build-aws/prepare-dist.sh
- ./tests/aws-test/build-aws/prepare-dist.sh
- pip install -r ./frozenrequirements.txt
- pip install -r ./requirements.txt
- pip install -r ./requirements_dev.txt
......
......@@ -2875,6 +2875,28 @@
"title": "SearchQuery",
"type": "object"
},
"SearchQueryRequest": {
"properties": {
"cursor": {
"title": "Cursor",
"type": "string"
},
"limit": {
"title": "Limit",
"type": "integer"
},
"offset": {
"title": "Offset",
"type": "integer"
},
"query": {
"title": "Query",
"type": "string"
}
},
"title": "SearchQueryRequest",
"type": "object"
},
"Session": {
"description": "model of session exposed ",
"example": {
......@@ -9182,6 +9204,81 @@
]
}
},
"/alpha/ddms/v3/query/wellbores/byname": {
"post": {
"description": "Get Wellbores object by name. <p>The wellbore kind is *:wks:master-data--Wellbore:*\n returns all records directly based on existing schemas. The query is done on data.FacilityName field</p>\n<p>Required roles: 'users.datalake.viewers' or 'users.datalake.editors' or 'users.datalake.admins'.\n\"In addition, users must be a member of data groups to access the data.</p>\n",
"operationId": "query_wellbores_by_name_alpha_ddms_v3_query_wellbores_byname_post",
"parameters": [
{
"in": "query",
"name": "names",
"required": true,
"schema": {
"title": "Names",
"type": "string"
}
},
{
"description": "identifier of the data partition to query",
"in": "header",
"name": "data-partition-id",
"required": false,
"schema": {
"description": "identifier of the data partition to query",
"minLength": 1,
"title": "data partition id",
"type": "string"
}
}
],
"requestBody": {
"content": {
"application/json": {
"schema": {
"allOf": [
{
"$ref": "#/components/schemas/SearchQueryRequest"
}
],
"default": {},
"title": "Body"
}
}
}
},
"responses": {
"200": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/CursorQueryResponse"
}
}
},
"description": "Successful Response"
},
"422": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/HTTPValidationError"
}
}
},
"description": "Validation Error"
}
},
"security": [
{
"HTTPBearer": []
}
],
"summary": "Query with cursor or offset, get wellbores",
"tags": [
"ALPHA feature: search v3"
]
}
},
"/ddms/fastquery/logs": {
"post": {
"description": "Get all Logs object. <p>The Logs kind is\n *:wks:log:* returns all records IDs directly based on existing schemas</p>\n<p>Required roles: 'users.datalake.viewers' or 'users.datalake.editors' or 'users.datalake.admins'.\n\"In addition, users must be a member of data groups to access the data.</p>\n",
......
......@@ -24,11 +24,9 @@ INTEGRATION_TEST_OUTPUT_DIR=${INTEGRATION_TEST_OUTPUT_DIR:-$OUTPUT_DIR}/testing
rm -rf "$INTEGRATION_TEST_OUTPUT_DIR"
mkdir -p "$INTEGRATION_TEST_OUTPUT_DIR"
if [ ! -e frozenrequirements.txt ]; then
echo "File frozenrequirements.txt does not exist!"
else
cp frozenrequirements.txt tests/aws-test/build-aws/requirements.txt
fi
cp requirements.txt tests/aws-test/build-aws
cp requirements_dev.txt tests/aws-test/build-aws
cp -r tests/aws-test "${INTEGRATION_TEST_OUTPUT_DIR}"
cp -r tests/integration "${INTEGRATION_TEST_OUTPUT_DIR}"
cp -r schema/indexation "${INTEGRATION_TEST_OUTPUT_DIR}"
......@@ -9,7 +9,6 @@ which yum && sudo yum install -y python3 python3-pip python3-venv || echo "Not R
python3 -m venv env
source env/bin/activate
python3 -m pip install --upgrade pip
python3 -m pip install -r frozenrequirements.txt
python3 -m pip install -r requirements.txt
python3 -m pip install -r requirements_dev.txt
python3 -m pip install wheel pytest pytest-cov
......
......@@ -40,6 +40,7 @@ cd deployment/osdu-core/os-wellbore-domain-services/testing
python3 -m venv env
source env/bin/activate
pip install -r ./aws-test/build-aws/requirements.txt
pip install -r ./aws-test/build-aws/requirements_dev.txt
rm -rf test-reports/
mkdir test-reports
......
......@@ -346,4 +346,20 @@ def build_request_search_logs() -> RequestRunner:
},
payload='{ "query": "" }'
)
return RequestRunner(rq_proto)
\ No newline at end of file
return RequestRunner(rq_proto)
def build_request_search_wellbore_by_name() -> RequestRunner:
rq_proto = Request(
name='search wellbore by name',
method='POST',
url='{{base_url}}/alpha/ddms/v3/{{search_query_type}}/wellbores/byname?names=wdms_e2e_search_refs_v%2A',
headers={
'accept': 'application/json',
'data-partition-id': '{{data_partition}}',
'Connection': '{{header_connection}}',
'Authorization': 'Bearer {{token}}',
},
payload='{}'
)
return RequestRunner(rq_proto)
......@@ -94,3 +94,11 @@ def test_search_logset_by_wellbore_attribute(with_wdms_env, set_search_query_typ
assert resobj.totalCount >= 2
@pytest.mark.tag('search')
@pytest.mark.dependency(depends=["test_setup_for_search"])
def test_search_wellbore_by_name(with_wdms_env):
#Only search and no fast search
env = with_wdms_env
env.set('search_query_type', 'query')
resobj = build_request_search_wellbore_by_name().call(with_wdms_env, assert_status=200).get_response_obj()
assert resobj.totalCount >= 1
import pytest
import app.routers.search.search_v3_wellbore as search_v3_wellbore
import app.routers.search.search_v3 as search_v3
ADDED_QUERY_PARAMS = [
......@@ -31,3 +32,28 @@ RELATIONSHIPS_QUERY_PARAMS = [
@pytest.mark.parametrize("id, user_query, expected_query", RELATIONSHIPS_QUERY_PARAMS)
def test_added_relationships_query(id, user_query, expected_query):
assert search_v3.added_relationships_query(id, 'WellboreID', user_query) == expected_query
NAMES_QUERY_PARAMS = [
(None, None, 'data.FacilityName:None'),
('Fab OR Fred', None, 'data.FacilityName:Fab OR Fred'),
('Fab', 'data.AnyField:\\"any value\\"', 'data.FacilityName:Fab AND (data.AnyField:\\"any value\\")'),
]
@pytest.mark.parametrize("names, user_query, expected_query", NAMES_QUERY_PARAMS)
def test_update_query_with_names_based_search(names, user_query, expected_query):
assert search_v3_wellbore.update_query_with_names_based_search(names, user_query) == expected_query
ESCAPE_CHAR_PARAMS = [
('', ''),
('not char to escape', 'not char to escape'),
('wildcard * ? not to escape', 'wildcard * ? not to escape'),
(r'all other to escape +-=><!(){}[]^"~:\ /', r'all other to escape \+\-\=\>\<\!\(\)\{\}\[\]\^\"\~\:\\ \/'),
]
@pytest.mark.parametrize("input_str, expected_str", ESCAPE_CHAR_PARAMS)
def test_escape_forbidden_characters_for_search(input_str, expected_str):
assert search_v3_wellbore.escape_forbidden_characters_for_search(input_str) == expected_str
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment