Commit 038357e1 authored by Jeremie Hallal's avatar Jeremie Hallal
Browse files

Merge branch 'alpha_v2_fix_offset_limit_params' into 'master'

fix limit and offset query parameters on log v2

See merge request osdu/platform/domain-data-mgmt-services/wellbore/wellbore-domain-services!161
parents 36ef6233 5b3efc4b
Pipeline #50878 passed with stages
in 11 minutes and 29 seconds
......@@ -19,6 +19,7 @@ from contextlib import suppress
from fastapi import APIRouter, Depends, HTTPException, Request, status
from fastapi.responses import Response
import dask.dataframe as dd
import pandas as pd
from app.bulk_persistence import DataframeSerializerAsync, JSONOrient, get_dataframe
......@@ -131,6 +132,9 @@ class DataFrameRender:
@staticmethod
@with_trace('process_params')
async def process_params(df, params: GetDataParams):
if isinstance(df, pd.DataFrame):
df = dd.from_pandas(df, npartitions=1)
if params.curves:
selection = list(map(str.strip, params.curves.split(',')))
columns = DataFrameRender.get_matching_column(selection, set(df))
......@@ -142,10 +146,7 @@ class DataFrameRender:
df = df.loc[~df.index.isin(index)]
if params.limit and params.limit > 0:
try:
df = df.head(params.limit, npartitions=-1, compute=False) # dask async
except:
df = df.head(params.limit)
df = df.head(params.limit, npartitions=-1, compute=False)
return df
@staticmethod
......
......@@ -116,8 +116,10 @@ def build_request_post_chunk(entity_type: str, record_id: str, session_id: str,
url = build_base_url(entity_type) + f'/{record_id}/sessions/{session_id}/data'
return build_request(f'{entity_type} post data', 'POST', url, payload=payload)
def build_request_get_data(entity_type: str, record_id: str) -> RequestRunner:
def build_request_get_data(entity_type: str, record_id: str, filters=None) -> RequestRunner:
url = build_base_url(entity_type) + f'/{record_id}/data'
if filters:
url = url + '?' + '&'.join(f'{k}={v}' for k, v in filters.items())
return build_request(f'{entity_type} get data', 'GET', url)
......@@ -485,4 +487,18 @@ def test_get_data_from_record_data_without_dask(with_wdms_env, entity_type, seri
result = build_request_get_data(entity_type, record_id).call(with_wdms_env, headers=headers, assert_status=200)
pd.testing.assert_frame_equal(data, serializer.read(result.response.content), check_dtype=False)
# check type set to false since in Json dType is lost so int32 can become int64
\ No newline at end of file
# check type set to false since in Json dType is lost so int32 can become int64
result = build_request_get_data(entity_type, record_id, filters={'curves': 'MD'}).call(with_wdms_env, headers=headers, assert_status=200)
pd.testing.assert_frame_equal(data[['MD']], serializer.read(result.response.content), check_dtype=False)
result = build_request_get_data(entity_type, record_id, filters={'limit': 5}).call(with_wdms_env, headers=headers, assert_status=200)
pd.testing.assert_frame_equal(data[:5], serializer.read(result.response.content), check_dtype=False)
result = build_request_get_data(entity_type, record_id, filters={'offset': 4}).call(with_wdms_env, headers=headers, assert_status=200)
pd.testing.assert_frame_equal(data[4:], serializer.read(result.response.content), check_dtype=False)
result = build_request_get_data(entity_type, record_id,
filters={'curves': 'X', 'offset': 2, 'limit': 5}
).call(with_wdms_env, headers=headers, assert_status=200)
pd.testing.assert_frame_equal(data[['X']][2:7], serializer.read(result.response.content), check_dtype=False)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment