Commit cee4fa3c authored by Jeremie Hallal's avatar Jeremie Hallal
Browse files

fix limit and offset query parameters on log v2

parent c8df76b4
Pipeline #50867 failed with stages
in 14 minutes and 22 seconds
......@@ -19,6 +19,7 @@ from contextlib import suppress
from fastapi import APIRouter, Depends, HTTPException, Request, status
from fastapi.responses import Response
import dask.dataframe as dd
import pandas as pd
from app.bulk_persistence import DataframeSerializerAsync, JSONOrient, get_dataframe
......@@ -129,6 +130,9 @@ class DataFrameRender:
async def process_params(df, params: GetDataParams):
if isinstance(df, pd.DataFrame):
df = dd.from_pandas(df, npartitions=1)
if params.curves:
selection = list(map(str.strip, params.curves.split(',')))
columns = DataFrameRender.get_matching_column(selection, set(df))
......@@ -140,10 +144,7 @@ class DataFrameRender:
df = df.loc[~df.index.isin(index)]
if params.limit and params.limit > 0:
df = df.head(params.limit, npartitions=-1, compute=False) # dask async
df = df.head(params.limit)
df = df.head(params.limit, npartitions=-1, compute=False)
return df
......@@ -116,8 +116,10 @@ def build_request_post_chunk(entity_type: str, record_id: str, session_id: str,
url = build_base_url(entity_type) + f'/{record_id}/sessions/{session_id}/data'
return build_request(f'{entity_type} post data', 'POST', url, payload=payload)
def build_request_get_data(entity_type: str, record_id: str) -> RequestRunner:
def build_request_get_data(entity_type: str, record_id: str, filters=None) -> RequestRunner:
url = build_base_url(entity_type) + f'/{record_id}/data'
if filters:
url = url + '?' + '&'.join(f'{k}={v}' for k, v in filters.items())
return build_request(f'{entity_type} get data', 'GET', url)
......@@ -485,4 +487,18 @@ def test_get_data_from_record_data_without_dask(with_wdms_env, entity_type, seri
result = build_request_get_data(entity_type, record_id).call(with_wdms_env, headers=headers, assert_status=200)
pd.testing.assert_frame_equal(data,, check_dtype=False)
# check type set to false since in Json dType is lost so int32 can become int64
\ No newline at end of file
# check type set to false since in Json dType is lost so int32 can become int64
result = build_request_get_data(entity_type, record_id, filters={'curves': 'MD'}).call(with_wdms_env, headers=headers, assert_status=200)
pd.testing.assert_frame_equal(data[['MD']],, check_dtype=False)
result = build_request_get_data(entity_type, record_id, filters={'limit': 5}).call(with_wdms_env, headers=headers, assert_status=200)
pd.testing.assert_frame_equal(data[:5],, check_dtype=False)
result = build_request_get_data(entity_type, record_id, filters={'offset': 4}).call(with_wdms_env, headers=headers, assert_status=200)
pd.testing.assert_frame_equal(data[4:],, check_dtype=False)
result = build_request_get_data(entity_type, record_id,
filters={'curves': 'X', 'offset': 2, 'limit': 5}
).call(with_wdms_env, headers=headers, assert_status=200)
pd.testing.assert_frame_equal(data[['X']][2:7],, check_dtype=False)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment