fix limit and offset query parameters on log v2

app/bulk_utils.py

@@ -19,6 +19,7 @@ from contextlib import suppress
 
 from fastapi import APIRouter, Depends, HTTPException, Request, status
 from fastapi.responses import Response
+import dask.dataframe as dd
 import pandas as pd
 
 from app.bulk_persistence import DataframeSerializerAsync, JSONOrient, get_dataframe
@@ -131,6 +132,9 @@ class DataFrameRender:
     @staticmethod
     @with_trace('process_params')
     async def process_params(df, params: GetDataParams):
+        if isinstance(df, pd.DataFrame):
+            df = dd.from_pandas(df, npartitions=1)
+
         if params.curves:
             selection = list(map(str.strip, params.curves.split(',')))
             columns = DataFrameRender.get_matching_column(selection, set(df))
@@ -142,10 +146,7 @@ class DataFrameRender:
             df = df.loc[~df.index.isin(index)]
 
         if params.limit and params.limit > 0:
-            try:
-                df = df.head(params.limit, npartitions=-1, compute=False)  # dask async
-            except:
-                df = df.head(params.limit)
+            df = df.head(params.limit, npartitions=-1, compute=False)
         return df
 
     @staticmethod

tests/integration/functional/tests/test_chunking.py

@@ -116,8 +116,10 @@ def build_request_post_chunk(entity_type: str, record_id: str, session_id: str,
     url = build_base_url(entity_type) + f'/{record_id}/sessions/{session_id}/data'
     return build_request(f'{entity_type} post data', 'POST', url, payload=payload)
 
-def build_request_get_data(entity_type: str, record_id: str) -> RequestRunner:
+def build_request_get_data(entity_type: str, record_id: str, filters=None) -> RequestRunner:
     url = build_base_url(entity_type) + f'/{record_id}/data'
+    if filters:
+        url = url + '?' + '&'.join(f'{k}={v}' for k, v in filters.items())
     return build_request(f'{entity_type} get data', 'GET', url)
 
 
@@ -485,4 +487,18 @@ def test_get_data_from_record_data_without_dask(with_wdms_env, entity_type, seri
 
         result = build_request_get_data(entity_type, record_id).call(with_wdms_env, headers=headers, assert_status=200)
         pd.testing.assert_frame_equal(data, serializer.read(result.response.content), check_dtype=False)
-        # check type set to false since in Json dType is lost so int32 can become int64
\ No newline at end of file
+        # check type set to false since in Json dType is lost so int32 can become int64
+
+        result = build_request_get_data(entity_type, record_id, filters={'curves': 'MD'}).call(with_wdms_env, headers=headers, assert_status=200)
+        pd.testing.assert_frame_equal(data[['MD']], serializer.read(result.response.content), check_dtype=False)
+
+        result = build_request_get_data(entity_type, record_id, filters={'limit': 5}).call(with_wdms_env, headers=headers, assert_status=200)
+        pd.testing.assert_frame_equal(data[:5], serializer.read(result.response.content), check_dtype=False)
+
+        result = build_request_get_data(entity_type, record_id, filters={'offset': 4}).call(with_wdms_env, headers=headers, assert_status=200)
+        pd.testing.assert_frame_equal(data[4:], serializer.read(result.response.content), check_dtype=False)
+
+        result = build_request_get_data(entity_type, record_id,
+                                        filters={'curves': 'X', 'offset': 2, 'limit': 5}
+                                        ).call(with_wdms_env, headers=headers, assert_status=200)
+        pd.testing.assert_frame_equal(data[['X']][2:7], serializer.read(result.response.content), check_dtype=False)