Commit b4e254b7 authored by Victor Dosda's avatar Victor Dosda
Browse files

Merge branch 'master' into bulk-api-v2-VD

parents 1a65fd6b a157c10b
Pipeline #50374 canceled with stages
in 14 minutes and 27 seconds
......@@ -170,9 +170,9 @@ class DaskBulkStorage:
returns a Future<None>
we should be able to change or support other format easily ?
schema={} instead of 'infer' fixes wrong inference for columns of type string starting with nan values
return self.client.submit(dd.to_parquet, ddf, path, schema="infer",
return self.client.submit(dd.to_parquet, ddf, path, schema={}, engine='pyarrow',
def _save_with_pandas(self, path, pdf: dd.DataFrame):
......@@ -165,6 +165,23 @@ async def test_session_update_add_new_columns(test_session, dask_storage: DaskBu
ddf = await dask_storage.load_bulk(test_session.recordId, new_bulk_id)
await compare_frame(df_ref, ddf)
async def test_session_update_add_new_columns_shifted(test_session, dask_storage: DaskBulkStorage):
A = generate_df(['A'], range(100))
C = generate_df(['A', 'strC'], index=range(100, 200))
df_ref = pd.concat([A,C])
bulk_id = await dask_storage.save_blob(A, record_id=test_session.recordId)
#await dask_storage.session_add_chunk(test_session, B)
await dask_storage.session_add_chunk(test_session, C)
new_bulk_id = await dask_storage.session_commit(test_session, from_bulk_id=bulk_id)
assert bulk_id != new_bulk_id
ddf = await dask_storage.load_bulk(test_session.recordId, new_bulk_id)
await compare_frame(df_ref, ddf)
......@@ -299,12 +299,12 @@ def test_send_all_data_once_post_data_v2_get_data_v3(setup_client,
['TVD', 'float_X', 'str_X', 'date_X'],
['MD', 'X'],
['MD', 'float_X'],
['MD', 'str_MD'],
# BELOW test cases FAIL with UPDATE mode:
# => If adding new column Date/String not starting at first index AND override an existing column
# ['MD', 'str_MD'],
# ['MD', 'date_X'],
# ['MD', 'float_X', 'str_X', 'date_X'],
#['MD', 'date_X'],
#['MD', 'float_X', 'str_X', 'date_X'],
@pytest.mark.parametrize("session_mode", [
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment