Commit dc13dfab authored by Jeremie Hallal's avatar Jeremie Hallal
Browse files

add test test_parquet_maintain_float_type

parent 2f6f3d63
......@@ -7,6 +7,7 @@ Apache-2.0
========================================================================
The following software have components provided under the terms of this license:
- aiobotocore (from https://github.com/aio-libs/aiobotocore)
- aiohttp (from https://github.com/aio-libs/aiohttp/)
- async-timeout (from https://github.com/aio-libs/async_timeout/)
- boto3 (from https://github.com/boto/boto3)
......@@ -37,6 +38,7 @@ The following software have components provided under the terms of this license:
- packaging (from https://github.com/pypa/packaging)
- pandas (from http://pandas.pydata.org)
- pep517 (from https://github.com/takluyver/pep517)
- proto-plus (from https://github.com/googleapis/proto-plus-python.git)
- pyarrow (from https://arrow.apache.org/)
- pytest-asyncio (from https://github.com/pytest-dev/pytest-asyncio)
- pytest-dependency (from https://github.com/RKrahl/pytest-dependency)
......@@ -69,6 +71,7 @@ The following software have components provided under the terms of this license:
- pyasn1-modules (from http://sourceforge.net/projects/pyasn1/)
- pycparser (from https://github.com/eliben/pycparser)
- tblib (from https://github.com/ionelmc/python-tblib)
- wrapt (from https://github.com/GrahamDumpleton/wrapt)
========================================================================
BSD-3-Clause
......@@ -113,10 +116,12 @@ The following software have components provided under the terms of this license:
- python-dateutil (from https://dateutil.readthedocs.org)
- python-rapidjson (from https://github.com/python-rapidjson/python-rapidjson)
- requests-oauthlib (from https://github.com/requests/requests-oauthlib)
- s3fs (from http://github.com/dask/s3fs/)
- starlette (from https://github.com/encode/starlette)
- tblib (from https://github.com/ionelmc/python-tblib)
- toolz (from http://github.com/pytoolz/toolz/)
- uvicorn (from https://github.com/tomchristie/uvicorn)
- wrapt (from https://github.com/GrahamDumpleton/wrapt)
- zict (from http://github.com/dask/zict/)
========================================================================
......@@ -132,6 +137,7 @@ The following software have components provided under the terms of this license:
- numpy (from http://www.numpy.org)
- pandas (from http://pandas.pydata.org)
- partd (from http://github.com/dask/partd/)
- s3fs (from http://github.com/dask/s3fs/)
- toolz (from http://github.com/pytoolz/toolz/)
========================================================================
......@@ -220,6 +226,7 @@ The following software have components provided under the terms of this license:
- PyYAML (from http://pyyaml.org/wiki/PyYAML)
- adal (from https://github.com/AzureAD/azure-activedirectory-library-for-python)
- aiohttp (from https://github.com/aio-libs/aiohttp/)
- aioitertools (from https://github.com/jreese/aioitertools)
- aioredis (from https://github.com/aio-libs/aioredis)
- anyio (from https://pypi.org/project/anyio/3.3.0/)
- asgiref (from http://github.com/django/asgiref/)
......@@ -272,7 +279,7 @@ The following software have components provided under the terms of this license:
- sniffio (from https://github.com/python-trio/sniffio)
- structlog (from http://www.structlog.org/)
- toml (from https://github.com/uiri/toml)
- tomli (from https://pypi.org/project/tomli/1.1.0/)
- tomli (from https://pypi.org/project/tomli/1.2.1/)
- urllib3 (from https://urllib3.readthedocs.io/)
- xmltodict (from https://github.com/martinblech/xmltodict)
- zipp (from https://github.com/jaraco/zipp)
......
......@@ -821,6 +821,57 @@ def test_session_update_previous_version(setup_client, entity_type):
pd.testing.assert_frame_equal(expected_df, res_df)
@pytest.mark.parametrize("entity_type", ['WellLog', 'Log'])
def test_parquet_maintain_float_type(setup_client, entity_type):
""" send float32 and float64 columns and check if the type is maintain """
client = setup_client
record_id = _create_record(client, entity_type)
chunking_url = Definitions[entity_type]['chunking_url']
df = generate_df(['MD', 'float_32', 'float_64'], range(5))
df = df.astype({'float_32': 'float32', 'float_64': 'float64'})
# Without session
write_response = client.post(f'{chunking_url}/{record_id}/data',
data=df.to_parquet(engine="pyarrow"),
headers={'Content-Type': 'application/x-parquet'})
assert write_response.status_code == 200
get_response = client.get(f'{chunking_url}/{record_id}/data')
assert get_response.status_code == 200
res_df = _create_df_from_response(get_response)
pd.testing.assert_frame_equal(df, res_df)
# With session
session_response = client.post(f'{chunking_url}/{record_id}/sessions', json={'mode': 'update'})
assert session_response.status_code == 200
session_id = session_response.json()['id']
new_chunk = generate_df(['MD', 'float_32', 'float_64'], range(5, 10))
new_chunk = new_chunk.astype({'float_32': 'float32', 'float_64': 'float64'})
chunk_response = client.post(f'{chunking_url}/{record_id}/sessions/{session_id}/data',
data=new_chunk.to_parquet(engine="pyarrow"),
headers={'Content-Type': 'application/x-parquet'})
assert chunk_response.status_code == 200
commit_response = client.patch(f'{chunking_url}/{record_id}/sessions/{session_id}', json={'state': 'commit'})
assert commit_response.status_code == 200
df = pd.concat([df, new_chunk])
get_response = client.get(f'{chunking_url}/{record_id}/data')
assert get_response.status_code == 200
res_df = _create_df_from_response(get_response)
pd.testing.assert_frame_equal(df, res_df)
# with curve selection
for curve in ('float_32', 'float_64'):
get_response = client.get(f'{chunking_url}/{record_id}/data', params={'curves': curve})
assert get_response.status_code == 200
res_df = _create_df_from_response(get_response)
pd.testing.assert_frame_equal(df[[curve]], res_df)
# todo:
# - concurrent sessions using fromVersion in Integrations tests
# - index: check if dataframe has an index
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment