Skip to content
Snippets Groups Projects
Commit 0da78180 authored by Cyril Monmouton's avatar Cyril Monmouton
Browse files

Catalog/enh: trace attributes when reading bulk

parent d2d94f74
No related branches found
No related tags found
1 merge request!460Improve statibility of fetching of bulk catalog by not using Dask workers
Pipeline #98334 failed
......@@ -33,7 +33,7 @@ from app.conf import Config
from .dask_worker_plugin import DaskWorkerPlugin
from .errors import BulkRecordNotFound, BulkNotProcessable, internal_bulk_exceptions
from .traces import map_with_trace, submit_with_trace, trace_attributes_root_span
from .traces import map_with_trace, submit_with_trace, trace_attributes_root_span, trace_attributes_current_span
from .utils import (WDMS_INDEX_NAME, by_pairs, do_merge, join_dataframes, worker_capture_timing_handlers,
get_num_rows, set_index, index_union)
from ..dataframe_validators import is_reserved_column_name, DataFrameValidationFunc
......@@ -180,6 +180,11 @@ class DaskBulkStorage:
if index_df:
dfs.append(index_df)
trace_attributes_current_span({
'parquet-files-to-load-count': len(files_to_load),
'df-to-merge-count': len(dfs)
})
if not dfs:
raise RuntimeError("cannot find requested columns")
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment