From 98697084b1cafccc671bd308bf12fad95a4308fc Mon Sep 17 00:00:00 2001 From: Luc Yriarte Date: Fri, 4 Mar 2022 16:43:21 +0100 Subject: [PATCH 01/12] BulkURI import direct from app.bulk_persistence module --- app/model/log_bulk.py | 2 +- app/routers/ddms_v2/log_ddms_v2.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/model/log_bulk.py b/app/model/log_bulk.py index 1ab79edc..acf15efa 100644 --- a/app/model/log_bulk.py +++ b/app/model/log_bulk.py @@ -18,7 +18,7 @@ from jsonpath_ng import parse as parse_jsonpath from jsonpath_ng.jsonpath import Parent as JsonParent from odes_storage.models import Record -from app.bulk_persistence.bulk_uri import BulkURI +from app.bulk_persistence import BulkURI class LogBulkHelper: diff --git a/app/routers/ddms_v2/log_ddms_v2.py b/app/routers/ddms_v2/log_ddms_v2.py index bb30a694..d05107bb 100644 --- a/app/routers/ddms_v2/log_ddms_v2.py +++ b/app/routers/ddms_v2/log_ddms_v2.py @@ -38,7 +38,7 @@ from odes_storage.models import ( from pydantic import BaseModel, Field from app.bulk_persistence import DataframeSerializerAsync, DataframeSerializerSync, JSONOrient, MimeTypes, get_dataframe -from app.bulk_persistence.bulk_uri import BulkURI +from app.bulk_persistence import BulkURI from app.clients.storage_service_client import get_storage_record_service from app.model.log_bulk import LogBulkHelper from app.model.model_curated import log -- GitLab From a5efb8664c0a5573836f8ff3b0e1fcbfd46cf29c Mon Sep 17 00:00:00 2001 From: Luc Yriarte Date: Fri, 4 Mar 2022 16:47:14 +0100 Subject: [PATCH 02/12] MimeTypes import direct from app.bulk_persistence module --- app/bulk_persistence/__init__.py | 2 +- app/routers/common_parameters.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/bulk_persistence/__init__.py b/app/bulk_persistence/__init__.py index 3be6223f..06b7753e 100644 --- a/app/bulk_persistence/__init__.py +++ b/app/bulk_persistence/__init__.py @@ -20,7 +20,7 @@ from .dask.dask_bulk_storage_local import make_local_dask_bulk_storage from .dataframe_persistence import create_and_store_dataframe, get_dataframe, download_bulk from .dataframe_serializer import DataframeSerializerAsync, DataframeSerializerSync from .json_orient import JSONOrient -from .mime_types import MimeTypes +from .mime_types import MimeTypes, MimeType from .tenant_provider import resolve_tenant from .exceptions import UnknownChannelsException, InvalidBulkException, NoBulkException, NoDataException, RecordNotFoundException from .consistency_checks import ConsistencyException, DataConsistencyChecks diff --git a/app/routers/common_parameters.py b/app/routers/common_parameters.py index bfdf16b3..4ab2d78f 100644 --- a/app/routers/common_parameters.py +++ b/app/routers/common_parameters.py @@ -1,7 +1,7 @@ from fastapi import Query, Request, HTTPException from pandas import DataFrame -from app.bulk_persistence.mime_types import MimeType, MimeTypes +from app.bulk_persistence import MimeType, MimeTypes from app.bulk_persistence import JSONOrient -- GitLab From bc7c8291d648125040f6e502223a3bba524fd19e Mon Sep 17 00:00:00 2001 From: Luc Yriarte Date: Fri, 4 Mar 2022 17:04:32 +0100 Subject: [PATCH 03/12] Export storage_path_builder hash_record_id --- app/bulk_persistence/__init__.py | 1 + app/routers/delete/delete_bulk_data.py | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/bulk_persistence/__init__.py b/app/bulk_persistence/__init__.py index 06b7753e..b87893f6 100644 --- a/app/bulk_persistence/__init__.py +++ b/app/bulk_persistence/__init__.py @@ -17,6 +17,7 @@ from .bulk_filter import BulkReadFilters, BulkReadFilterOperator from .model_chunking import GetDataParams, DataframeBasicDescribe, DataframeDescribe from .dask.dask_bulk_storage import DaskBulkStorage from .dask.dask_bulk_storage_local import make_local_dask_bulk_storage +from .dask.storage_path_builder import hash_record_id from .dataframe_persistence import create_and_store_dataframe, get_dataframe, download_bulk from .dataframe_serializer import DataframeSerializerAsync, DataframeSerializerSync from .json_orient import JSONOrient diff --git a/app/routers/delete/delete_bulk_data.py b/app/routers/delete/delete_bulk_data.py index e1544396..53ef5836 100644 --- a/app/routers/delete/delete_bulk_data.py +++ b/app/routers/delete/delete_bulk_data.py @@ -12,11 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from app.bulk_persistence import resolve_tenant +from app.bulk_persistence import resolve_tenant, hash_record_id from osdu.core.api.storage.blob_storage_base import BlobStorageBase import asyncio -from app.bulk_persistence.dask.storage_path_builder import hash_record_id from app.clients import StorageRecordServiceClient from app.clients.storage_service_client import get_storage_record_service from app.routers.bulk.bulk_uri_dependencies import BulkIdAccess -- GitLab From f168b97720a7d64101c878f90a618365a63d6236 Mon Sep 17 00:00:00 2001 From: Luc Yriarte Date: Thu, 31 Mar 2022 15:13:01 +0200 Subject: [PATCH 04/12] un-needed internal import --- app/bulk_persistence/dask/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/bulk_persistence/dask/__init__.py b/app/bulk_persistence/dask/__init__.py index ab4e198a..e69de29b 100644 --- a/app/bulk_persistence/dask/__init__.py +++ b/app/bulk_persistence/dask/__init__.py @@ -1 +0,0 @@ -from . import dask_config -- GitLab From f22f382014c520e23b6f18b9e50eab8b2546e5f4 Mon Sep 17 00:00:00 2001 From: Luc Yriarte Date: Wed, 2 Mar 2022 17:33:54 +0100 Subject: [PATCH 05/12] Bulk persistence module exports refactoring: consistency checks, uri dependencies # Conflicts: # app/consistency/welllog_consistency.py --- app/bulk_persistence/__init__.py | 1 + app/consistency/welllog_consistency.py | 3 +-- app/routers/bulk/bulk_routes.py | 4 ++-- app/routers/bulk/bulk_uri_dependencies.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/bulk_persistence/__init__.py b/app/bulk_persistence/__init__.py index b87893f6..3eeb93e3 100644 --- a/app/bulk_persistence/__init__.py +++ b/app/bulk_persistence/__init__.py @@ -20,6 +20,7 @@ from .dask.dask_bulk_storage_local import make_local_dask_bulk_storage from .dask.storage_path_builder import hash_record_id from .dataframe_persistence import create_and_store_dataframe, get_dataframe, download_bulk from .dataframe_serializer import DataframeSerializerAsync, DataframeSerializerSync +from .dataframe_validators import auto_cast_columns_to_string, DataFrameValidationFunc, no_validation from .json_orient import JSONOrient from .mime_types import MimeTypes, MimeType from .tenant_provider import resolve_tenant diff --git a/app/consistency/welllog_consistency.py b/app/consistency/welllog_consistency.py index 650d7c7a..54c95f98 100644 --- a/app/consistency/welllog_consistency.py +++ b/app/consistency/welllog_consistency.py @@ -7,9 +7,8 @@ from dask.dataframe.core import DataFrame as DaskDataFrame from odes_storage.models import Record from app.helper.traces import with_trace -from app.bulk_persistence.consistency_checks import ConsistencyException, DataConsistencyChecks from app.bulk_persistence.dask.dask_bulk_storage import BulkRecordNotFound -from app.bulk_persistence import DaskBulkStorage +from app.bulk_persistence import DaskBulkStorage, ConsistencyException, DataConsistencyChecks from app.bulk_persistence.dask.traces import submit_with_trace from app.model.model_utils import from_record from app.model.osdu_model import WellLog110 diff --git a/app/routers/bulk/bulk_routes.py b/app/routers/bulk/bulk_routes.py index 6f1c3d1e..c4df4ea1 100644 --- a/app/routers/bulk/bulk_routes.py +++ b/app/routers/bulk/bulk_routes.py @@ -57,13 +57,13 @@ from app.routers.sessions import ( ) # imports from bulk persistence -from app.bulk_persistence.dataframe_validators import (auto_cast_columns_to_string, +from app.bulk_persistence import (auto_cast_columns_to_string, DataFrameValidationFunc, no_validation) from app.bulk_persistence import JSONOrient, get_dataframe, download_bulk from app.bulk_persistence import DaskBulkStorage from app.bulk_persistence.dask.errors import BulkError, BulkRecordNotFound, FilterError, TooManyColumnsRequested -from app.bulk_persistence.mime_types import MimeTypes, MimeType +from app.bulk_persistence import MimeTypes, MimeType from app.bulk_persistence.dask.traces import trace_dataframe_attributes, trace_attributes_root_span diff --git a/app/routers/bulk/bulk_uri_dependencies.py b/app/routers/bulk/bulk_uri_dependencies.py index cc592398..7840d07a 100644 --- a/app/routers/bulk/bulk_uri_dependencies.py +++ b/app/routers/bulk/bulk_uri_dependencies.py @@ -3,7 +3,7 @@ from abc import ABC from typing import Optional from fastapi import Request -from app.bulk_persistence.bulk_uri import BulkURI +from app.bulk_persistence import BulkURI from app.model.log_bulk import LogBulkHelper -- GitLab From 13fdef692837bf7a4c23268407fe04fa41530dac Mon Sep 17 00:00:00 2001 From: Luc Yriarte Date: Fri, 4 Mar 2022 17:22:19 +0100 Subject: [PATCH 06/12] Export dask traces # Conflicts: # app/consistency/welllog_consistency.py # app/routers/ddms_v2/persistence.py # Conflicts: # app/consistency/welllog_consistency.py --- app/bulk_persistence/__init__.py | 1 + app/consistency/welllog_consistency.py | 3 +-- app/routers/bulk/bulk_routes.py | 2 +- app/routers/ddms_v2/persistence.py | 4 +--- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/app/bulk_persistence/__init__.py b/app/bulk_persistence/__init__.py index 3eeb93e3..cd92f38a 100644 --- a/app/bulk_persistence/__init__.py +++ b/app/bulk_persistence/__init__.py @@ -18,6 +18,7 @@ from .model_chunking import GetDataParams, DataframeBasicDescribe, DataframeDesc from .dask.dask_bulk_storage import DaskBulkStorage from .dask.dask_bulk_storage_local import make_local_dask_bulk_storage from .dask.storage_path_builder import hash_record_id +from .dask.traces import trace_dataframe_attributes, submit_with_trace, trace_attributes_root_span from .dataframe_persistence import create_and_store_dataframe, get_dataframe, download_bulk from .dataframe_serializer import DataframeSerializerAsync, DataframeSerializerSync from .dataframe_validators import auto_cast_columns_to_string, DataFrameValidationFunc, no_validation diff --git a/app/consistency/welllog_consistency.py b/app/consistency/welllog_consistency.py index 54c95f98..bba5fee9 100644 --- a/app/consistency/welllog_consistency.py +++ b/app/consistency/welllog_consistency.py @@ -8,8 +8,7 @@ from odes_storage.models import Record from app.helper.traces import with_trace from app.bulk_persistence.dask.dask_bulk_storage import BulkRecordNotFound -from app.bulk_persistence import DaskBulkStorage, ConsistencyException, DataConsistencyChecks -from app.bulk_persistence.dask.traces import submit_with_trace +from app.bulk_persistence import DaskBulkStorage, ConsistencyException, DataConsistencyChecks, submit_with_trace from app.model.model_utils import from_record from app.model.osdu_model import WellLog110 from app.context import get_ctx diff --git a/app/routers/bulk/bulk_routes.py b/app/routers/bulk/bulk_routes.py index c4df4ea1..75dfc986 100644 --- a/app/routers/bulk/bulk_routes.py +++ b/app/routers/bulk/bulk_routes.py @@ -64,7 +64,7 @@ from app.bulk_persistence import JSONOrient, get_dataframe, download_bulk from app.bulk_persistence import DaskBulkStorage from app.bulk_persistence.dask.errors import BulkError, BulkRecordNotFound, FilterError, TooManyColumnsRequested from app.bulk_persistence import MimeTypes, MimeType -from app.bulk_persistence.dask.traces import trace_dataframe_attributes, trace_attributes_root_span +from app.bulk_persistence import trace_dataframe_attributes, trace_attributes_root_span from app.bulk_persistence import DataConsistencyChecks diff --git a/app/routers/ddms_v2/persistence.py b/app/routers/ddms_v2/persistence.py index ce5eb4af..4d6ea54d 100644 --- a/app/routers/ddms_v2/persistence.py +++ b/app/routers/ddms_v2/persistence.py @@ -16,12 +16,10 @@ import pandas as pd from odes_storage.models import Record -from app.bulk_persistence import create_and_store_dataframe -from app.bulk_persistence import get_dataframe +from app.bulk_persistence import create_and_store_dataframe, get_dataframe, trace_dataframe_attributes from app.context import Context from app.model.log_bulk import LogBulkHelper -from app.bulk_persistence.dask.traces import trace_dataframe_attributes from app.helper.traces import with_trace from app.helper.logger import get_logger -- GitLab From a6bd9e14e7f8906f5167af647c42b15636e82d38 Mon Sep 17 00:00:00 2001 From: Luc Yriarte Date: Fri, 4 Mar 2022 17:27:50 +0100 Subject: [PATCH 07/12] Bulk routes partial imports regroup --- app/routers/bulk/bulk_routes.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/app/routers/bulk/bulk_routes.py b/app/routers/bulk/bulk_routes.py index 75dfc986..df60e0c5 100644 --- a/app/routers/bulk/bulk_routes.py +++ b/app/routers/bulk/bulk_routes.py @@ -58,13 +58,14 @@ from app.routers.sessions import ( # imports from bulk persistence from app.bulk_persistence import (auto_cast_columns_to_string, - DataFrameValidationFunc, - no_validation) -from app.bulk_persistence import JSONOrient, get_dataframe, download_bulk -from app.bulk_persistence import DaskBulkStorage + DataFrameValidationFunc, no_validation, + JSONOrient, + get_dataframe, download_bulk, + DaskBulkStorage, + MimeTypes, MimeType, + trace_dataframe_attributes, trace_attributes_root_span + ) from app.bulk_persistence.dask.errors import BulkError, BulkRecordNotFound, FilterError, TooManyColumnsRequested -from app.bulk_persistence import MimeTypes, MimeType -from app.bulk_persistence import trace_dataframe_attributes, trace_attributes_root_span from app.bulk_persistence import DataConsistencyChecks -- GitLab From ac5933f0b290efa182f9860f442d46d2fd13ba50 Mon Sep 17 00:00:00 2001 From: Luc Yriarte Date: Thu, 31 Mar 2022 15:22:52 +0200 Subject: [PATCH 08/12] More bulk persistence imports refactoring on bulk router utils and trajectory consistency # Conflicts: # app/consistency/trajectory_consistency.py # app/routers/bulk/utils.py --- app/bulk_persistence/__init__.py | 2 +- app/consistency/trajectory_consistency.py | 3 +-- app/routers/bulk/utils.py | 11 +++-------- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/app/bulk_persistence/__init__.py b/app/bulk_persistence/__init__.py index cd92f38a..a3fa4833 100644 --- a/app/bulk_persistence/__init__.py +++ b/app/bulk_persistence/__init__.py @@ -21,7 +21,7 @@ from .dask.storage_path_builder import hash_record_id from .dask.traces import trace_dataframe_attributes, submit_with_trace, trace_attributes_root_span from .dataframe_persistence import create_and_store_dataframe, get_dataframe, download_bulk from .dataframe_serializer import DataframeSerializerAsync, DataframeSerializerSync -from .dataframe_validators import auto_cast_columns_to_string, DataFrameValidationFunc, no_validation +from .dataframe_validators import auto_cast_columns_to_string, columns_type_must_be_string, DataFrameValidationFunc, no_validation from .json_orient import JSONOrient from .mime_types import MimeTypes, MimeType from .tenant_provider import resolve_tenant diff --git a/app/consistency/trajectory_consistency.py b/app/consistency/trajectory_consistency.py index 982ee048..d5ed8cb7 100644 --- a/app/consistency/trajectory_consistency.py +++ b/app/consistency/trajectory_consistency.py @@ -9,8 +9,7 @@ from app.model.osdu_model import WellboreTrajectory110 from app.helper.traces import with_trace from app.bulk_persistence.consistency_checks import ConsistencyException, DataConsistencyChecks from app.bulk_persistence.dask.dask_bulk_storage import BulkRecordNotFound -from app.bulk_persistence import DaskBulkStorage -from app.bulk_persistence.dask.traces import submit_with_trace +from app.bulk_persistence import DaskBulkStorage, submit_with_trace from app.model.model_utils import from_record from app.context import get_ctx diff --git a/app/routers/bulk/utils.py b/app/routers/bulk/utils.py index 3351b638..d7e48c16 100644 --- a/app/routers/bulk/utils.py +++ b/app/routers/bulk/utils.py @@ -12,15 +12,10 @@ import pandas as pd from pyarrow.lib import ArrowInvalid from app.bulk_persistence.dask.errors import FilterError, internal_bulk_exceptions, BulkCurvesNotFound -from app.bulk_persistence.dask.traces import trace_dataframe_attributes -from app.bulk_persistence import DaskBulkStorage -from app.bulk_persistence.dataframe_validators import auto_cast_columns_to_string, columns_type_must_be_string, \ +from app.bulk_persistence import DaskBulkStorage, DataframeSerializerAsync, \ + MimeTypes, MimeType, JSONOrient, trace_dataframe_attributes, capture_timings, \ + auto_cast_columns_to_string, columns_type_must_be_string, \ no_validation, DataFrameValidationFunc -from app.bulk_persistence import DataframeSerializerAsync -from app.bulk_persistence.mime_types import MimeTypes, MimeType -from app.bulk_persistence import JSONOrient -from app.bulk_persistence import capture_timings - from app.clients.storage_service_client import get_storage_record_service from app.context import get_ctx, Context from app.utils import OpenApiHandler -- GitLab From 5ff2862abfd7889698101dab06a33633f9ea29e4 Mon Sep 17 00:00:00 2001 From: Luc Yriarte Date: Thu, 31 Mar 2022 17:28:11 +0200 Subject: [PATCH 09/12] Export dask errors # Conflicts: # app/bulk_persistence/__init__.py # app/consistency/welllog_consistency.py --- app/bulk_persistence/__init__.py | 1 + app/consistency/trajectory_consistency.py | 6 +++--- app/consistency/welllog_consistency.py | 4 ++-- app/routers/bulk/bulk_routes.py | 9 +++------ app/routers/bulk/utils.py | 4 ++-- 5 files changed, 11 insertions(+), 13 deletions(-) diff --git a/app/bulk_persistence/__init__.py b/app/bulk_persistence/__init__.py index a3fa4833..d75320c6 100644 --- a/app/bulk_persistence/__init__.py +++ b/app/bulk_persistence/__init__.py @@ -33,6 +33,7 @@ from .capture_timings import capture_timings from .sessions_storage import Session, SessionsStorage, \ SessionNotFound, SessionInvalidState, SessionUpdatedEtagUnmatched, SessionException, \ SessionState, SessionUpdateMode, SessionInternal, CommitSessionResponse +from .dask.errors import BulkError, BulkRecordNotFound, BulkCurvesNotFound, TooManyColumnsRequested, FilterError, internal_bulk_exceptions # TMP: this should probably not be exposed outside of the bulk_persistence package from .temp_dir import get_temp_dir diff --git a/app/consistency/trajectory_consistency.py b/app/consistency/trajectory_consistency.py index d5ed8cb7..205599fe 100644 --- a/app/consistency/trajectory_consistency.py +++ b/app/consistency/trajectory_consistency.py @@ -7,9 +7,9 @@ from odes_storage.models import Record from app.model.osdu_model import WellboreTrajectory110 from app.helper.traces import with_trace -from app.bulk_persistence.consistency_checks import ConsistencyException, DataConsistencyChecks -from app.bulk_persistence.dask.dask_bulk_storage import BulkRecordNotFound -from app.bulk_persistence import DaskBulkStorage, submit_with_trace +from app.bulk_persistence import ConsistencyException, DataConsistencyChecks, \ + BulkRecordNotFound, \ + DaskBulkStorage, submit_with_trace from app.model.model_utils import from_record from app.context import get_ctx diff --git a/app/consistency/welllog_consistency.py b/app/consistency/welllog_consistency.py index bba5fee9..5409a847 100644 --- a/app/consistency/welllog_consistency.py +++ b/app/consistency/welllog_consistency.py @@ -7,8 +7,8 @@ from dask.dataframe.core import DataFrame as DaskDataFrame from odes_storage.models import Record from app.helper.traces import with_trace -from app.bulk_persistence.dask.dask_bulk_storage import BulkRecordNotFound -from app.bulk_persistence import DaskBulkStorage, ConsistencyException, DataConsistencyChecks, submit_with_trace +from app.bulk_persistence import BulkRecordNotFound, \ + DaskBulkStorage, ConsistencyException, DataConsistencyChecks, submit_with_trace from app.model.model_utils import from_record from app.model.osdu_model import WellLog110 from app.context import get_ctx diff --git a/app/routers/bulk/bulk_routes.py b/app/routers/bulk/bulk_routes.py index df60e0c5..08f23c18 100644 --- a/app/routers/bulk/bulk_routes.py +++ b/app/routers/bulk/bulk_routes.py @@ -63,13 +63,10 @@ from app.bulk_persistence import (auto_cast_columns_to_string, get_dataframe, download_bulk, DaskBulkStorage, MimeTypes, MimeType, - trace_dataframe_attributes, trace_attributes_root_span + trace_dataframe_attributes, trace_attributes_root_span, + BulkError, BulkRecordNotFound, FilterError, TooManyColumnsRequested, + DataConsistencyChecks ) -from app.bulk_persistence.dask.errors import BulkError, BulkRecordNotFound, FilterError, TooManyColumnsRequested - - -from app.bulk_persistence import DataConsistencyChecks - router = APIRouter(route_class=TracingRoute) # router dedicated to bulk APIs diff --git a/app/routers/bulk/utils.py b/app/routers/bulk/utils.py index d7e48c16..2d501245 100644 --- a/app/routers/bulk/utils.py +++ b/app/routers/bulk/utils.py @@ -11,11 +11,11 @@ import dask.dataframe as dd import pandas as pd from pyarrow.lib import ArrowInvalid -from app.bulk_persistence.dask.errors import FilterError, internal_bulk_exceptions, BulkCurvesNotFound from app.bulk_persistence import DaskBulkStorage, DataframeSerializerAsync, \ MimeTypes, MimeType, JSONOrient, trace_dataframe_attributes, capture_timings, \ auto_cast_columns_to_string, columns_type_must_be_string, \ - no_validation, DataFrameValidationFunc + no_validation, DataFrameValidationFunc, \ + FilterError, internal_bulk_exceptions, BulkCurvesNotFound from app.clients.storage_service_client import get_storage_record_service from app.context import get_ctx, Context from app.utils import OpenApiHandler -- GitLab From 838739ef6e73f50f55b592bf6f73bbe5bbc44fe2 Mon Sep 17 00:00:00 2001 From: Luc Yriarte Date: Fri, 8 Apr 2022 10:46:51 +0200 Subject: [PATCH 10/12] Import consistency checks at bulk persistence top level --- app/consistency/reference_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/consistency/reference_check.py b/app/consistency/reference_check.py index a473d4ee..40eb75f0 100644 --- a/app/consistency/reference_check.py +++ b/app/consistency/reference_check.py @@ -1,7 +1,7 @@ import pandas as pd import math from pydantic import BaseModel -from app.bulk_persistence.consistency_checks import ConsistencyException, DataConsistencyChecks +from app.bulk_persistence import ConsistencyException, DataConsistencyChecks class ReferenceCurveException(ConsistencyException): -- GitLab From 7b52124e757c61ffb5ee85d700d5dc69aa6b34f7 Mon Sep 17 00:00:00 2001 From: Luc Yriarte Date: Fri, 8 Apr 2022 15:58:58 +0200 Subject: [PATCH 11/12] Make imports from app outside bulk_persistence module explicit --- app/bulk_persistence/blob_storage.py | 2 +- app/bulk_persistence/capture_timings.py | 2 +- app/bulk_persistence/dask/bulk_catalog.py | 2 +- app/bulk_persistence/dask/session_file_meta.py | 4 ++-- app/bulk_persistence/dask/utils.py | 2 +- app/bulk_persistence/dataframe_persistence.py | 2 +- app/bulk_persistence/dataframe_serializer.py | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/app/bulk_persistence/blob_storage.py b/app/bulk_persistence/blob_storage.py index 0bfd476a..95d69411 100644 --- a/app/bulk_persistence/blob_storage.py +++ b/app/bulk_persistence/blob_storage.py @@ -52,7 +52,7 @@ from .mime_types import MimeType, MimeTypes # - using faster format, e.g. hd5 # - threshold about the busyness of the service (if not busy and not huge data -> direct write) # - better proc fork and arg serialization -from ..helper.traces import with_trace +from app.helper.traces import with_trace def export_to_parquet( diff --git a/app/bulk_persistence/capture_timings.py b/app/bulk_persistence/capture_timings.py index 10e3f1e7..fafe58ff 100644 --- a/app/bulk_persistence/capture_timings.py +++ b/app/bulk_persistence/capture_timings.py @@ -4,7 +4,7 @@ from functools import wraps import asyncio from time import perf_counter, process_time -from ..helper.logger import get_logger +from app.helper.logger import get_logger def make_log_captured_timing_handler(level=INFO): diff --git a/app/bulk_persistence/dask/bulk_catalog.py b/app/bulk_persistence/dask/bulk_catalog.py index 3f091266..22e2b2d1 100644 --- a/app/bulk_persistence/dask/bulk_catalog.py +++ b/app/bulk_persistence/dask/bulk_catalog.py @@ -22,7 +22,7 @@ from contextlib import suppress from dataclasses import dataclass from typing import Dict, Iterable, List, NamedTuple, Optional, Set -from ...helper.traces import with_trace +from app.helper.traces import with_trace from ..capture_timings import capture_timings from .storage_path_builder import join, remove_protocol from .utils import worker_capture_timing_handlers diff --git a/app/bulk_persistence/dask/session_file_meta.py b/app/bulk_persistence/dask/session_file_meta.py index 448bd1d4..ae8f2faa 100644 --- a/app/bulk_persistence/dask/session_file_meta.py +++ b/app/bulk_persistence/dask/session_file_meta.py @@ -23,8 +23,8 @@ from distributed.worker import get_client import pandas as pd from .utils import share_items -from ...helper.logger import get_logger -from ...helper.traces import with_trace +from app.helper.logger import get_logger +from app.helper.traces import with_trace from ..sessions_storage import Session from ..capture_timings import capture_timings diff --git a/app/bulk_persistence/dask/utils.py b/app/bulk_persistence/dask/utils.py index 350f1d73..06750ef6 100644 --- a/app/bulk_persistence/dask/utils.py +++ b/app/bulk_persistence/dask/utils.py @@ -20,7 +20,7 @@ import dask.dataframe as dd import pandas as pd import pyarrow.parquet as pa -from ...helper.logger import get_logger +from app.helper.logger import get_logger from ..capture_timings import capture_timings diff --git a/app/bulk_persistence/dataframe_persistence.py b/app/bulk_persistence/dataframe_persistence.py index 6f677a59..89d5585a 100644 --- a/app/bulk_persistence/dataframe_persistence.py +++ b/app/bulk_persistence/dataframe_persistence.py @@ -30,7 +30,7 @@ from .bulk_id import new_bulk_id from .dask.errors import internal_bulk_exceptions from .mime_types import MimeTypes, MimeType from .tenant_provider import resolve_tenant -from ..helper.traces import with_trace +from app.helper.traces import with_trace async def create_and_store_dataframe(ctx: Context, df: pd.DataFrame) -> str: diff --git a/app/bulk_persistence/dataframe_serializer.py b/app/bulk_persistence/dataframe_serializer.py index a2b059ea..84cdee1c 100644 --- a/app/bulk_persistence/dataframe_serializer.py +++ b/app/bulk_persistence/dataframe_serializer.py @@ -24,7 +24,7 @@ from pydantic import BaseModel from .json_orient import JSONOrient from .mime_types import MimeTypes, MimeType from app.pool_executor import get_pool_executor -from ..helper.traces import with_trace +from app.helper.traces import with_trace class DataframeSerializerSync: -- GitLab From 678915739698698234032c1167d58d7facf9ca5f Mon Sep 17 00:00:00 2001 From: Luc Yriarte Date: Tue, 12 Apr 2022 14:39:47 +0200 Subject: [PATCH 12/12] Top level bulk persistence imports for tests not directly related to persistence code --- tests/unit/model/log_bulk_test.py | 2 +- tests/unit/routers/filter_test.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/unit/model/log_bulk_test.py b/tests/unit/model/log_bulk_test.py index b5483639..31fd7ca4 100644 --- a/tests/unit/model/log_bulk_test.py +++ b/tests/unit/model/log_bulk_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. from app.model.log_bulk import LogBulkHelper -from app.bulk_persistence.bulk_uri import BulkURI +from app.bulk_persistence import BulkURI from tests.unit.test_utils import basic_record import uuid import pytest diff --git a/tests/unit/routers/filter_test.py b/tests/unit/routers/filter_test.py index b7a76ba3..2cebb338 100644 --- a/tests/unit/routers/filter_test.py +++ b/tests/unit/routers/filter_test.py @@ -1,8 +1,7 @@ import pytest -from app.bulk_persistence.dask.errors import FilterError from app.bulk_persistence import GetDataParams, \ - BulkReadFilterOperator, BulkReadFilters + BulkReadFilterOperator, BulkReadFilters, FilterError @pytest.mark.parametrize("filters, expected", [ -- GitLab