From cd7889b2e18d95ace60fb6630de3c34fa65b4a3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Lind?= Date: Tue, 2 Mar 2021 13:53:56 +0100 Subject: [PATCH 1/3] Use GetSerializationTargetbufferSize --- src/OpenVDS/VDS/Rle.cpp | 7 ------- src/OpenVDS/VDS/Rle.h | 7 +++++++ src/OpenVDS/VDS/VolumeDataStore.cpp | 32 +++++++++++++++++++++-------- 3 files changed, 31 insertions(+), 15 deletions(-) diff --git a/src/OpenVDS/VDS/Rle.cpp b/src/OpenVDS/VDS/Rle.cpp index 5f66e64b..75d589d6 100644 --- a/src/OpenVDS/VDS/Rle.cpp +++ b/src/OpenVDS/VDS/Rle.cpp @@ -29,13 +29,6 @@ namespace OpenVDS { -struct RLEHeader -{ - int32_t compressedSize; - int32_t originalSize; - int32_t rleUnitSize; -}; - template static uint64_t RlePack(T value) { diff --git a/src/OpenVDS/VDS/Rle.h b/src/OpenVDS/VDS/Rle.h index 65b0a912..5a792132 100644 --- a/src/OpenVDS/VDS/Rle.h +++ b/src/OpenVDS/VDS/Rle.h @@ -23,6 +23,13 @@ namespace OpenVDS { +struct RLEHeader +{ + int32_t compressedSize; + int32_t originalSize; + int32_t rleUnitSize; +}; + int32_t RleDecompress(uint8_t *target, int32_t targetSize, uint8_t* source); } diff --git a/src/OpenVDS/VDS/VolumeDataStore.cpp b/src/OpenVDS/VDS/VolumeDataStore.cpp index 8bffb875..ed9ac556 100644 --- a/src/OpenVDS/VDS/VolumeDataStore.cpp +++ b/src/OpenVDS/VDS/VolumeDataStore.cpp @@ -253,6 +253,27 @@ static uint64_t GetConstantValueVolumeDataHash(const DataBlock &dataBlock, cons } } +static int64_t GetSerializationTargetBufferSize(int64_t sourceSize, CompressionMethod compressionMethod) +{ + if(CompressionMethod_IsWavelet(compressionMethod)) + { + return std::max(int64_t(128*128*128*2), sourceSize) * 2 * 2; + } + else if(compressionMethod == CompressionMethod::RLE) + { + return sizeof(DataBlockDescriptor) + sizeof(RLEHeader) + sourceSize * 2; + } + else if(compressionMethod == CompressionMethod::Zip) + { + return sizeof(DataBlockDescriptor) + compressBound((uLong)sourceSize); + } + else + { + assert(compressionMethod == CompressionMethod::None); + return sizeof(DataBlockDescriptor) + sourceSize; + } +} + bool DeserializeVolumeData(const std::vector &serializedData, VolumeDataChannelDescriptor::Format format, CompressionMethod compressionMethod, const FloatRange &valueRange, float integerScale, float integerOffset, bool isUseNoValue, float noValue, int32_t adaptiveLevel, DataBlock &dataBlock, std::vector &destination, Error &error) { if(CompressionMethod_IsWavelet(compressionMethod)) @@ -563,19 +584,16 @@ VolumeDataStore::SerializeVolumeData(const VolumeDataChunk& chunk, const DataBlo dataBlockHeader.SizeY = dataBlock.Size[1]; dataBlockHeader.SizeZ = dataBlock.Size[2]; + destinationBuffer.resize(size_t(GetSerializationTargetBufferSize(int64_t(chunkData.size()), compressionMethod))); switch (compressionMethod) { case CompressionMethod::None: { - int32_t byteSize = GetByteSize(dataBlock) + sizeof(DataBlockDescriptor); - - destinationBuffer.resize(byteSize); - void *targetBuffer = destinationBuffer.data(); memcpy(targetBuffer, &dataBlockHeader, sizeof(dataBlockHeader)); targetBuffer = ((uint8_t *)targetBuffer) + sizeof(dataBlockHeader); - bool isConstant = CopyDataBlockIntoLinearBuffer(dataBlock, chunkData.data(), targetBuffer, byteSize - sizeof(dataBlockHeader)); + bool isConstant = CopyDataBlockIntoLinearBuffer(dataBlock, chunkData.data(), targetBuffer, int32_t(destinationBuffer.size() - sizeof(dataBlockHeader))); if(isConstant) { @@ -597,12 +615,10 @@ VolumeDataStore::SerializeVolumeData(const VolumeDataChunk& chunk, const DataBlo auto& layer = *chunk.layer; return GetConstantValueVolumeDataHash(dataBlock, tmpdata.get(), layer.GetValueRange(), layer.GetIntegerScale(), layer.GetIntegerOffset(), layer.IsUseNoValue(), layer.GetNoValue()); } - unsigned long compressedMaxSize = compressBound(tmpbuffersize); - destinationBuffer.resize(compressedMaxSize + sizeof(dataBlockHeader)); void *targetBuffer = destinationBuffer.data(); memcpy(targetBuffer, &dataBlockHeader, sizeof(dataBlockHeader)); targetBuffer = ((uint8_t *)targetBuffer) + sizeof(dataBlockHeader); - unsigned long compressedSize = compressedMaxSize; + unsigned long compressedSize = (unsigned long)destinationBuffer.size(); int status = compress((uint8_t *)targetBuffer, &compressedSize, tmpdata.get(), tmpbuffersize); destinationBuffer.resize(compressedSize + sizeof(dataBlockHeader)); -- GitLab From f62031caed2f464d0794b58a6bd71c6265ba47b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Lind?= Date: Tue, 2 Mar 2021 14:22:19 +0100 Subject: [PATCH 2/3] Support clang compiler on msvc --- CMake/SetCompilerFlagsForTarget.cmake | 5 +++-- src/OpenVDS/IO/IOManagerDms.cpp | 2 +- tests/OpenVDS/GlobalStateTest.cpp | 3 +-- tests/OpenVDS/MultiThreadedRequests.cpp | 3 +-- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/CMake/SetCompilerFlagsForTarget.cmake b/CMake/SetCompilerFlagsForTarget.cmake index 41fdf142..ae095d61 100644 --- a/CMake/SetCompilerFlagsForTarget.cmake +++ b/CMake/SetCompilerFlagsForTarget.cmake @@ -6,12 +6,13 @@ function(setCompilerFlagsForTarget target) if(MSVC_VERSION GREATER_EQUAL 1915) target_compile_options(${target} PRIVATE $<$>:/JMC>) endif() - target_compile_options(${target} PRIVATE /MP) else() set_target_properties(${target} PROPERTIES LINK_FLAGS_RELEASE -s) endif() - if (NOT (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")) + if (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + target_compile_options(${target} PRIVATE /MP) + else() target_compile_options(${target} PRIVATE -Werror=return-type) endif() endfunction() diff --git a/src/OpenVDS/IO/IOManagerDms.cpp b/src/OpenVDS/IO/IOManagerDms.cpp index 1ca9167d..4e79fad8 100644 --- a/src/OpenVDS/IO/IOManagerDms.cpp +++ b/src/OpenVDS/IO/IOManagerDms.cpp @@ -94,7 +94,7 @@ namespace OpenVDS //no headers are exposed - request_ptr->m_done; + request_ptr->m_done = true; request_ptr->m_handler->Completed(*request_ptr, request_ptr->m_error); }); } diff --git a/tests/OpenVDS/GlobalStateTest.cpp b/tests/OpenVDS/GlobalStateTest.cpp index af089f68..eac7cbd5 100644 --- a/tests/OpenVDS/GlobalStateTest.cpp +++ b/tests/OpenVDS/GlobalStateTest.cpp @@ -62,7 +62,6 @@ TEST(GlobalState, basic) SlowIOManager* slowIOManager = new SlowIOManager(requestDelayMs, inMemory.get()); std::unique_ptr handle(OpenVDS::Open(slowIOManager, error), OpenVDS::Close); - OpenVDS::VolumeDataLayout *layout = OpenVDS::GetLayout(handle.get()); OpenVDS::VolumeDataAccessManager accessManager = OpenVDS::GetAccessManager(handle.get()); ThreadPool threadPool(threadCount); @@ -101,7 +100,7 @@ TEST(GlobalState, basic) requestData.maxPos[2] = requestData.minPos[2] + width_dist(gen); requestData.voxelCount = (requestData.maxPos[0] - requestData.minPos[0]) * (requestData.maxPos[1] - requestData.minPos[1]) * (requestData.maxPos[2] - requestData.minPos[2]); } - auto threadedRequest = [&threadRequestData, accessManager, layout] () mutable + auto threadedRequest = [&threadRequestData, accessManager] () mutable { std::vector ret; ret.reserve(threadRequestData.size()); diff --git a/tests/OpenVDS/MultiThreadedRequests.cpp b/tests/OpenVDS/MultiThreadedRequests.cpp index 68ca8e3e..7fd42227 100644 --- a/tests/OpenVDS/MultiThreadedRequests.cpp +++ b/tests/OpenVDS/MultiThreadedRequests.cpp @@ -51,7 +51,6 @@ TEST(Multithreading, requests) SlowIOManager* slowIOManager = new SlowIOManager(requestDelayMs, inMemory.get()); std::unique_ptr handle(OpenVDS::Open(slowIOManager, error), OpenVDS::Close); - OpenVDS::VolumeDataLayout *layout = OpenVDS::GetLayout(handle.get()); OpenVDS::VolumeDataAccessManager accessManager = OpenVDS::GetAccessManager(handle.get()); ThreadPool threadPool(threadCount); @@ -93,7 +92,7 @@ TEST(Multithreading, requests) requestData.maxPos[2] = requestData.minPos[2] + width_dist(gen); requestData.voxelCount = (requestData.maxPos[0] - requestData.minPos[0]) * (requestData.maxPos[1] - requestData.minPos[1]) * (requestData.maxPos[2] - requestData.minPos[2]); } - auto threadedRequest = [&threadRequestData, accessManager, layout] () mutable + auto threadedRequest = [&threadRequestData, accessManager] () mutable { std::vector ret; ret.reserve(threadRequestData.size()); -- GitLab From fe12d80dedc5ee1beb94ba36a97b41215e5287b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Lind?= Date: Tue, 2 Mar 2021 16:45:53 +0100 Subject: [PATCH 3/3] Add shrink to compressed size --- src/OpenVDS/VDS/VolumeDataStore.cpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/OpenVDS/VDS/VolumeDataStore.cpp b/src/OpenVDS/VDS/VolumeDataStore.cpp index ed9ac556..430afd51 100644 --- a/src/OpenVDS/VDS/VolumeDataStore.cpp +++ b/src/OpenVDS/VDS/VolumeDataStore.cpp @@ -573,6 +573,18 @@ bool VolumeDataStore::DeserializeVolumeData(const VolumeDataChunk& volumeDataChu return ret; } +struct ShrinkToSizeOnExit +{ + ShrinkToSizeOnExit(std::vector& to_shrink) + : to_shrink(to_shrink) + {} + ~ShrinkToSizeOnExit() + { + to_shrink.shrink_to_fit(); + } + std::vector to_shrink; +}; + uint64_t VolumeDataStore::SerializeVolumeData(const VolumeDataChunk& chunk, const DataBlock& dataBlock, const std::vector& chunkData, CompressionMethod compressionMethod, float, std::vector& destinationBuffer) { @@ -585,6 +597,8 @@ VolumeDataStore::SerializeVolumeData(const VolumeDataChunk& chunk, const DataBlo dataBlockHeader.SizeZ = dataBlock.Size[2]; destinationBuffer.resize(size_t(GetSerializationTargetBufferSize(int64_t(chunkData.size()), compressionMethod))); + ShrinkToSizeOnExit autoShrinker(destinationBuffer); + switch (compressionMethod) { case CompressionMethod::None: -- GitLab