Commit 6d0f82c2 authored by Jørgen Lind's avatar Jørgen Lind
Browse files

Update wavelet for uint8/16 formats

parent 2331ccee
......@@ -169,6 +169,11 @@ target_include_directories(openvds_objects PRIVATE "VDS")
target_compile_definitions(openvds_objects PRIVATE openvds_EXPORTS)
target_compile_definitions(openvds_objects PRIVATE -DHUEBDS_EXPORTS= ) # We're not building HueBulkDataStore as a shared library
#needed for ENABLE_SSE_TRANSFORM
if (NOT (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC"))
target_compile_options(openvds_objects PRIVATE -msse4.1)
endif()
addSystemHeadersToTarget(openvds_objects "${include_3rdparty}")
if (WIN32)
......
......@@ -90,7 +90,8 @@ bool VolumeDataStore::Verify(const VolumeDataChunk &volumeDataChunk, const std::
int32_t createSizeZ = waveletHeader[4];
int32_t dimensions = waveletHeader[5];
isValid = dataVersion == WAVELET_DATA_VERSION_1_4 &&
isValid = dataVersion >= WAVELET_DATA_VERSION_1_4 &&
dataVersion <= WAVELET_DATA_VERSION_1_5 &&
(compressedSize <= int32_t(serializedData.size()) || !isFullyRead) &&
(createSizeX == voxelSize[0] ) &&
(createSizeY == voxelSize[1] || dimensions < 2) &&
......@@ -269,7 +270,7 @@ bool DeserializeVolumeData(const std::vector<uint8_t> &serializedData, VolumeDat
int32_t dataVersion = ((int32_t *)data)[0];
(void)dataVersion;
assert(dataVersion == WAVELET_DATA_VERSION_1_4);
assert(dataVersion >= WAVELET_DATA_VERSION_1_4 && dataVersion <= WAVELET_DATA_VERSION_1_5);
bool isNormalize = false;
bool isLossless = false;
......
......@@ -38,6 +38,36 @@
namespace OpenVDS
{
template<typename T>
void convertFloatToIntegerType(const DataBlock &sourceDataBlock, const std::vector<uint8_t> &sourceData, const DataBlock &targetDataBlock, std::vector<uint8_t> &targetData)
{
assert(sourceDataBlock.Format == VolumeDataChannelDescriptor::Format_R32);
int32_t sourceSizeX = sourceDataBlock.Size[0];
int32_t sourceSizeY = sourceDataBlock.Size[1];
int32_t sourceSizeZ = sourceDataBlock.Size[2];
int32_t sourceAllocatedSizeX = sourceDataBlock.AllocatedSize[0];
int32_t sourceAllocatedSizeY = sourceDataBlock.AllocatedSize[1];
uint32_t sourceElementSize = GetElementSize(sourceDataBlock);
int32_t targetAllocatedSizeX = targetDataBlock.AllocatedSize[0];
int32_t targetAllocatedSizeY = targetDataBlock.AllocatedSize[1];
uint32_t targetElementSize = GetElementSize(targetDataBlock);
for (int32_t iZ = 0; iZ < sourceSizeZ; iZ++)
{
for (int32_t iY = 0; iY < sourceSizeY; iY++)
{
T* target = reinterpret_cast<T*>(targetData.data() + (iZ * targetAllocatedSizeY * targetAllocatedSizeX + iY * targetAllocatedSizeX) * targetElementSize);
const float * source = reinterpret_cast<const float *>(sourceData.data() + (iZ * sourceAllocatedSizeY * sourceAllocatedSizeX + iY * sourceAllocatedSizeX) * sourceElementSize);
for (int32_t iX = 0; iX < sourceSizeX; iX++, target++, source++)
{
*target = T(*source);
}
}
}
}
bool Wavelet_Decompress(const void *compressedData, int nCompressedAdaptiveDataSize, VolumeDataChannelDescriptor::Format dataBlockFormat, const FloatRange &valueRange, float integerScale, float integerOffset, bool isUseNoValue, float noValue, bool isNormalize, int nDecompressLevel, bool isLossless, DataBlock &dataBlock, std::vector<uint8_t> &target, Error &error)
{
if (isLossless)
......@@ -53,9 +83,12 @@ bool Wavelet_Decompress(const void *compressedData, int nCompressedAdaptiveDataS
createSize[0] = intHeaderPtr[2];
createSize[1] = intHeaderPtr[3];
createSize[2] = intHeaderPtr[4];
int32_t dimensions = intHeaderPtr[5];
int32_t dimensions = intHeaderPtr[5] & 0xff;
uint32_t integerInfo = (intHeaderPtr[5] >> 8) & 0xff;
if (dataVersion != WAVELET_DATA_VERSION_1_4 ||
if (dataVersion < WAVELET_DATA_VERSION_1_4 ||
dataVersion > WAVELET_DATA_VERSION_1_5 ||
dimensions < 1 ||
dimensions > 3 ||
createSize[0] < 0 ||
......@@ -74,7 +107,7 @@ bool Wavelet_Decompress(const void *compressedData, int nCompressedAdaptiveDataS
return false;
target.resize(GetAllocatedByteSize(dataBlock));
Wavelet wavelet(compressedData,
Wavelet wavelet(integerInfo, compressedData,
dataBlock.Size[0],
dataBlock.Size[1],
dataBlock.Size[2],
......@@ -84,6 +117,17 @@ bool Wavelet_Decompress(const void *compressedData, int nCompressedAdaptiveDataS
dimensions,
dataVersion);
if (integerInfo & WAVELET_INTEGERINFO_ISINTEGER)
{
// Check if lossless pass was used (U32)
if (!(integerInfo & WAVELET_INTEGERINFO_ISCOMPRESSEDWITHDIFFPASS))
{
// don't do lossless pass
isLossless = false;
}
}
float threshold;
float startThreshold;
float waveletNoValue;
......@@ -93,6 +137,26 @@ bool Wavelet_Decompress(const void *compressedData, int nCompressedAdaptiveDataS
if (!wavelet.DeCompress(true, -1, -1, -1, &startThreshold, &threshold, dataBlock.Format, valueRange, integerScale, integerOffset, isUseNoValue, noValue, &isAnyNoValue, &waveletNoValue, isNormalize, nDecompressLevel, isLossless, nCompressedAdaptiveDataSize, dataBlock, target, error))
return false;
if (dataBlock.Format != dataBlockFormat)
{
DataBlock finalDataBlock;
if (!InitializeDataBlock(dataBlockFormat, VolumeDataChannelDescriptor::Components_1, Dimensionality(dimensions), createSize, finalDataBlock, error))
return false;
std::vector<uint8_t> finalDataTarget;
finalDataTarget.resize(GetAllocatedByteSize(finalDataBlock));
if (finalDataBlock.Format == VolumeDataChannelDescriptor::Format_U8)
{
convertFloatToIntegerType<uint8_t>(dataBlock, target, finalDataBlock, finalDataTarget);
}
else
{
convertFloatToIntegerType<uint16_t>(dataBlock, target, finalDataBlock, finalDataTarget);
}
target = std::move(finalDataTarget);
dataBlock = finalDataBlock;
}
return true;
}
......@@ -357,10 +421,11 @@ static int CalculateBufferSizeNeeded(int maxPixels, int maxChildren)
return size;
}
Wavelet::Wavelet(const void *compressedData, int32_t transformSizeX, int32_t transformSizeY, int32_t transformSizeZ, int32_t allocatedSizeX, int32_t allocatedSizeY, int32_t allocatedSizeZ, int32_t dimensions, int32_t dataVersion)
Wavelet::Wavelet(uint32_t integerInfo, const void *compressedData, int32_t transformSizeX, int32_t transformSizeY, int32_t transformSizeZ, int32_t allocatedSizeX, int32_t allocatedSizeY, int32_t allocatedSizeZ, int32_t dimensions, int32_t dataVersion)
{
m_readCompressedData = (const uint32_t *)compressedData;
m_noValueData = nullptr;
m_integerInfo = integerInfo;
m_dataVersion = dataVersion;
m_dimensions = dimensions;
......@@ -693,7 +758,7 @@ static void DecompressZerosAlongX(const uint8_t *in, void *pic, int elementSize,
bool Wavelet::DeCompress(bool isTransform, int32_t decompressInfo, float decompressSlice, int32_t decompressFlip, float* startThreshold, float* threshold, VolumeDataChannelDescriptor::Format dataBlockFormat, const FloatRange& valueRange, float integerScale, float integerOffset, bool isUseNoValue, float noValue, bool* isAnyNoValue, float* waveletNoValue, bool isNormalize, int decompressLevel, bool isLossless, int compressedAdaptiveDataSize, DataBlock& dataBlock, std::vector<uint8_t>& target, Error& error)
{
assert(m_dataVersion == WAVELET_DATA_VERSION_1_4);
assert(m_dataVersion >= WAVELET_DATA_VERSION_1_4 && m_dataVersion <= WAVELET_DATA_VERSION_1_5);
InitCoder();
int32_t *startOfCompressedData = (int32_t *)m_readCompressedData;
......@@ -767,9 +832,10 @@ bool Wavelet::DeCompress(bool isTransform, int32_t decompressInfo, float decompr
std::vector<uint8_t> cpuTempData;
cpuTempData.resize(cpuTempDecodeSizeNeeded);
bool isInteger = m_integerInfo & WAVELET_INTEGERINFO_ISINTEGER;
WaveletAdaptiveLL_DecodeIterator decodeIterator = WaveletAdaptiveLL_CreateDecodeIterator((uint8_t*)m_readCompressedData, floatReadWriteData, m_allocatedSizeX, m_allocatedSizeY, m_allocatedSizeZ, *threshold, *startThreshold, m_transformMask, transformData, m_transformIterations,
m_pixelSetChildren.get(), m_pixelSetChildrenCount, m_pixelSetPixelInSignificant.get(), m_pixelSetPixelInSignificantCount,
m_allocatedHalfSizeX, m_allocatedHalfSizeX * m_allocatedHalfSizeY, cpuTempData.data(), m_allocatedHalfSizeX * m_allocatedHalfSizeY * m_allocatedHalfSizeZ, m_allocatedSizeX * m_allocatedSizeY * m_allocatedSizeZ, decompressLevel);
m_allocatedHalfSizeX, m_allocatedHalfSizeX * m_allocatedHalfSizeY, cpuTempData.data(), m_allocatedHalfSizeX * m_allocatedHalfSizeY * m_allocatedHalfSizeZ, m_allocatedSizeX * m_allocatedSizeY * m_allocatedSizeZ, decompressLevel, isInteger);
int size = WaveletAdaptiveLL_DecompressAdaptive(decodeIterator);
(void)size;
......@@ -909,13 +975,13 @@ void Wavelet::InverseTransform(float *source)
#pragma omp parallel for num_threads(threadCount) schedule(guided)
for (int32_t iD1 = 0; iD1 < bandSize[1]; ++iD1)
{
Wavelet_InverseTransformSliceInterleave(tempBuffer.data() + iD1 * bufferPitchX, bufferPitchXY, source + iD1 * m_allocatedSizeX, m_allocatedSizeXY, bandSizeX, bandSizeZ);
Wavelet_InverseTransformSliceInterleave(tempBuffer.data() + iD1 * bufferPitchX, bufferPitchXY, source + iD1 * m_allocatedSizeX, m_allocatedSizeXY, bandSizeX, bandSizeZ, m_integerInfo);
}
#pragma omp parallel for num_threads(threadCount) schedule(guided)
for (int32_t iD2 = 0; iD2 < bandSize[2]; ++iD2)
{
Wavelet_InverseTransformSlice(tempBuffer.data() + iD2 * bufferPitchXY, bufferPitchX, tempBuffer.data() + iD2 * bufferPitchXY, bufferPitchX, bandSizeX, bandSizeY);
Wavelet_InverseTransformSlice(tempBuffer.data() + iD2 * bufferPitchXY, bufferPitchX, tempBuffer.data() + iD2 * bufferPitchXY, bufferPitchX, bandSizeX, bandSizeY, m_integerInfo);
for (int32_t iD1 = 0; iD1 < bandSize[1]; ++iD1)
{
......@@ -924,7 +990,7 @@ void Wavelet::InverseTransform(float *source)
// Wavelet transform x
float *readLine = tempBuffer.data() + (iD2 * bufferPitchXY + iD1Interleaved * bufferPitchX);
Wavelet_InverseTransformLine(readLine, bandSizeX);
Wavelet_InverseTransformLine(readLine, bandSizeX, m_integerInfo);
Wavelet_InterleaveLine(source + (iD1 * m_allocatedSizeX + iD2 * m_allocatedSizeXY), readLine, readLine + ((bandSizeX + 1) >> 1), bandSizeX);
}
}
......@@ -936,7 +1002,7 @@ void Wavelet::InverseTransform(float *source)
#pragma omp parallel for num_threads(threadCount) schedule(guided)
for (int32_t iD1 = 0; iD1 < bandSize[1]; ++iD1)
{
Wavelet_InverseTransformSliceInterleave(write + iD1 * writePitchX, writePitchXY, read + iD1 * readPitchX, readPitchXY, bandSizeX, bandSizeZ);
Wavelet_InverseTransformSliceInterleave(write + iD1 * writePitchX, writePitchXY, read + iD1 * readPitchX, readPitchXY, bandSizeX, bandSizeZ, m_integerInfo);
}
read = write;
......@@ -953,7 +1019,7 @@ void Wavelet::InverseTransform(float *source)
#pragma omp parallel for num_threads(threadCount) schedule(guided)
for (int32_t iD2 = 0; iD2 < bandSize[2]; ++iD2)
{
Wavelet_InverseTransformSliceInterleave(write + iD2 * writePitchXY, writePitchX, read + iD2 * readPitchXY, readPitchX, bandSizeX, bandSizeY);
Wavelet_InverseTransformSliceInterleave(write + iD2 * writePitchXY, writePitchX, read + iD2 * readPitchXY, readPitchX, bandSizeX, bandSizeY, m_integerInfo);
}
read = write;
......@@ -975,7 +1041,7 @@ void Wavelet::InverseTransform(float *source)
// Wavelet transform x
float *readDisplaced = read + (iD1 * readPitchX + iD2 * readPitchXY);
Wavelet_InverseTransformLine(readDisplaced, bandSizeX);
Wavelet_InverseTransformLine(readDisplaced, bandSizeX, m_integerInfo);
Wavelet_InterleaveLine(write + (iD1 * writePitchX + iD2 * writePitchXY), readDisplaced, readDisplaced + ((bandSizeX + 1) >> 1), bandSizeX);
}
}
......
......@@ -56,6 +56,7 @@ class Wavelet
int32_t m_allocatedHalfSizeY;
int32_t m_allocatedHalfSizeZ;
int32_t m_pixelSetChildrenCount;
uint32_t m_integerInfo;
std::unique_ptr<float[]> m_maxTree;
std::unique_ptr<float[]> m_maxTreeMaxOnly;
......@@ -70,10 +71,10 @@ class Wavelet
int32_t m_pixelSetPixelInSignificantCount;
public:
Wavelet(const void *compressedData, int32_t transformSizeX, int32_t transformSizeY, int32_t transformSizeZ, int32_t allocatedSizeX, int32_t allocatedSizeY, int32_t allocatedSizeZ, int32_t dimensions, int32_t dataVersion);
Wavelet(uint32_t integerIfno, const void *compressedData, int32_t transformSizeX, int32_t transformSizeY, int32_t transformSizeZ, int32_t allocatedSizeX, int32_t allocatedSizeY, int32_t allocatedSizeZ, int32_t dimensions, int32_t dataVersion);
void InitCoder();
bool DeCompress(bool isTransform, int32_t decompressInfo, float decompressSlice, int32_t decompressFlip, float *startThreshold, float *threshold, VolumeDataChannelDescriptor::Format dataBlockFormat, const FloatRange &valueRange, float integerScale, float integerOffset, bool isUseNoValue, float noValue, bool *isAnyNoValue, float *waveletNoValue, bool isNormalize, int decompressLevel, bool isLossless, int compressedAdaptiveDataSize, DataBlock &dataBlock, std::vector<uint8_t> &target, Error &error);
bool DeCompress(bool isTransform, int32_t decompressInfo, float decompressSlice, int32_t decompressFlip, float *startThreshold, float *threshold, VolumeDataChannelDescriptor::Format dataBlockFormat, const FloatRange &originalValueRange, float integerScale, float integerOffset, bool isUseNoValue, float noValue, bool *isAnyNoValue, float *waveletNoValue, bool isNormalize, int decompressLevel, bool isLossless, int compressedAdaptiveDataSize, DataBlock &dataBlock, std::vector<uint8_t> &target, Error &error);
void DeCompressNoValuesHeader();
void InverseTransform(float *source);
void DeCompressNoValues(float* noValue, std::vector<uint32_t> &buffer);
......
......@@ -207,12 +207,13 @@ static int32_t CompileTransformData(uint8_t* compiledTransformData, int32_t* fir
WaveletAdaptiveLL_DecodeIterator WaveletAdaptiveLL_CreateDecodeIterator(uint8_t* stream, float* picture, int sizeX, int sizeY, int sizeZ,
const float threshold, const float startThreshold, int* transformMask, Wavelet_TransformData* transformData, int transformDataCount,
Wavelet_PixelSetChildren* pixelSetChildren, int pixelSetChildrenCount, Wavelet_PixelSetPixel* pixelSetPixelInSignificant, int pixelSetPixelInsignificantCount,
int maxSizeX, int maxSizeXY, uint8_t* tempBufferCPU, int maxChildren, int maxPixels, int decompressLevel)
int maxSizeX, int maxSizeXY, uint8_t* tempBufferCPU, int maxChildren, int maxPixels, int decompressLevel, bool isInteger)
{
int sizeXY = sizeX * sizeY;
WaveletAdaptiveLL_DecodeIterator decodeIterator;
decodeIterator.isInteger = isInteger;
decodeIterator.sizeX = sizeX;
decodeIterator.sizeY = sizeY;
decodeIterator.sizeZ = sizeZ;
......@@ -955,6 +956,13 @@ static void DecodeAllBits(const WaveletAdaptiveLL_DecodeIterator& decodeIterator
int threads = std::min(8, omp_get_max_threads());
(void)threads;
float startValue = 0.0f;
if (!decodeIterator.isInteger)
{
startValue = threshold * 0.5f;
}
#pragma omp parallel for schedule(dynamic, 256) num_threads(threads)
for (int32_t parentValue = 0; parentValue < values; parentValue++)
{
......@@ -965,7 +973,7 @@ static void DecodeAllBits(const WaveletAdaptiveLL_DecodeIterator& decodeIterator
int32_t bit = value & 7;
int32_t bytePos = value >> 3;
float rvalue = threshold * 0.5f;
float rvalue = startValue;
float currentThreshold = minLevelThreshold;
......
......@@ -70,6 +70,7 @@ struct WaveletAdaptiveLL_DecodeIterator
float startThreshold;
bool isAllNormal;
bool isInteger;
uint8_t transformMask[12];
uint8_t children[8];
......@@ -84,7 +85,7 @@ struct WaveletAdaptiveLL_DecodeIterator
WaveletAdaptiveLL_DecodeIterator WaveletAdaptiveLL_CreateDecodeIterator(uint8_t *streamCPU, float *pictureCPU, int sizeX, int sizeY, int sizeZ,
const float threshold, const float startThreshold, int *transformMask, Wavelet_TransformData *transformData, int transformDataCount,
Wavelet_PixelSetChildren *pixelSetChildren, int pixelSetChildrenCount, Wavelet_PixelSetPixel *pixelSetPixelInSignificant, int pixelSetPixelInsignificantCount,
int maxSizeX, int maxSizeXY, uint8_t *tempBufferCPU, int maxChildren, int maxPixels, int decompressLevel);
int maxSizeX, int maxSizeXY, uint8_t *tempBufferCPU, int maxChildren, int maxPixels, int decompressLevel, bool isInteger);
int32_t WaveletAdaptiveLL_DecompressAdaptive(WaveletAdaptiveLL_DecodeIterator decodeIterator);
int32_t WaveletAdaptiveLL_DecompressLossless(uint8_t *in, float *pic, int32_t sizeX, int32_t sizeY, int32_t sizeZ, int32_t allocatedSizeX, int32_t allocatedSizeXY);
......
This diff is collapsed.
......@@ -25,6 +25,7 @@
#define WAVELET_MIN_COMPRESSION_TOLERANCE 0.01f
#define WAVELET_DATA_VERSION_1_4 (671) // progressive wavelet transform
#define WAVELET_DATA_VERSION_1_5 (672) // U8, U16 and U32 native/lossless compression
#define WAVELET_MIN_COMPRESSED_HEADER (6 * 4)
......@@ -43,6 +44,14 @@
namespace OpenVDS
{
enum Wavelet_IntegerInfo
{
WAVELET_INTEGERINFO_ISINTEGER = (1 << 0),
WAVELET_INTEGERINFO_ISLOSSLESSOPTIMIZED = (1 << 1),
WAVELET_INTEGERINFO_ISCOMPRESSEDWITHDIFFPASS = (1 << 2),
WAVELET_INTEGERINFO_16BIT = (1 << 3)
};
bool Wavelet_Decompress(const void *compressedData, int nCompressedAdaptiveDataSize, VolumeDataChannelDescriptor::Format dataBlockFormat, const FloatRange &valueRange, float integerScale, float integerOffset, bool isUseNoValue, float noValue, bool isNormalize, int nDecompressLevel, bool isLossless, DataBlock &dataBlock, std::vector<uint8_t> &target, Error &error);
struct Wavelet_PixelSetPixel
......
......@@ -71,9 +71,10 @@ add_test_executable(segy_tests
../src/SEGYUtils/SEGYUtils/SEGYFileInfo.h)
target_include_directories(segy_tests PRIVATE ../src/SEGYUtils)
add_test_executable(deserialize VDS/DeserializeVolumeDataTest.cpp)
add_test_executable(vds_integration_tests
VDS/ParseVDSJsonTest.cpp
VDS/DeserializeVolumeDataTest.cpp
VDS/RequestVolumeCleanupThread.cpp
VDS/ParseConnectionStringTest.cpp
)
......
......@@ -20,6 +20,8 @@
#include <VDS/VolumeDataStore.h>
#include <VDS/DataBlock.h>
#include <IO/File.h>
#include <OpenVDS/ValueConversion.h>
#include <OpenVDS/Range.h>
#include <cstdlib>
#include <cmath>
......@@ -31,50 +33,61 @@ namespace OpenVDS
bool DeserializeVolumeData(const std::vector<uint8_t>& serializedData, VolumeDataChannelDescriptor::Format format, CompressionMethod compressionMethod, const FloatRange& valueRange, float integerScale, float integerOffset, bool isUseNoValue, float noValue, int32_t adaptiveLevel, DataBlock& dataBlock, std::vector<uint8_t>& destination, Error& error);
}
static void Stats(const OpenVDS::DataBlock &aDataBlock, const std::vector<uint8_t> &aData, const OpenVDS::DataBlock &bDataBlock, const std::vector<uint8_t> &bData, double &diff, double &maxError, double &deviation, double &samples)
template<typename T>
float getErrorDelta(const OpenVDS::QuantizingValueConverterWithNoValue<T, float, false> &resultConverter, const void *referenceDataPtr, const void *waveletDataPtr, int offset)
{
T waveletValue = *(reinterpret_cast<const T *>(waveletDataPtr) + offset);
float waveletFloatValue = float(waveletValue);
float referenceValue = *(reinterpret_cast<const float *>(referenceDataPtr) + offset);
T referenceValueConverted = resultConverter.ConvertValue(referenceValue);
float referenceValueConvertedFloat = float(referenceValueConverted);
return waveletFloatValue - referenceValueConvertedFloat;
}
static void Stats(const OpenVDS::FloatRange &valueRange, const OpenVDS::DataBlock &referenceDataBlock, const std::vector<uint8_t> &referenceData, const OpenVDS::DataBlock &waveletDataBlock, const std::vector<uint8_t> &waveletData, float &diff, float &maxError, float &deviation, float &samples)
{
diff = 0.0;
samples = 0.0;
assert(aDataBlock.Size[0] == bDataBlock.Size[0] &&
aDataBlock.Size[1] == bDataBlock.Size[1] &&
aDataBlock.Size[2] == bDataBlock.Size[2]);
assert(referenceDataBlock.Size[0] == waveletDataBlock.Size[0] &&
referenceDataBlock.Size[1] == waveletDataBlock.Size[1] &&
referenceDataBlock.Size[2] == waveletDataBlock.Size[2]);
assert(aDataBlock.Format == bDataBlock.Format);
assert(referenceDataBlock.Format == OpenVDS::VolumeDataChannelDescriptor::Format_R32);
OpenVDS::VolumeDataChannelDescriptor::Format format = aDataBlock.Format;
const void *a = bData.data();
OpenVDS::VolumeDataChannelDescriptor::Format waveletFormat = waveletDataBlock.Format;
const void *referenceDataPtr = referenceData.data();
const void *b = aData.data();
const void *waveletDataPtr = waveletData.data();
double localAverage = 0.0;
float localAverage = 0.0;
maxError = 0.0;
OpenVDS::QuantizingValueConverterWithNoValue<uint8_t, float, false> resultConverter8(valueRange.Min, valueRange.Max, 1.0f, 0.0f, 0.0f, 0.0f);
OpenVDS::QuantizingValueConverterWithNoValue<uint16_t, float, false> resultConverter16(valueRange.Min, valueRange.Max, 1.0f, 0.0f, 0.0f, 0.0f);
for (int i2 = 0; i2 < aDataBlock.Size[2]; i2++)
for (int i2 = 0; i2 < referenceDataBlock.Size[2]; i2++)
{
for (int i1 = 0; i1 < aDataBlock.Size[1]; i1++)
for (int i1 = 0; i1 < referenceDataBlock.Size[1]; i1++)
{
for (int i0 = 0; i0 < aDataBlock.Size[0]; i0++)
for (int i0 = 0; i0 < referenceDataBlock.Size[0]; i0++)
{
int32_t offset = i0 + i1 * aDataBlock.Pitch[1] + i2 * aDataBlock.Pitch[2];
int32_t offset = i0 + i1 * referenceDataBlock.Pitch[1] + i2 * referenceDataBlock.Pitch[2];
double errorDelta;
float errorDelta;
if (format == OpenVDS::VolumeDataChannelDescriptor::Format_R32)
if (waveletFormat == OpenVDS::VolumeDataChannelDescriptor::Format_R32)
{
errorDelta = *(((float *)b) + offset) - *(((float *)a) + offset);
errorDelta = *((float *)waveletDataPtr + offset) - *((float *)referenceDataPtr + offset);
}
else if (format == OpenVDS::VolumeDataChannelDescriptor::Format_U16)
else if (waveletFormat == OpenVDS::VolumeDataChannelDescriptor::Format_U16)
{
errorDelta = (float)*(((uint16_t *)b) + offset) - (float)*(((uint16_t*)a) + offset);
errorDelta /= 65535.0f;
errorDelta = getErrorDelta<uint16_t>(resultConverter16, referenceDataPtr, waveletDataPtr, offset);
}
else if (format == OpenVDS::VolumeDataChannelDescriptor::Format_U8)
else if (waveletFormat == OpenVDS::VolumeDataChannelDescriptor::Format_U8)
{
errorDelta = (float)*(((uint8_t *)b) + offset) - (float)*(((uint8_t *)a) + offset);
errorDelta /= 255.0f;
errorDelta = getErrorDelta<uint8_t>(resultConverter8, referenceDataPtr, waveletDataPtr, offset);
}
else
{
......@@ -91,32 +104,29 @@ static void Stats(const OpenVDS::DataBlock &aDataBlock, const std::vector<uint8_
}
}
double average = localAverage / samples;
float average = localAverage / samples;
deviation = 0.0;
for (int i2 = 0; i2 < aDataBlock.Size[2]; i2++)
{
for (int i1 = 0; i1 < aDataBlock.Size[1]; i1++)
for (int i2 = 0; i2 < referenceDataBlock.Size[2]; i2++)
{ for (int i1 = 0; i1 < referenceDataBlock.Size[1]; i1++)
{
for (int i0 = 0; i0 < aDataBlock.Size[0]; i0++)
for (int i0 = 0; i0 < referenceDataBlock.Size[0]; i0++)
{
int32_t offset = i0 + i1 * aDataBlock.Pitch[1] + i2 * aDataBlock.Pitch[2];
int32_t offset = i0 + i1 * referenceDataBlock.Pitch[1] + i2 * referenceDataBlock.Pitch[2];
double errorDelta;
float errorDelta;
if (format == OpenVDS::VolumeDataChannelDescriptor::Format_R32)
if (waveletFormat == OpenVDS::VolumeDataChannelDescriptor::Format_R32)
{
errorDelta = *(((float *)b) + offset) - *(((float *)a) + offset);
errorDelta = *(((float *)waveletDataPtr) + offset) - *(((float *)referenceDataPtr) + offset);
}
else if (format == OpenVDS::VolumeDataChannelDescriptor::Format_U16)
else if (waveletFormat == OpenVDS::VolumeDataChannelDescriptor::Format_U16)
{
errorDelta = (float)*(((uint16_t *)b) + offset) - (float)*(((uint16_t *)a) + offset);
errorDelta /= 65535.0f;
errorDelta = getErrorDelta<uint16_t>(resultConverter16, referenceDataPtr, waveletDataPtr, offset);
}
else if (format == OpenVDS::VolumeDataChannelDescriptor::Format_U8)
else if (waveletFormat == OpenVDS::VolumeDataChannelDescriptor::Format_U8)
{
errorDelta = (float)*(((uint8_t *)b) + offset) - (float)*(((uint8_t *)a) + offset);
errorDelta /= 255.0f;
errorDelta = getErrorDelta<uint8_t>(resultConverter8, referenceDataPtr, waveletDataPtr, offset);
}
else
{
......@@ -146,63 +156,130 @@ static std::vector<uint8_t> LoadTestFile(const std::string &file)
return serializedData;
}
GTEST_TEST(VDS_integration, DeSerializeVolumeData)
float getOnePercentValueRange(const OpenVDS::FloatRange &range, OpenVDS::VolumeDataChannelDescriptor::Format format)
{
OpenVDS::Error error;
if (format == OpenVDS::VolumeDataChannelDescriptor::Format_R32)
{
return (range.Max - range.Min) / 100.0f;
}
else if (format == OpenVDS::VolumeDataChannelDescriptor::Format_U8)
{
return float(std::numeric_limits<uint8_t>::max()) / 100.0f;
}
else if (format == OpenVDS::VolumeDataChannelDescriptor::Format_U16)
{
return float(std::numeric_limits<uint16_t>::max()) / 100.0f;
}
std::vector<uint8_t> serializedNone = LoadTestFile("/chunk.CompressionMethod_None");
std::vector<uint8_t> dataNone;
OpenVDS::DataBlock dataBlockNone;
OpenVDS::DeserializeVolumeData(serializedNone, OpenVDS::VolumeDataChannelDescriptor::Format_R32, OpenVDS::CompressionMethod::None, OpenVDS::FloatRange(-0.07883811742067337f, 0.07883811742067337f), 1.0f, 0.0f, false, 0.0f, 0, dataBlockNone, dataNone, error);
EXPECT_EQ(error.code, 0);
std::vector<uint8_t> serializedWavelet = LoadTestFile("/chunk.CompressionMethod_Wavelet");
return 0.0;
}
void verify_wavelet(const OpenVDS::FloatRange &valueRange, const OpenVDS::DataBlock &dataBlockNone, const std::vector<uint8_t> &dataNone, const std::string &file, OpenVDS::VolumeDataChannelDescriptor::Format format)
{
OpenVDS::Error error;
std::vector<uint8_t> serializedWavelet = LoadTestFile(file);
std::vector<uint8_t> dataWavelet;
OpenVDS::DataBlock dataBlockWavelet;
OpenVDS::DeserializeVolumeData(serializedWavelet, OpenVDS::VolumeDataChannelDescriptor::Format_R32, OpenVDS::CompressionMethod::Wavelet, OpenVDS::FloatRange(-0.07883811742067337f, 0.07883811742067337f), 1.0f, 0.0f, false, 0.0f, 0, dataBlockWavelet, dataWavelet, error);
OpenVDS::DeserializeVolumeData(serializedWavelet, format, OpenVDS::CompressionMethod::Wavelet, valueRange, 1.0f, 0.0f, false, 0.0f, 0, dataBlockWavelet, dataWavelet, error);
EXPECT_EQ(error.code, 0);
EXPECT_EQ(dataNone.size(), dataWavelet.size());
double diff;
double maxError;
double deviation;
double samples = 0;
Stats(dataBlockNone, dataNone, dataBlockWavelet, dataWavelet, diff, maxError, deviation, samples);
int dataNoneScale = OpenVDS::GetVoxelFormatByteSize(dataBlockNone.Format);
int compressedDataScale = OpenVDS::GetVoxelFormatByteSize(format);
EXPECT_EQ(dataNone.size() / dataNoneScale, dataWavelet.size() / compressedDataScale);
float diff;
float maxError;
float deviation;
float samples = 0;
Stats(valueRange, dataBlockNone, dataNone, dataBlockWavelet, dataWavelet, diff, maxError, deviation, samples);
//double variance = deviation / samples;
//double std_dev = sqrt(variance);
double avg_diff = diff / samples;
double one_procent_range = (0.07883811742067337 + 0.07883811742067337) / 100;
double one_procent_range = getOnePercentValueRange(valueRange, format);
EXPECT_TRUE(avg_diff < one_procent_range * 2);
}
template<typename T>
void verify_lossless_typed(const OpenVDS::FloatRange &valueRange, const OpenVDS::DataBlock &referenceDataBlock, const std::vector<uint8_t> &referenceData, const std::vector<uint8_t> &deserializedData)
{
OpenVDS::QuantizingValueConverterWithNoValue<T, float, false> resultConverter(valueRange.Min, valueRange.Max, 1.0f, 0.0f, 0.0f, 0.0f);
const T* deserializedDataPtr = reinterpret_cast<const T*>(deserializedData.data());
const float* referenceDataPtr = reinterpret_cast<const float*>(referenceData.data());
for (int i2 = 0; i2 < referenceDataBlock.Size[2]; i2++)
{
for (int i1 = 0; i1 < referenceDataBlock.Size[1]; i1++)
{
for (int i0 = 0; i0 < referenceDataBlock.Size[0]; i0++)
{
int32_t offset = i0 + i1 * referenceDataBlock.Pitch[1] + i2 * referenceDataBlock.Pitch[2];
T convertedReferenceValue = resultConverter.ConvertValue(referenceDataPtr[offset]);
T deserializedValue = deserializedDataPtr[offset];
if (deserializedValue != convertedReferenceValue)
{
fmt::print(stderr, "Failed to verify equality: non_compression = {}, compressed = {}\n", convertedReferenceValue, deserializedValue);
EXPECT_TRUE(false);
}
}
}
}
}
std::vector<uint8_t> serializedWaveletLossless = LoadTestFile("/chunk.CompressionMethod_WaveletLossless");
std::vector<uint8_t> dataWaveletLossless;
OpenVDS::DataBlock dataBlockWaveletLossless;
OpenVDS::DeserializeVolumeData(serializedWaveletLossless, OpenVDS::VolumeDataChannelDescriptor::Format_R32, OpenVDS::CompressionMethod::WaveletLossless, OpenVDS::FloatRange(-0.07883811742067337f, 0.07883811742067337f), 1.0f, 0.0f, false, 0.0f, -1, dataBlockWaveletLossless, dataWaveletLossless, error);