VolumeDataStore.cpp 25.3 KB
Newer Older
Jørgen Lind's avatar
Jørgen Lind committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
/****************************************************************************
** Copyright 2019 The Open Group
** Copyright 2019 Bluware, Inc.
**
** Licensed under the Apache License, Version 2.0 (the "License");
** you may not use this file except in compliance with the License.
** You may obtain a copy of the License at
**
**     http://www.apache.org/licenses/LICENSE-2.0
**
** Unless required by applicable law or agreed to in writing, software
** distributed under the License is distributed on an "AS IS" BASIS,
** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
** See the License for the specific language governing permissions and
** limitations under the License.
****************************************************************************/

#include "VolumeDataStore.h"

20
21
#include "VolumeDataLayoutImpl.h"

Jørgen Lind's avatar
Jørgen Lind committed
22
23
#include "Wavelet.h"
#include "VolumeDataHash.h"
Jørgen Lind's avatar
Jørgen Lind committed
24
#include "Rle.h"
Jørgen Lind's avatar
Jørgen Lind committed
25
#include "DataBlock.h"
26
#include "ParsedMetadata.h"
Jørgen Lind's avatar
Jørgen Lind committed
27
#include "CompilerDefines.h"
28
#include <OpenVDS/ValueConversion.h>
29
#include <VDS/VDS.h>
Jørgen Lind's avatar
Jørgen Lind committed
30
#include <VDS/GlobalStateImpl.h>
Jørgen Lind's avatar
Jørgen Lind committed
31

32
#include <fmt/format.h>
Jørgen Lind's avatar
Jørgen Lind committed
33
34
35
36

#include <stdlib.h>
#include <assert.h>

Jørgen Lind's avatar
Jørgen Lind committed
37
38
#include <zlib.h>

Jørgen Lind's avatar
Jørgen Lind committed
39
40
namespace OpenVDS
{
Jørgen Lind's avatar
Jørgen Lind committed
41
42
43
44
45
46
47
48
49
50

VolumeDataStore::VolumeDataStore(OpenOptions::ConnectionType connectionType)
  : m_globalStateVds(static_cast<GlobalStateImpl *>(GetGlobalState())->downloaded[connectionType],
                     static_cast<GlobalStateImpl *>(GetGlobalState())->downloadedChunks[connectionType],
                     static_cast<GlobalStateImpl *>(GetGlobalState())->decompressed[connectionType],
                     static_cast<GlobalStateImpl *>(GetGlobalState())->decompressedChunks[connectionType])
{
}


51
static bool CompressionMethodIsWavelet(CompressionMethod compressionMethod)
Jørgen Lind's avatar
Jørgen Lind committed
52
53
54
55
56
57
58
{
  return compressionMethod == CompressionMethod::Wavelet ||
         compressionMethod == CompressionMethod::WaveletNormalizeBlock ||
         compressionMethod == CompressionMethod::WaveletLossless ||
         compressionMethod == CompressionMethod::WaveletNormalizeBlockLossless;
}

59
static uint32_t GetByteSize(const DataBlockDescriptor &descriptor)
Jørgen Lind's avatar
Jørgen Lind committed
60
61
{
  int32_t size[DataStoreDimensionality_Max];
62
63
64
  size[0] = descriptor.SizeX;
  size[1] = descriptor.SizeY;
  size[2] = descriptor.SizeZ;
Jørgen Lind's avatar
Jørgen Lind committed
65
  size[3] = 1;
66
  return GetByteSize(size, descriptor.Format, descriptor.Components);
Jørgen Lind's avatar
Jørgen Lind committed
67
68
}

69
bool VolumeDataStore::Verify(const VolumeDataChunk &volumeDataChunk, const std::vector<uint8_t> &serializedData, CompressionMethod compressionMethod, bool isFullyRead)
Jørgen Lind's avatar
Jørgen Lind committed
70
71
72
73
74
{
  bool isValid = false;

  int32_t voxelSize[DataStoreDimensionality_Max];

75
  volumeDataChunk.layer->GetChunkVoxelSize(volumeDataChunk.index, voxelSize);
Jørgen Lind's avatar
Jørgen Lind committed
76

77
  if(serializedData.empty())
Jørgen Lind's avatar
Jørgen Lind committed
78
79
80
  {
    isValid = true;
  }
81
  else if(CompressionMethodIsWavelet(compressionMethod))
Jørgen Lind's avatar
Jørgen Lind committed
82
83
84
85
86
87
88
89
90
91
92
93
  {
    if(serializedData.size() >= sizeof(int32_t) * 6)
    {
      const int32_t *waveletHeader = (const int32_t *)serializedData.data();

      int32_t dataVersion    = waveletHeader[0];
      int32_t compressedSize = waveletHeader[1];
      int32_t createSizeX    = waveletHeader[2];
      int32_t createSizeY    = waveletHeader[3];
      int32_t createSizeZ    = waveletHeader[4];
      int32_t dimensions     = waveletHeader[5];

94
95
      isValid = dataVersion >= WAVELET_DATA_VERSION_1_4 &&
                dataVersion <= WAVELET_DATA_VERSION_1_5 &&
Jørgen Lind's avatar
Jørgen Lind committed
96
                (compressedSize <= int32_t(serializedData.size()) || !isFullyRead) &&
Jørgen Lind's avatar
Jørgen Lind committed
97
98
99
100
101
102
103
104
                (createSizeX == voxelSize[0]                  ) &&
                (createSizeY == voxelSize[1] || dimensions < 2) &&
                (createSizeZ == voxelSize[2] || dimensions < 3) &&
                dimensions >= 1 &&
                dimensions <= 3;
    }
  }
  else if(compressionMethod == CompressionMethod::None ||
105
          compressionMethod == CompressionMethod::RLE ||
Jørgen Lind's avatar
Jørgen Lind committed
106
107
108
109
110
          compressionMethod == CompressionMethod::Zip)
  {
    if(serializedData.size() > sizeof(DataBlockDescriptor))
    {
      DataBlockDescriptor *serializedDescriptor = (DataBlockDescriptor*)serializedData.data();
111
      isValid = serializedDescriptor->IsValid(voxelSize);
Jørgen Lind's avatar
Jørgen Lind committed
112
113
114
115
116
117
    }
  }

  return isValid;
}

118
static void CopyLinearBufferIntoDataBlock(const void *sourceBuffer, const DataBlock &dataBlock, std::vector<uint8_t> &targetBuffer)
Jørgen Lind's avatar
Jørgen Lind committed
119
120
{

121
122
123
  int32_t sizeX = dataBlock.Size[0];
  int32_t sizeY = dataBlock.Size[1];
  int32_t sizeZ = dataBlock.Size[2];
Jørgen Lind's avatar
Jørgen Lind committed
124

125
126
  int32_t allocatedSizeX = dataBlock.AllocatedSize[0];
  int32_t allocatedSizeY = dataBlock.AllocatedSize[1];
Jørgen Lind's avatar
Jørgen Lind committed
127

128
  if(dataBlock.Format == VolumeDataChannelDescriptor::Format_1Bit)
Jørgen Lind's avatar
Jørgen Lind committed
129
  {
130
    sizeX = ((sizeX * dataBlock.Components) + 7) / 8;
Jørgen Lind's avatar
Jørgen Lind committed
131
132
  }

133
  uint32_t elementSize = GetElementSize(dataBlock);
Jørgen Lind's avatar
Jørgen Lind committed
134
135
136
137
138

  for(int32_t iZ = 0; iZ < sizeZ; iZ++)
  {
    for(int32_t iY = 0; iY < sizeY; iY++)
    {
Jørgen Lind's avatar
Jørgen Lind committed
139
      uint8_t *target = targetBuffer.data()                             +  (iZ * allocatedSizeY + iY) * allocatedSizeX * elementSize;
Jørgen Lind's avatar
Jørgen Lind committed
140
      const uint8_t *source = static_cast<const uint8_t*>(sourceBuffer) +  (iZ * sizeY          + iY) * sizeX          * elementSize;
Jørgen Lind's avatar
Jørgen Lind committed
141
      memcpy(target, source, size_t(sizeX) * elementSize);
Jørgen Lind's avatar
Jørgen Lind committed
142
143
144
145
    }
  }
}

146
static bool CopyDataBlockIntoLinearBuffer(const DataBlock &dataBlock, const void *sourceBuffer, void *targetBuffer, int32_t bufferSize)
147
148
{
  int32_t size[DataStoreDimensionality_Max];
149
  memcpy(size, dataBlock.Size, sizeof(size));
150
  int32_t allocatedSize[DataStoreDimensionality_Max];
151
  memcpy(allocatedSize, dataBlock.AllocatedSize, sizeof(allocatedSize));
152

153
  int32_t elementSize = int32_t(GetElementSize(dataBlock));
154

155
  if(dataBlock.Format == VolumeDataChannelDescriptor::Format_1Bit)
156
  {
157
    size[0] = ((size[0] * dataBlock.Components) + 7) / 8;
158
159
160
161
  }

  // Check if first row is constant
  bool isConstant = true;
162
  switch(dataBlock.Format)
163
164
165
  {
  default:
    fprintf(stderr, "Illegal format\n");
Jørgen Lind's avatar
Jørgen Lind committed
166
    return true;
167
168
  case VolumeDataChannelDescriptor::Format_1Bit:
    isConstant = (reinterpret_cast<const uint8_t*>(sourceBuffer)[0] == 0x00 || reinterpret_cast<const uint8_t*>(sourceBuffer)[0] == 0xff);
Jørgen Lind's avatar
Jørgen Lind committed
169
    FALLTHROUGH;
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
  case VolumeDataChannelDescriptor::Format_U8:
    for(int32_t iX = 1; isConstant && iX < size[0]; iX++)
    {
      isConstant = reinterpret_cast<const uint8_t *>(sourceBuffer)[0] == reinterpret_cast<const uint8_t *>(sourceBuffer)[iX];
    }
    break;
  case VolumeDataChannelDescriptor::Format_U16:
    for(int32_t iX = 1; isConstant && iX < size[0]; iX++)
    {
      isConstant = reinterpret_cast<const uint16_t *>(sourceBuffer)[0] == reinterpret_cast<const uint16_t *>(sourceBuffer)[iX];
    }
    break;

  case VolumeDataChannelDescriptor::Format_R32:
  case VolumeDataChannelDescriptor::Format_U32:
    for(int32_t iX = 1; isConstant && iX < size[0]; iX++)
    {
      isConstant = reinterpret_cast<const uint32_t *>(sourceBuffer)[0] == reinterpret_cast<const uint32_t *>(sourceBuffer)[iX];
    }
    break;

  case VolumeDataChannelDescriptor::Format_U64:
  case VolumeDataChannelDescriptor::Format_R64:
    for(int32_t iX = 1; isConstant && iX < size[0]; iX++)
    {
      isConstant = reinterpret_cast<const uint64_t *>(sourceBuffer)[0] == reinterpret_cast<const uint64_t *>(sourceBuffer)[iX];
    }
    break;
  }

  assert(bufferSize >= size[0] * size[1] * size[2] * elementSize);

  for(int32_t iZ = 0; iZ < size[2]; iZ++)
  {
    for(int32_t iY = 0; iY < size[1]; iY++)
    {
      uint8_t *puSource = (uint8_t *)sourceBuffer;
      uint8_t *puTarget = (uint8_t *)targetBuffer;

      puSource += (iZ * allocatedSize[1] + iY) * allocatedSize[0] * elementSize;
      puTarget += (iZ * size[1]          + iY) * size[0]          * elementSize;

      if(isConstant)
      {
        isConstant = (memcmp(sourceBuffer, puSource, size[0] * elementSize) == 0);
      }

      memcpy(puTarget, puSource, size[0] * elementSize);
    }
  }

  return isConstant;
}

template<typename T>
225
static uint64_t GetConstantValueVolumeDataHash(T value, const Range<float> &valueRange, float integerScale, float integerOffset, bool isUseNoValue, float noValue)
226
227
228
{
  if (isUseNoValue)
  {
229
    if(value == ConvertNoValue<T>(noValue))
230
231
232
233
234
    {
      return VolumeDataHash::NOVALUE;
    }
    else
    {
235
      QuantizingValueConverterWithNoValue<float, T, true> converter(valueRange.Min, valueRange.Max, integerScale, integerOffset, noValue, noValue, false);
236
      return VolumeDataHash(converter.ConvertValue(value)).CalculateHash();
237
238
239
240
    }
  }
  else
  {
241
    QuantizingValueConverterWithNoValue<float, T, false> converter(valueRange.Min, valueRange.Max, integerScale, integerOffset, noValue, noValue, false);
242
    return VolumeDataHash(converter.ConvertValue(value)).CalculateHash();
243
244
245
  }
}

246
static uint64_t GetConstantValueVolumeDataHash(const DataBlock  &dataBlock, const uint8_t *sourceBuffer, const Range<float> &valueRange, float integerScale, float integerOffset, bool isUseNoValue, float noValue)
247
{
248
  switch(dataBlock.Format)
249
250
251
  {
  case VolumeDataChannelDescriptor::Format_1Bit:
    assert(reinterpret_cast<const uint8_t *>(sourceBuffer)[0] == 0x00 || reinterpret_cast<const uint8_t *>(sourceBuffer)[0] == 0xff);
252
253
254
255
256
257
258
    return VolumeDataHash((float)(reinterpret_cast<const uint8_t *>(sourceBuffer)[0] != 0)).CalculateHash();
  case VolumeDataChannelDescriptor::Format_U8:  return GetConstantValueVolumeDataHash(reinterpret_cast<const uint8_t  *>(sourceBuffer)[0], valueRange, integerScale, integerOffset, isUseNoValue, noValue);
  case VolumeDataChannelDescriptor::Format_U16: return GetConstantValueVolumeDataHash(reinterpret_cast<const uint16_t *>(sourceBuffer)[0], valueRange, integerScale, integerOffset, isUseNoValue, noValue);
  case VolumeDataChannelDescriptor::Format_R32: return GetConstantValueVolumeDataHash(reinterpret_cast<const float *>(sourceBuffer)[0], valueRange, integerScale, integerOffset, isUseNoValue, noValue);
  case VolumeDataChannelDescriptor::Format_U32: return GetConstantValueVolumeDataHash(reinterpret_cast<const uint32_t *>(sourceBuffer)[0], valueRange, integerScale, integerOffset, isUseNoValue, noValue);
  case VolumeDataChannelDescriptor::Format_R64: return GetConstantValueVolumeDataHash(reinterpret_cast<const double *>(sourceBuffer)[0], valueRange, integerScale, integerOffset, isUseNoValue, noValue);
  case VolumeDataChannelDescriptor::Format_U64: return GetConstantValueVolumeDataHash(reinterpret_cast<const uint64_t *>(sourceBuffer)[0], valueRange, integerScale, integerOffset, isUseNoValue, noValue);
259
260
261
262
263
264
265

  default:
    assert(0 && "Unknown format");
    return VolumeDataHash::UNKNOWN;
  }
}

266
bool DeserializeVolumeData(const std::vector<uint8_t> &serializedData, VolumeDataChannelDescriptor::Format format, CompressionMethod compressionMethod, const FloatRange &valueRange, float integerScale, float integerOffset, bool isUseNoValue, float noValue, int32_t adaptiveLevel, DataBlock &dataBlock, std::vector<uint8_t> &destination, Error &error)
Jørgen Lind's avatar
Jørgen Lind committed
267
{
268
  if(CompressionMethodIsWavelet(compressionMethod))
Jørgen Lind's avatar
Jørgen Lind committed
269
270
271
272
  {
    const void *data = serializedData.data();

    int32_t dataVersion = ((int32_t *)data)[0];
Jørgen Lind's avatar
Jørgen Lind committed
273
    (void)dataVersion;
274
    assert(dataVersion >= WAVELET_DATA_VERSION_1_4 && dataVersion <= WAVELET_DATA_VERSION_1_5);
Jørgen Lind's avatar
Jørgen Lind committed
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302

    bool isNormalize = false;
    bool isLossless = false;

    if((compressionMethod == CompressionMethod::WaveletNormalizeBlock) ||
       (compressionMethod == CompressionMethod::WaveletNormalizeBlockLossless))
    {
      isNormalize = true;
    }

    if((compressionMethod == CompressionMethod::WaveletLossless) ||
       (compressionMethod == CompressionMethod::WaveletNormalizeBlockLossless))
    {
      isLossless = true;
    }

    // if adaptive level is 0 or more, don't use lossless data
    if (adaptiveLevel >= 0)
    {
      isLossless = false;
    }
    else
    {
      assert(adaptiveLevel == -1);
      // Now we can use lossless if there is lossless data - set iAdaptivelevel to max adaptive.
      adaptiveLevel = 0;
    }

Jørgen Lind's avatar
Jørgen Lind committed
303
    if (!Wavelet_Decompress(data, int32_t(serializedData.size()), format, valueRange, integerScale, integerOffset, isUseNoValue, noValue, isNormalize, adaptiveLevel, isLossless, dataBlock, destination, error))
Jørgen Lind's avatar
Jørgen Lind committed
304
      return false;
Jørgen Lind's avatar
Jørgen Lind committed
305
  }
306
  else if(compressionMethod == CompressionMethod::RLE)
Jørgen Lind's avatar
Jørgen Lind committed
307
308
309
  {
    DataBlockDescriptor *dataBlockDescriptor = (DataBlockDescriptor *)serializedData.data();

310
    if(!dataBlockDescriptor->IsValid())
Jørgen Lind's avatar
Jørgen Lind committed
311
    {
312
313
      error.code = -1;
      error.string = "Failed to decode DataBlockDescriptor";
Jørgen Lind's avatar
Jørgen Lind committed
314
315
316
      return false;
    }

317
    if (!InitializeDataBlock(*dataBlockDescriptor, dataBlock, error))
Jørgen Lind's avatar
Jørgen Lind committed
318
319
320
321
      return false;

    void * source = dataBlockDescriptor + 1;

322
    int32_t byteSize = GetByteSize(*dataBlockDescriptor);
Jørgen Lind's avatar
Jørgen Lind committed
323
324
    std::unique_ptr<uint8_t[]>buffer(new uint8_t[byteSize]);

325
    int32_t decompressedSize = RleDecompress((uint8_t *)buffer.get(), byteSize, (uint8_t *)source);
Jørgen Lind's avatar
Jørgen Lind committed
326
    (void)decompressedSize;
Jørgen Lind's avatar
Jørgen Lind committed
327
328
    assert(decompressedSize == byteSize);

329
    int allocatedSize = GetAllocatedByteSize(dataBlock);
Jørgen Lind's avatar
Jørgen Lind committed
330
    destination.resize(allocatedSize);
331
    CopyLinearBufferIntoDataBlock(buffer.get(), dataBlock, destination);
Jørgen Lind's avatar
Jørgen Lind committed
332
333
334
335
336
  }
  else if(compressionMethod == CompressionMethod::Zip)
  {
    DataBlockDescriptor *dataBlockDescriptor = (DataBlockDescriptor *)serializedData.data();

337
    if(!dataBlockDescriptor->IsValid())
Jørgen Lind's avatar
Jørgen Lind committed
338
    {
339
340
      error.code = -1;
      error.string = "Failed to decode DataBlockDescriptor";
Jørgen Lind's avatar
Jørgen Lind committed
341
342
      return false;
    }
343
    if (!InitializeDataBlock(*dataBlockDescriptor, dataBlock, error))
Jørgen Lind's avatar
Jørgen Lind committed
344
345
346
347
      return false;

    void * source = dataBlockDescriptor + 1;

348
    int32_t byteSize = GetByteSize(*dataBlockDescriptor);
Jørgen Lind's avatar
Jørgen Lind committed
349
    std::unique_ptr<uint8_t[]> buffer(new uint8_t[byteSize]);
Jørgen Lind's avatar
Jørgen Lind committed
350
351
352

    unsigned long destLen = byteSize;

Jørgen Lind's avatar
Jørgen Lind committed
353
    int status = uncompress(buffer.get(), &destLen, (uint8_t *)source, uint32_t(serializedData.size() - sizeof(DataBlockDescriptor)));
Jørgen Lind's avatar
Jørgen Lind committed
354

Jørgen Lind's avatar
Jørgen Lind committed
355
356
    if (status != Z_OK)
    {
Jørgen Lind's avatar
Jørgen Lind committed
357
      fprintf(stderr, "zlib uncompress failed (status %d) in VolumeDataStore_c::DeSerialize\n", status);
Jørgen Lind's avatar
Jørgen Lind committed
358
      return false;
Jørgen Lind's avatar
Jørgen Lind committed
359
    }
Jørgen Lind's avatar
Jørgen Lind committed
360

361
    int allocatedSize = GetAllocatedByteSize(dataBlock);
Jørgen Lind's avatar
Jørgen Lind committed
362
    destination.resize(allocatedSize);
363
    CopyLinearBufferIntoDataBlock(buffer.get(), dataBlock, destination);
Jørgen Lind's avatar
Jørgen Lind committed
364
365
366
367
368
  }
  else if(compressionMethod == CompressionMethod::None)
  {
    DataBlockDescriptor *dataBlockDescriptor = (DataBlockDescriptor *)serializedData.data();

369
    if(!dataBlockDescriptor->IsValid())
Jørgen Lind's avatar
Jørgen Lind committed
370
    {
371
372
      error.code = -1;
      error.string = "Failed to decode DataBlockDescriptor";
Jørgen Lind's avatar
Jørgen Lind committed
373
374
      return false;
    }
375
    if (!InitializeDataBlock(*dataBlockDescriptor, dataBlock, error))
Jørgen Lind's avatar
Jørgen Lind committed
376
377
378
379
      return false;

    void * source = dataBlockDescriptor + 1;

380
381
382
383
384
385
386
387
388
389
    size_t sourceDataBlockSize = serializedData.size() - sizeof(*dataBlockDescriptor);
    size_t requiredDataBlockSize = size_t(GetByteSize(*dataBlockDescriptor));

    if (sourceDataBlockSize != requiredDataBlockSize)
    {
      error.string = "Required size of uncompressed chunk is not present. Possible data corruptions.";
      error.code = -1;
      return false;
    }

390
    int32_t byteSize = GetAllocatedByteSize(dataBlock);
391
    destination.resize(byteSize);
392
    CopyLinearBufferIntoDataBlock(source, dataBlock, destination);
Jørgen Lind's avatar
Jørgen Lind committed
393
394
  }

395
  if(dataBlock.Format != format)
Jørgen Lind's avatar
Jørgen Lind committed
396
  {
397
    error.string = "Formats doesn't match in deserialization\n";
398
    error.code = -1;
Jørgen Lind's avatar
Jørgen Lind committed
399
400
401
402
403
    return false;
  }
  return true;
}

404
static float GetConvertedConstantValue(VolumeDataChannelDescriptor const &volumeDataChannelDescriptor, VolumeDataChannelDescriptor::Format format, float noValue, VolumeDataHash const &constantValueVolumeDataHash)
Jørgen Lind's avatar
Jørgen Lind committed
405
406
407
{
  if(format == VolumeDataChannelDescriptor::Format_1Bit)
  {
408
    return float(constantValueVolumeDataHash.GetConstantValue(0) != 0);
Jørgen Lind's avatar
Jørgen Lind committed
409
410
411
  }
  else if(format == VolumeDataChannelDescriptor::Format_U8 || format == VolumeDataChannelDescriptor::Format_U16)
  {
412
    if(constantValueVolumeDataHash.IsNoValue())
Jørgen Lind's avatar
Jørgen Lind committed
413
414
415
    {
      if(format == VolumeDataChannelDescriptor::Format_U8)
      {
416
        return ConvertNoValue<uint8_t>(noValue);
Jørgen Lind's avatar
Jørgen Lind committed
417
418
419
420
      }
      else
      {
        assert(format == VolumeDataChannelDescriptor::Format_U16);
421
        return ConvertNoValue<uint16_t>(noValue);
Jørgen Lind's avatar
Jørgen Lind committed
422
423
424
      }
    }

425
    int32_t buckets = (format == VolumeDataChannelDescriptor::Format_U8 ? 256 : 65536) - volumeDataChannelDescriptor.IsUseNoValue();
Jørgen Lind's avatar
Jørgen Lind committed
426
427
428
429

    float reciprocalScale;
    float offset;

430
    if(volumeDataChannelDescriptor.GetFormat() != VolumeDataChannelDescriptor::Format_U8 && volumeDataChannelDescriptor.GetFormat() != VolumeDataChannelDescriptor::Format_U16)
Jørgen Lind's avatar
Jørgen Lind committed
431
    {
432
433
      offset = volumeDataChannelDescriptor.GetValueRange().Min;
      reciprocalScale = float(buckets - 1) / rangeSize(volumeDataChannelDescriptor.GetValueRange());
Jørgen Lind's avatar
Jørgen Lind committed
434
435
436
    }
    else
    {
437
438
      offset = volumeDataChannelDescriptor.GetIntegerOffset();
      reciprocalScale = 1.0f / volumeDataChannelDescriptor.GetIntegerScale();
Jørgen Lind's avatar
Jørgen Lind committed
439
440
    }

441
    return (float)QuantizeValueWithReciprocalScale(constantValueVolumeDataHash.GetConstantValue(noValue), offset, reciprocalScale, buckets);
Jørgen Lind's avatar
Jørgen Lind committed
442
443
444
445
  }
  else
  {
    assert(format == VolumeDataChannelDescriptor::Format_R32 || format == VolumeDataChannelDescriptor::Format_U32 || format == VolumeDataChannelDescriptor::Format_R64 || format == VolumeDataChannelDescriptor::Format_U64);
446
    return constantValueVolumeDataHash.GetConstantValue(noValue);
Jørgen Lind's avatar
Jørgen Lind committed
447
448
449
450
  }
}

template <typename T>
451
static void FillConstantValueBuffer(std::vector<uint8_t> &buffer, int32_t allocatedElements, float value)
Jørgen Lind's avatar
Jørgen Lind committed
452
{
453
  T v = ConvertValue<T>(value);
Jørgen Lind's avatar
Jørgen Lind committed
454
455
456
457
458
459
460
  T *b = reinterpret_cast<T *>(buffer.data());
  for(int32_t element = 0; element < allocatedElements; element++)
  {
    b[element] = v;
  }
}

461
bool VolumeDataStore::CreateConstantValueDataBlock(VolumeDataChunk const &volumeDataChunk, VolumeDataChannelDescriptor::Format format, float noValue, VolumeDataChannelDescriptor::Components components, VolumeDataHash const &constantValueVolumeDataHash, DataBlock &dataBlock, std::vector<uint8_t> &buffer, Error &error)
Jørgen Lind's avatar
Jørgen Lind committed
462
463
{
  int32_t size[4];
464
465
  volumeDataChunk.layer->GetChunkVoxelSize(volumeDataChunk.index, size);
  int32_t dimensionality = volumeDataChunk.layer->GetChunkDimensionality();
466
  if (!InitializeDataBlock(format, components, Dimensionality(dimensionality), size, dataBlock, error))
Jørgen Lind's avatar
Jørgen Lind committed
467
468
469
    return false;

 
470
  int32_t allocatedSize = GetAllocatedByteSize(dataBlock);
Jørgen Lind's avatar
Jørgen Lind committed
471
472
  buffer.resize(allocatedSize);

473
  int32_t allocatedElements = dataBlock.AllocatedSize[0] * dataBlock.AllocatedSize[1] * dataBlock.AllocatedSize[2] * dataBlock.AllocatedSize[3] * dataBlock.Components;
Jørgen Lind's avatar
Jørgen Lind committed
474

475
  float convertedConstantValue = GetConvertedConstantValue(volumeDataChunk.layer->GetVolumeDataChannelDescriptor(), format, noValue, constantValueVolumeDataHash);
Jørgen Lind's avatar
Jørgen Lind committed
476

477
  assert(dataBlock.Format == format);
Jørgen Lind's avatar
Jørgen Lind committed
478
479
480
481
482
483
484

  VolumeDataChannelDescriptor::Format effectiveFormat = format;

  // Use U8 format fill methods for 1-bit
  if(format == VolumeDataChannelDescriptor::Format_1Bit)
  {
    effectiveFormat = VolumeDataChannelDescriptor::Format_U8;
485
    convertedConstantValue = ConvertValue<bool>(convertedConstantValue) ? 255.0f : 0.0f;
Jørgen Lind's avatar
Jørgen Lind committed
486
487
488
489
490
  }

  switch (effectiveFormat)
  {
  default:
491
    error.code = -1;
492
    error.string = "Invalid format in createConstantValuedataBlock";
Jørgen Lind's avatar
Jørgen Lind committed
493
    return false;
494
495
496
497
498
499
  case VolumeDataChannelDescriptor::Format_U8:  FillConstantValueBuffer<uint8_t>(buffer, allocatedElements, convertedConstantValue); break;
  case VolumeDataChannelDescriptor::Format_U16: FillConstantValueBuffer<uint16_t>(buffer, allocatedElements, convertedConstantValue); break;
  case VolumeDataChannelDescriptor::Format_R32: FillConstantValueBuffer<float>(buffer, allocatedElements, convertedConstantValue); break;
  case VolumeDataChannelDescriptor::Format_U32: FillConstantValueBuffer<uint32_t>(buffer, allocatedElements, convertedConstantValue); break;
  case VolumeDataChannelDescriptor::Format_R64: FillConstantValueBuffer<double>(buffer, allocatedElements, convertedConstantValue); break;
  case VolumeDataChannelDescriptor::Format_U64: FillConstantValueBuffer<uint64_t>(buffer, allocatedElements, convertedConstantValue); break;
Jørgen Lind's avatar
Jørgen Lind committed
500
501
502
503
504
  }

  return true;
}

Jørgen Lind's avatar
Jørgen Lind committed
505
bool VolumeDataStore::DeserializeVolumeData(const VolumeDataChunk& volumeDataChunk, const std::vector<uint8_t>& serializedData, const std::vector<uint8_t>& metadata, CompressionMethod compressionMethod, int32_t adaptiveLevel, VolumeDataChannelDescriptor::Format loadFormat, DataBlock& dataBlock, std::vector<uint8_t>& target, Error& error)
506
{
507
  uint64_t volumeDataHashValue = VolumeDataHash::UNKNOWN;
508

509
510
  bool waveletAdaptive = CompressionMethodIsWavelet(compressionMethod) && metadata.size() == sizeof(uint64_t) + sizeof(uint8_t[WAVELET_ADAPTIVE_LEVELS]);
  if (!waveletAdaptive && metadata.size() != sizeof(uint64_t))
511
  {
512
513
514
    error.code = -1;
    error.string = fmt::format("Invalid metadata of size {} for chunk: {}/{}", metadata.size(), GetLayerName(*volumeDataChunk.layer), volumeDataChunk.index);
    return false;
515
516
  }

517
  memcpy(&volumeDataHashValue, metadata.data(), sizeof(uint64_t));
518

Jørgen Lind's avatar
Jørgen Lind committed
519
520
  VolumeDataHash volumeDataHash(volumeDataHashValue);

521
522
523
  VolumeDataLayer const *volumeDataLayer = volumeDataChunk.layer;

  if (volumeDataHash.IsConstant())
Jørgen Lind's avatar
Jørgen Lind committed
524
  {
525
    return CreateConstantValueDataBlock(volumeDataChunk, volumeDataLayer->GetFormat(), volumeDataLayer->GetNoValue(), volumeDataLayer->GetComponents(), volumeDataHash, dataBlock, target, error);
Jørgen Lind's avatar
Jørgen Lind committed
526
  }
527
  else if(serializedData.empty())
Jørgen Lind's avatar
Jørgen Lind committed
528
  {
529
    error.code = -1;
530
    error.string = fmt::format("Missing data for chunk: {}/{}", GetLayerName(*volumeDataChunk.layer), volumeDataChunk.index);
Jørgen Lind's avatar
Jørgen Lind committed
531
532
533
    return false;
  }

534
  if (!Verify(volumeDataChunk, serializedData, compressionMethod, !waveletAdaptive))
Jørgen Lind's avatar
Jørgen Lind committed
535
  {
536
537
538
    error.code = -1;
    error.string = fmt::format("Invalid header (e.g. unsupported Wavelet compression version) for chunk: {}/{}", GetLayerName(*volumeDataChunk.layer), volumeDataChunk.index);
    return false;
539
540
  }

541
  volumeDataHash = uint64_t(volumeDataHash) ^ (uint64_t(adaptiveLevel) + 1) * 0x4068934683409867ULL;
542

543
544
  //create a value range from scale and offset so that conversion to 8 or 16 bit is done correctly inside deserialization
  FloatRange deserializeValueRange = volumeDataLayer->GetValueRange();
545

546
547
548
549
550
551
552
553
554
555
556
  if (volumeDataLayer->GetFormat() == VolumeDataChannelDescriptor::Format_U16 || volumeDataLayer->GetFormat() == VolumeDataChannelDescriptor::Format_U8)
  {
    if (loadFormat == VolumeDataChannelDescriptor::Format_U16)
    {
      deserializeValueRange.Min = volumeDataLayer->GetIntegerOffset();
      deserializeValueRange.Max = volumeDataLayer->GetIntegerScale() * (volumeDataLayer->IsUseNoValue() ? 65534.0f : 65535.0f) + volumeDataLayer->GetIntegerOffset();
    }
    else if (loadFormat == VolumeDataChannelDescriptor::Format_U8 && volumeDataLayer->GetFormat() != VolumeDataChannelDescriptor::Format_U16)
    {
      deserializeValueRange.Min = volumeDataLayer->GetIntegerOffset();
      deserializeValueRange.Max = volumeDataLayer->GetIntegerScale() * (volumeDataLayer->IsUseNoValue() ? 254.0f : 255.0f) + volumeDataLayer->GetIntegerOffset();
Jørgen Lind's avatar
Jørgen Lind committed
557
558
559
    }
  }

Jørgen Lind's avatar
Jørgen Lind committed
560
561
562
  bool ret = OpenVDS::DeserializeVolumeData(serializedData, loadFormat, compressionMethod, deserializeValueRange, volumeDataLayer->GetIntegerScale(), volumeDataLayer->GetIntegerOffset(), volumeDataLayer->IsUseNoValue(), volumeDataLayer->GetNoValue(), adaptiveLevel, dataBlock, target, error);
  m_globalStateVds.addDecompressed(target.size());
  return ret;
Jørgen Lind's avatar
Jørgen Lind committed
563
}
Jørgen Lind's avatar
Jørgen Lind committed
564

565
566
uint64_t
VolumeDataStore::SerializeVolumeData(const VolumeDataChunk& chunk, const DataBlock& dataBlock, const std::vector<uint8_t>& chunkData, CompressionMethod compressionMethod, std::vector<uint8_t>& destinationBuffer)
Jørgen Lind's avatar
Jørgen Lind committed
567
{
568
  DataBlockDescriptor dataBlockHeader;
569
570
571
572
573
574
  dataBlockHeader.Components = dataBlock.Components;
  dataBlockHeader.Dimensionality = dataBlock.Dimensionality;
  dataBlockHeader.Format = dataBlock.Format;
  dataBlockHeader.SizeX = dataBlock.Size[0];
  dataBlockHeader.SizeY = dataBlock.Size[1];
  dataBlockHeader.SizeZ = dataBlock.Size[2];
575
576
577
578
579

  switch (compressionMethod)
  {
  case CompressionMethod::None:
  {
580
    int32_t byteSize = GetByteSize(dataBlock) + sizeof(DataBlockDescriptor);
581
582
583
584
585
586
587

    destinationBuffer.resize(byteSize);

    void *targetBuffer = destinationBuffer.data();
    memcpy(targetBuffer, &dataBlockHeader, sizeof(dataBlockHeader));
    targetBuffer = ((uint8_t *)targetBuffer) + sizeof(dataBlockHeader);

588
    bool isConstant = CopyDataBlockIntoLinearBuffer(dataBlock, chunkData.data(), targetBuffer, byteSize - sizeof(dataBlockHeader));
589
590
591

    if(isConstant)
    {
592
      destinationBuffer.resize(0);
593
      auto& layer = *chunk.layer;
594
      return GetConstantValueVolumeDataHash(dataBlock, (const uint8_t *) targetBuffer, layer.GetValueRange(), layer.GetIntegerScale(), layer.GetIntegerOffset(), layer.IsUseNoValue(), layer.GetNoValue());
595
596
597
598
599
    }
    break;
  }
  case CompressionMethod::Zip:
  {
600
    uint32_t tmpbuffersize = GetByteSize(dataBlock);
601
    std::unique_ptr<uint8_t[]> tmpdata(new uint8_t[tmpbuffersize]);
602
    bool isConstant = CopyDataBlockIntoLinearBuffer(dataBlock, chunkData.data(), tmpdata.get(), tmpbuffersize);
603
604
605

    if (isConstant)
    {
606
      destinationBuffer.resize(0);
607
      auto& layer = *chunk.layer;
608
      return GetConstantValueVolumeDataHash(dataBlock, tmpdata.get(), layer.GetValueRange(), layer.GetIntegerScale(), layer.GetIntegerOffset(), layer.IsUseNoValue(), layer.GetNoValue());
609
610
611
612
613
614
    }
    unsigned long compressedMaxSize = compressBound(tmpbuffersize);
    destinationBuffer.resize(compressedMaxSize + sizeof(dataBlockHeader));
    void *targetBuffer = destinationBuffer.data();
    memcpy(targetBuffer, &dataBlockHeader, sizeof(dataBlockHeader));
    targetBuffer = ((uint8_t *)targetBuffer) + sizeof(dataBlockHeader);
Jørgen Lind's avatar
Jørgen Lind committed
615
    unsigned long compressedSize = compressedMaxSize;
616
617
618
619
620
    int status = compress((uint8_t *)targetBuffer, &compressedSize, tmpdata.get(), tmpbuffersize);
    destinationBuffer.resize(compressedSize + sizeof(dataBlockHeader));

    if (status != Z_OK)
    {
621
      throw std::runtime_error("zlib compression failed");
622
623
624
625
    }
    break;
  }
  default:
626
    throw std::runtime_error("Invalid compression method specified when serializing a VolumeDataChunk");
627
  }
628
  return VolumeDataHash::UNKNOWN;
Jørgen Lind's avatar
Jørgen Lind committed
629
630
}

Jørgen Lind's avatar
Jørgen Lind committed
631
}