VolumeDataAccessManagerImpl.h 14.4 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
/****************************************************************************
** Copyright 2019 The Open Group
** Copyright 2019 Bluware, Inc.
**
** Licensed under the Apache License, Version 2.0 (the "License");
** you may not use this file except in compliance with the License.
** You may obtain a copy of the License at
**
**     http://www.apache.org/licenses/LICENSE-2.0
**
** Unless required by applicable law or agreed to in writing, software
** distributed under the License is distributed on an "AS IS" BASIS,
** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
** See the License for the specific language governing permissions and
** limitations under the License.
****************************************************************************/

#ifndef VOLUMEDATAACCESSMANAGERIMPL_H
#define VOLUMEDATAACCESSMANAGERIMPL_H

#include <OpenVDS/VolumeDataAccess.h>

#include "IntrusiveList.h"
#include "VolumeDataPageAccessorImpl.h"
#include <IO/IOManager.h>

27
28
#include "VolumeDataChunk.h"
#include "VolumeDataLayer.h"
Jørgen Lind's avatar
Jørgen Lind committed
29
#include "VolumeDataRequestProcessor.h"
30
31
32
33

#include <map>
#include "Base64.h"

34
35
namespace OpenVDS
{
36
37
class MetadataPage;

38
class ReadChunkTransfer : public TransferDownloadHandler
39
40
41
42
43
44
45
{
public:
  ReadChunkTransfer(CompressionMethod compressionMethod, int adaptiveLevel)
    : m_compressionMethod(compressionMethod)
    , m_adaptiveLevel(adaptiveLevel)
  {}

Jørgen Lind's avatar
Jørgen Lind committed
46
  ~ReadChunkTransfer() override
47
48
49
  {
  }

50
  void HandleMetadata(const std::string& key, const std::string& header) override
51
  {
52
    if (key == "vdschunkmetadata")
53
    {
Morten Ofstad's avatar
Morten Ofstad committed
54
      if (!Base64Decode(header.data(), (int)header.size(), m_metadata))
55
      {
56
57
        m_error.code = -1;
        m_error.string = "Failed to decode chunk metadata";
58
59
60
61
      }
    }
  }

62
  void HandleData(std::vector<uint8_t>&& data) override
63
64
65
  {
    m_data = data;
  }
66

67
  void Completed(const Request &req, const Error & error) override
68
69
70
71
72
73
74
75
76
77
78
79
80
  {
    m_error = error;
  }
  
  CompressionMethod m_compressionMethod;

  int m_adaptiveLevel;

  Error m_error;

  std::vector<uint8_t> m_data;
  std::vector<uint8_t> m_metadata;
};
Jørgen Lind's avatar
Jørgen Lind committed
81
82

struct PendingDownloadRequest
83
84
85
{
  MetadataPage* m_lockedMetadataPage;

Jørgen Lind's avatar
Jørgen Lind committed
86
  std::shared_ptr<Request> m_activeTransfer;
87
88
  std::shared_ptr<ReadChunkTransfer> m_transferHandle;

Jørgen Lind's avatar
Jørgen Lind committed
89
  PendingDownloadRequest() : m_lockedMetadataPage(nullptr)
90
91
92
  {
  }

Jørgen Lind's avatar
Jørgen Lind committed
93
  explicit PendingDownloadRequest(MetadataPage* lockedMetadataPage) : m_lockedMetadataPage(lockedMetadataPage), m_activeTransfer(nullptr)
94
95
  {
  }
Jørgen Lind's avatar
Jørgen Lind committed
96
  explicit PendingDownloadRequest(std::shared_ptr<Request> activeTransfer, std::shared_ptr<ReadChunkTransfer> handler) : m_lockedMetadataPage(nullptr), m_activeTransfer(activeTransfer), m_transferHandle(handler)
97
98
99
100
  {
  }
};

Jørgen Lind's avatar
Jørgen Lind committed
101
inline bool operator<(const VolumeDataChunk &a, const VolumeDataChunk &b)
102
{
103
  if (a.Layer->GetChunkDimensionGroup() == b.Layer->GetChunkDimensionGroup())
104
  {
105
    if (a.Layer->GetLOD() == b.Layer->GetLOD())
106
    {
107
      if (a.Layer->GetChannelIndex() == b.Layer->GetChannelIndex())
108
      {
109
        return a.Index < b.Index;
110
111
112
      }
      else
      {
113
        return a.Layer->GetChannelIndex() < b.Layer->GetChannelIndex();
114
115
116
117
      }
    }
    else
    {
118
      return a.Layer->GetLOD() < b.Layer->GetLOD();
119
120
    }
  }
121
  return DimensionGroupUtil::GetDimensionsNDFromDimensionGroup(a.Layer->GetChunkDimensionGroup()) < DimensionGroupUtil::GetDimensionsNDFromDimensionGroup(b.Layer->GetChunkDimensionGroup());
122
123
}

Jørgen Lind's avatar
Jørgen Lind committed
124
125
struct PendingUploadRequest
{
126
127
128
129
130
131
132
133
134
135
136
137
138
  std::string url;
  std::string contentDispositionName;
  std::vector<std::pair<std::string, std::string>> metaMap;
  std::shared_ptr<std::vector<uint8_t>> data;
  std::function<void(const Request & request, const Error & error)> completedCallback;
  uint32_t attempts;
  std::shared_ptr<Request> request;

  PendingUploadRequest()
    : request()
    , attempts(0)
  {
  }
139

140
141
142
143
144
145
146
  PendingUploadRequest(IOManager &ioManager,  const std::string &url, const std::string contentDispositionName,  std::vector<std::pair<std::string, std::string>> &metaMap, std::shared_ptr<std::vector<uint8_t>> data, std::function<void(const Request & request, const Error & error)> completedCallback)
    : url(url)
    , contentDispositionName(contentDispositionName)
    , metaMap(metaMap)
    , data(data)
    , completedCallback(completedCallback)
    , attempts(0)
147
  {
148
    StartNewUpload(ioManager);
149
150
  }

151
  void StartNewUpload(IOManager &ioManager)
152
  {
153
    request = ioManager.UploadBinary(url, contentDispositionName, metaMap, data, completedCallback);
154
155
    attempts++;
  }
Jørgen Lind's avatar
Jørgen Lind committed
156
157
158
159
160
161
162
163
164
165
166
167
};

struct UploadError
{
  UploadError(const Error &error, const std::string &urlObject)
    : error(error)
    , urlObject(urlObject)
  {}
  Error error;
  std::string urlObject;
};

168
169
170
class VolumeDataAccessManagerImpl : public VolumeDataAccessManager
{
public:
Jørgen Lind's avatar
Jørgen Lind committed
171
  VolumeDataAccessManagerImpl(VDS &vds);
Jørgen Lind's avatar
Jørgen Lind committed
172
  ~VolumeDataAccessManagerImpl() override;
173
  VolumeDataLayout const *GetVolumeDataLayout() const override;
174
  VolumeDataLayoutImpl const *GetVolumeDataLayoutImpl() const;
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
  VolumeDataPageAccessor *CreateVolumeDataPageAccessor(VolumeDataLayout const *volumeDataLayout, DimensionsND dimensionsND, int lod, int channel, int maxPages, AccessMode accessMode) override;

  void  DestroyVolumeDataPageAccessor(VolumeDataPageAccessor *volumeDataPageAccessor) override;
  void  DestroyVolumeDataAccessor(VolumeDataAccessor *accessor) override;
  VolumeDataAccessor * CloneVolumeDataAccessor(VolumeDataAccessor const &accessor) override;

  bool  IsCompleted(int64_t requestID) override;
  bool  IsCanceled(int64_t requestID) override; 
  bool  WaitForCompletion(int64_t requestID, int millisecondsBeforeTimeout = 0) override;
  void  Cancel(int64_t requestID) override;
  float GetCompletionFactor(int64_t requestID) override;

  VolumeDataReadWriteAccessor<IntVector2, bool>     *Create2DVolumeDataAccessor1Bit(VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue) override;
  VolumeDataReadWriteAccessor<IntVector2, uint8_t>  *Create2DVolumeDataAccessorU8  (VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue) override;
  VolumeDataReadWriteAccessor<IntVector2, uint16_t> *Create2DVolumeDataAccessorU16 (VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue) override;
  VolumeDataReadWriteAccessor<IntVector2, uint32_t> *Create2DVolumeDataAccessorU32 (VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue) override;
  VolumeDataReadWriteAccessor<IntVector2, uint64_t> *Create2DVolumeDataAccessorU64 (VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue) override;
  VolumeDataReadWriteAccessor<IntVector2, float>    *Create2DVolumeDataAccessorR32 (VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue) override;
  VolumeDataReadWriteAccessor<IntVector2, double>   *Create2DVolumeDataAccessorR64 (VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue) override;

  VolumeDataReadWriteAccessor<IntVector3, bool>     *Create3DVolumeDataAccessor1Bit(VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue) override;
  VolumeDataReadWriteAccessor<IntVector3, uint8_t>  *Create3DVolumeDataAccessorU8  (VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue) override;
  VolumeDataReadWriteAccessor<IntVector3, uint16_t> *Create3DVolumeDataAccessorU16 (VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue) override;
  VolumeDataReadWriteAccessor<IntVector3, uint32_t> *Create3DVolumeDataAccessorU32 (VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue) override;
  VolumeDataReadWriteAccessor<IntVector3, uint64_t> *Create3DVolumeDataAccessorU64 (VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue) override;
  VolumeDataReadWriteAccessor<IntVector3, float>    *Create3DVolumeDataAccessorR32 (VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue) override;
  VolumeDataReadWriteAccessor<IntVector3, double>   *Create3DVolumeDataAccessorR64 (VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue) override;

  VolumeDataReadWriteAccessor<IntVector4, bool>     *Create4DVolumeDataAccessor1Bit(VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue) override;
  VolumeDataReadWriteAccessor<IntVector4, uint8_t>  *Create4DVolumeDataAccessorU8  (VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue) override;
  VolumeDataReadWriteAccessor<IntVector4, uint16_t> *Create4DVolumeDataAccessorU16 (VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue) override;
  VolumeDataReadWriteAccessor<IntVector4, uint32_t> *Create4DVolumeDataAccessorU32 (VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue) override;
  VolumeDataReadWriteAccessor<IntVector4, uint64_t> *Create4DVolumeDataAccessorU64 (VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue) override;
  VolumeDataReadWriteAccessor<IntVector4, float>    *Create4DVolumeDataAccessorR32 (VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue) override;
  VolumeDataReadWriteAccessor<IntVector4, double>   *Create4DVolumeDataAccessorR64 (VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue) override;

  VolumeDataReadAccessor<FloatVector2, float > *Create2DInterpolatingVolumeDataAccessorR32(VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue, InterpolationMethod interpolationMethod) override;
  VolumeDataReadAccessor<FloatVector2, double> *Create2DInterpolatingVolumeDataAccessorR64(VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue, InterpolationMethod interpolationMethod) override;
  VolumeDataReadAccessor<FloatVector3, float > *Create3DInterpolatingVolumeDataAccessorR32(VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue, InterpolationMethod interpolationMethod) override;
  VolumeDataReadAccessor<FloatVector3, double> *Create3DInterpolatingVolumeDataAccessorR64(VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue, InterpolationMethod interpolationMethod) override;
  VolumeDataReadAccessor<FloatVector4, float > *Create4DInterpolatingVolumeDataAccessorR32(VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue, InterpolationMethod interpolationMethod) override;
  VolumeDataReadAccessor<FloatVector4, double> *Create4DInterpolatingVolumeDataAccessorR64(VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue, InterpolationMethod interpolationMethod) override;

  int64_t RequestVolumeSubset(void *buffer, VolumeDataLayout const *volumeDataLayout, DimensionsND dimensionsND, int lod, int channel, const int (&minVoxelCoordinates)[Dimensionality_Max], const int (&maxVoxelCoordinates)[Dimensionality_Max], VolumeDataChannelDescriptor::Format format) override;
  int64_t RequestVolumeSubset(void *buffer, VolumeDataLayout const *volumeDataLayout, DimensionsND dimensionsND, int lOD, int channel, const int (&minVoxelCoordinates)[Dimensionality_Max], const int (&maxVoxelCoordinates)[Dimensionality_Max], VolumeDataChannelDescriptor::Format format, float replacementNoValue) override;
  int64_t RequestProjectedVolumeSubset(void *buffer, VolumeDataLayout const *volumeDataLayout, DimensionsND dimensionsND, int lod, int channel, const int (&minVoxelCoordinates)[Dimensionality_Max], const int (&maxVoxelCoordinates)[Dimensionality_Max], FloatVector4 const &voxelPlane, DimensionsND projectedDimensions, VolumeDataChannelDescriptor::Format format, InterpolationMethod interpolationMethod) override;
  int64_t RequestProjectedVolumeSubset(void *buffer, VolumeDataLayout const *volumeDataLayout, DimensionsND dimensionsND, int lod, int channel, const int (&minVoxelCoordinates)[Dimensionality_Max], const int (&maxVoxelCoordinates)[Dimensionality_Max], FloatVector4 const &voxelPlane, DimensionsND projectedDimensions, VolumeDataChannelDescriptor::Format format, InterpolationMethod interpolationMethod, float replacementNoValue) override;
  int64_t RequestVolumeSamples(float *buffer, VolumeDataLayout const *volumeDataLayout, DimensionsND dimensionsND, int lod, int channel, const float (*samplePositions)[Dimensionality_Max], int sampleCount, InterpolationMethod interpolationMethod) override;
  int64_t RequestVolumeSamples(float *buffer, VolumeDataLayout const *volumeDataLayout, DimensionsND dimensionsND, int lod, int channel, const float (*SamplePositions)[Dimensionality_Max], int sampleCount, InterpolationMethod interpolationMethod, float replacementNoValue) override;
  int64_t RequestVolumeTraces(float *buffer, VolumeDataLayout const *volumeDataLayout, DimensionsND dimensionsND, int lod, int channel, const float(*tracePositions)[Dimensionality_Max], int traceCount, InterpolationMethod interpolationMethod, int iTraceDimension) override;
  int64_t RequestVolumeTraces(float *buffer, VolumeDataLayout const *volumeDataLayout, DimensionsND dimensionsND, int lod, int channel, const float(*tracePositions)[Dimensionality_Max], int nTraceCount, InterpolationMethod eInterpolationMethod, int iTraceDimension, float rReplacementNoValue) override;
  int64_t PrefetchVolumeChunk(VolumeDataLayout const *volumeDataLayout, DimensionsND dimensionsND, int lod, int channel, int64_t chunk) override;
227

228
229
230
231
  bool PrepareReadChunkData(const VolumeDataChunk& chunk, bool verbose, Error& error);
  bool ReadChunk(const VolumeDataChunk& chunk, std::vector<uint8_t>& serializedData, std::vector<uint8_t>& metadata, CompressionInfo& compressionInfo, Error& error);
  void PageTransferCompleted(MetadataPage* metadataPage);
  bool WriteMetadataPage(MetadataPage* metadataPage, const std::vector<uint8_t> &data);
Jørgen Lind's avatar
Jørgen Lind committed
232

233
  int64_t RequestWriteChunk(const VolumeDataChunk &chunk, const DataBlock &dataBlock, const std::vector<uint8_t> &data);
234
  
235
  IOManager *GetIoManager() const { return m_ioManager; }
Jørgen Lind's avatar
Jørgen Lind committed
236

237
238
239
240
241
  void FlushUploadQueue() override;
  void ClearUploadErrors() override;
  void ForceClearAllUploadErrors() override;
  int32_t UploadErrorCount() override;
  void GetCurrentUploadError(const char **objectId, int32_t *errorCode, const char **errorString) override;
Jørgen Lind's avatar
Jørgen Lind committed
242

243
private:
Jørgen Lind's avatar
Jørgen Lind committed
244
  VDS &m_vds;
245
  IOManager *m_ioManager;
Jørgen Lind's avatar
Jørgen Lind committed
246
  VolumeDataRequestProcessor m_requestProcessor;
247
248
  IntrusiveList<VolumeDataPageAccessorImpl, &VolumeDataPageAccessorImpl::m_volumeDataPageAccessorListNode> m_volumeDataPageAccessorList;
  std::mutex m_mutex;
249
  std::condition_variable m_pendingRequestChangedCondition;
Jørgen Lind's avatar
Jørgen Lind committed
250
  std::map<VolumeDataChunk, PendingDownloadRequest> m_pendingDownloadRequests;
251
  std::map<int64_t, PendingUploadRequest> m_pendingUploadRequests;
Jørgen Lind's avatar
Jørgen Lind committed
252
253
  std::vector<std::unique_ptr<UploadError>> m_uploadErrors;
  uint32_t m_currentErrorIndex;
254
};
Jørgen Lind's avatar
Jørgen Lind committed
255

256
}
257
#endif //VOLUMEDATAACCESSMANAGERIMPL_H