VolumeDataAccessManagerImpl.h 13.6 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
/****************************************************************************
** Copyright 2019 The Open Group
** Copyright 2019 Bluware, Inc.
**
** Licensed under the Apache License, Version 2.0 (the "License");
** you may not use this file except in compliance with the License.
** You may obtain a copy of the License at
**
**     http://www.apache.org/licenses/LICENSE-2.0
**
** Unless required by applicable law or agreed to in writing, software
** distributed under the License is distributed on an "AS IS" BASIS,
** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
** See the License for the specific language governing permissions and
** limitations under the License.
****************************************************************************/

#ifndef VOLUMEDATAACCESSMANAGERIMPL_H
#define VOLUMEDATAACCESSMANAGERIMPL_H

#include <OpenVDS/VolumeDataAccess.h>

#include "IntrusiveList.h"
#include "VolumeDataPageAccessorImpl.h"
#include <IO/IOManager.h>

27
28
29
30
31
32
#include "VolumeDataChunk.h"
#include "VolumeDataLayer.h"

#include <map>
#include "Base64.h"

33
34
namespace OpenVDS
{
35
36
37
class LayerMetadataContainer;
class MetadataPage;

38
class ReadChunkTransfer : public TransferDownloadHandler
39
40
41
42
43
44
45
{
public:
  ReadChunkTransfer(CompressionMethod compressionMethod, int adaptiveLevel)
    : m_compressionMethod(compressionMethod)
    , m_adaptiveLevel(adaptiveLevel)
  {}

Jørgen Lind's avatar
Jørgen Lind committed
46
  ~ReadChunkTransfer() override
47
48
49
50
51
52
53
  {
  }

  void handleMetadata(const std::string& key, const std::string& header) override
  {
    if (key == "vds-chunk-metadata")
    {
Morten Ofstad's avatar
Morten Ofstad committed
54
      if (!Base64Decode(header.data(), (int)header.size(), m_metadata))
55
56
57
58
59
60
61
62
63
64
65
      {
        m_error.code = -1;
        m_error.string = "Failed to decode chunk metadata";
      }
    }
  }

  void handleData(std::vector<uint8_t>&& data) override
  {
    m_data = data;
  }
66
67

  void completed(const Request &req, const Error & error) override
68
69
70
71
72
73
74
75
76
77
78
79
80
  {
    m_error = error;
  }
  
  CompressionMethod m_compressionMethod;

  int m_adaptiveLevel;

  Error m_error;

  std::vector<uint8_t> m_data;
  std::vector<uint8_t> m_metadata;
};
Jørgen Lind's avatar
Jørgen Lind committed
81
82

struct PendingDownloadRequest
83
84
85
{
  MetadataPage* m_lockedMetadataPage;

Jørgen Lind's avatar
Jørgen Lind committed
86
  std::shared_ptr<Request> m_activeTransfer;
87
88
  std::shared_ptr<ReadChunkTransfer> m_transferHandle;

Jørgen Lind's avatar
Jørgen Lind committed
89
  PendingDownloadRequest() : m_lockedMetadataPage(nullptr)
90
91
92
  {
  }

Jørgen Lind's avatar
Jørgen Lind committed
93
  explicit PendingDownloadRequest(MetadataPage* lockedMetadataPage) : m_lockedMetadataPage(lockedMetadataPage), m_activeTransfer(nullptr)
94
95
  {
  }
Jørgen Lind's avatar
Jørgen Lind committed
96
  explicit PendingDownloadRequest(std::shared_ptr<Request> activeTransfer, std::shared_ptr<ReadChunkTransfer> handler) : m_lockedMetadataPage(nullptr), m_activeTransfer(activeTransfer), m_transferHandle(handler)
97
98
99
100
  {
  }
};

Jørgen Lind's avatar
Jørgen Lind committed
101
inline bool operator<(const VolumeDataChunk &a, const VolumeDataChunk &b)
102
103
104
{
  if (a.layer->getChunkDimensionGroup() == b.layer->getChunkDimensionGroup())
  {
105
    if (a.layer->getLOD() == b.layer->getLOD())
106
107
108
109
110
111
112
113
114
115
116
117
    {
      if (a.layer->getChannelIndex() == b.layer->getChannelIndex())
      {
        return a.chunkIndex < b.chunkIndex;
      }
      else
      {
        return a.layer->getChannelIndex() < b.layer->getChannelIndex();
      }
    }
    else
    {
118
      return a.layer->getLOD() < b.layer->getLOD();
119
120
121
122
123
    }
  }
  return DimensionGroupUtil::getDimensionsNDFromDimensionGroup(a.layer->getChunkDimensionGroup()) < DimensionGroupUtil::getDimensionsNDFromDimensionGroup(b.layer->getChunkDimensionGroup());
}

Jørgen Lind's avatar
Jørgen Lind committed
124
125
struct PendingUploadRequest
{
126
127
128
129
130
131
132
133
134
  std::shared_ptr<Request> m_request;

  MetadataPage* m_lockedMetadataPage;

  PendingUploadRequest() : m_request(), m_lockedMetadataPage()
  {
  }

  PendingUploadRequest(std::shared_ptr<Request> request, MetadataPage* lockedMetadataPage) : m_request(request), m_lockedMetadataPage(lockedMetadataPage) {}
Jørgen Lind's avatar
Jørgen Lind committed
135
136
137
138
139
140
141
142
143
144
145
146
};

struct UploadError
{
  UploadError(const Error &error, const std::string &urlObject)
    : error(error)
    , urlObject(urlObject)
  {}
  Error error;
  std::string urlObject;
};

147
148
149
class VolumeDataAccessManagerImpl : public VolumeDataAccessManager
{
public:
150
  VolumeDataAccessManagerImpl(VDSHandle &handle);
Jørgen Lind's avatar
Jørgen Lind committed
151
  ~VolumeDataAccessManagerImpl() override;
152
  VolumeDataLayout const *getVolumeDataLayout() const override;
153
  VolumeDataPageAccessor *createVolumeDataPageAccessor(VolumeDataLayout const *volumeDataLayout, DimensionsND dimensionsND, int lod, int channel, int maxPages, AccessMode accessMode) override;
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204

  void  destroyVolumeDataPageAccessor(VolumeDataPageAccessor *volumeDataPageAccessor) override;
  void  destroyVolumeDataAccessor(VolumeDataAccessor *accessor) override;
  VolumeDataAccessor * cloneVolumeDataAccessor(VolumeDataAccessor const &accessor) override;

  bool  isCompleted(int64_t requestID) override;
  bool  isCanceled(int64_t requestID) override; 
  bool  waitForCompletion(int64_t requestID, int millisecondsBeforeTimeout = 0) override;
  void  cancel(int64_t requestID) override;
  float getCompletionFactor(int64_t requestID) override;

  VolumeDataReadWriteAccessor<IntVector2, bool>     *create2DVolumeDataAccessor1Bit(VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue) override;
  VolumeDataReadWriteAccessor<IntVector2, uint8_t>  *create2DVolumeDataAccessorU8  (VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue) override;
  VolumeDataReadWriteAccessor<IntVector2, uint16_t> *create2DVolumeDataAccessorU16 (VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue) override;
  VolumeDataReadWriteAccessor<IntVector2, uint32_t> *create2DVolumeDataAccessorU32 (VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue) override;
  VolumeDataReadWriteAccessor<IntVector2, uint64_t> *create2DVolumeDataAccessorU64 (VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue) override;
  VolumeDataReadWriteAccessor<IntVector2, float>    *create2DVolumeDataAccessorR32 (VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue) override;
  VolumeDataReadWriteAccessor<IntVector2, double>   *create2DVolumeDataAccessorR64 (VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue) override;

  VolumeDataReadWriteAccessor<IntVector3, bool>     *create3DVolumeDataAccessor1Bit(VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue) override;
  VolumeDataReadWriteAccessor<IntVector3, uint8_t>  *create3DVolumeDataAccessorU8  (VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue) override;
  VolumeDataReadWriteAccessor<IntVector3, uint16_t> *create3DVolumeDataAccessorU16 (VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue) override;
  VolumeDataReadWriteAccessor<IntVector3, uint32_t> *create3DVolumeDataAccessorU32 (VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue) override;
  VolumeDataReadWriteAccessor<IntVector3, uint64_t> *create3DVolumeDataAccessorU64 (VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue) override;
  VolumeDataReadWriteAccessor<IntVector3, float>    *create3DVolumeDataAccessorR32 (VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue) override;
  VolumeDataReadWriteAccessor<IntVector3, double>   *create3DVolumeDataAccessorR64 (VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue) override;

  VolumeDataReadWriteAccessor<IntVector4, bool>     *create4DVolumeDataAccessor1Bit(VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue) override;
  VolumeDataReadWriteAccessor<IntVector4, uint8_t>  *create4DVolumeDataAccessorU8  (VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue) override;
  VolumeDataReadWriteAccessor<IntVector4, uint16_t> *create4DVolumeDataAccessorU16 (VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue) override;
  VolumeDataReadWriteAccessor<IntVector4, uint32_t> *create4DVolumeDataAccessorU32 (VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue) override;
  VolumeDataReadWriteAccessor<IntVector4, uint64_t> *create4DVolumeDataAccessorU64 (VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue) override;
  VolumeDataReadWriteAccessor<IntVector4, float>    *create4DVolumeDataAccessorR32 (VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue) override;
  VolumeDataReadWriteAccessor<IntVector4, double>   *create4DVolumeDataAccessorR64 (VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue) override;

  VolumeDataReadAccessor<FloatVector2, float > *create2DInterpolatingVolumeDataAccessorR32(VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue, InterpolationMethod interpolationMethod) override;
  VolumeDataReadAccessor<FloatVector2, double> *create2DInterpolatingVolumeDataAccessorR64(VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue, InterpolationMethod interpolationMethod) override;
  VolumeDataReadAccessor<FloatVector3, float > *create3DInterpolatingVolumeDataAccessorR32(VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue, InterpolationMethod interpolationMethod) override;
  VolumeDataReadAccessor<FloatVector3, double> *create3DInterpolatingVolumeDataAccessorR64(VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue, InterpolationMethod interpolationMethod) override;
  VolumeDataReadAccessor<FloatVector4, float > *create4DInterpolatingVolumeDataAccessorR32(VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue, InterpolationMethod interpolationMethod) override;
  VolumeDataReadAccessor<FloatVector4, double> *create4DInterpolatingVolumeDataAccessorR64(VolumeDataPageAccessor* volumeDataPageAccessor, float replacementNoValue, InterpolationMethod interpolationMethod) override;

  int64_t requestVolumeSubset(void *buffer, VolumeDataLayout const *volumeDataLayout, DimensionsND dimensionsND, int lod, int channel, const int (&minVoxelCoordinates)[Dimensionality_Max], const int (&maxVoxelCoordinates)[Dimensionality_Max], VolumeDataChannelDescriptor::Format format) override;
  int64_t requestVolumeSubset(void *buffer, VolumeDataLayout const *volumeDataLayout, DimensionsND dimensionsND, int lOD, int channel, const int (&minVoxelCoordinates)[Dimensionality_Max], const int (&maxVoxelCoordinates)[Dimensionality_Max], VolumeDataChannelDescriptor::Format format, float replacementNoValue) override;
  int64_t requestProjectedVolumeSubset(void *buffer, VolumeDataLayout const *volumeDataLayout, DimensionsND dimensionsND, int lod, int channel, const int (&minVoxelCoordinates)[Dimensionality_Max], const int (&maxVoxelCoordinates)[Dimensionality_Max], FloatVector4 const &voxelPlane, DimensionsND projectedDimensions, VolumeDataChannelDescriptor::Format format, InterpolationMethod interpolationMethod) override;
  int64_t requestProjectedVolumeSubset(void *buffer, VolumeDataLayout const *volumeDataLayout, DimensionsND dimensionsND, int lod, int channel, const int (&minVoxelCoordinates)[Dimensionality_Max], const int (&maxVoxelCoordinates)[Dimensionality_Max], FloatVector4 const &voxelPlane, DimensionsND projectedDimensions, VolumeDataChannelDescriptor::Format format, InterpolationMethod interpolationMethod, float replacementNoValue) override;
  int64_t requestVolumeSamples(float *buffer, VolumeDataLayout const *volumeDataLayout, DimensionsND dimensionsND, int lod, int channel, const float (*samplePositions)[Dimensionality_Max], int sampleCount, InterpolationMethod interpolationMethod) override;
  int64_t requestVolumeSamples(float *buffer, VolumeDataLayout const *volumeDataLayout, DimensionsND dimensionsND, int lod, int channel, const float (*SamplePositions)[Dimensionality_Max], int sampleCount, InterpolationMethod interpolationMethod, float replacementNoValue) override;
  int64_t requestVolumeTraces(float *buffer, VolumeDataLayout const *volumeDataLayout, DimensionsND dimensionsND, int lod, int channel, const float(*tracePositions)[Dimensionality_Max], int traceCount, InterpolationMethod interpolationMethod, int iTraceDimension) override;
  int64_t requestVolumeTraces(float *buffer, VolumeDataLayout const *volumeDataLayout, DimensionsND dimensionsND, int lod, int channel, const float(*tracePositions)[Dimensionality_Max], int nTraceCount, InterpolationMethod eInterpolationMethod, int iTraceDimension, float rReplacementNoValue) override;
  int64_t prefetchVolumeChunk(VolumeDataLayout const *volumeDataLayout, DimensionsND dimensionsND, int lod, int channel, int64_t chunk) override;
205

Jørgen Lind's avatar
Jørgen Lind committed
206
  bool prepareReadChunkData(const VolumeDataChunk& chunk, bool verbose, Error& error);
Jørgen Lind's avatar
Jørgen Lind committed
207
  bool readChunk(const VolumeDataChunk& chunk, std::vector<uint8_t>& serializedData, std::vector<uint8_t>& metadata, CompressionInfo& compressionInfo, Error& error);
Jørgen Lind's avatar
Jørgen Lind committed
208
  void pageTransferCompleted(MetadataPage* page);
Jørgen Lind's avatar
Jørgen Lind committed
209

210
211
  int64_t requestWriteChunk(const VolumeDataChunk &chunk, const DataBlock &dataBlock, const std::vector<uint8_t> &data);
  
Jørgen Lind's avatar
Jørgen Lind committed
212
  IOManager *getIoManager() const { return m_ioManager; }
Jørgen Lind's avatar
Jørgen Lind committed
213
214
215
216
217
218
219

  void flushUploadQueue() override;
  void clearUploadErrors() override;
  void forceClearAllUploadErrors() override;
  int32_t uploadErrorCount() override;
  void getCurrentUploadError(const char **objectId, int32_t *errorCode, const char **errorString) override;

Jørgen Lind's avatar
Jørgen Lind committed
220
  void addUploadError(const Error &error, VolumeDataLayer *layer, uint64_t chunk);
221
private:
222
  VDSHandle &m_handle;
223
  IOManager *m_ioManager;
224
  LayerMetadataContainer *m_layerMetadataContainer;
225
226
  IntrusiveList<VolumeDataPageAccessorImpl, &VolumeDataPageAccessorImpl::m_volumeDataPageAccessorListNode> m_volumeDataPageAccessorList;
  std::mutex m_mutex;
227
  std::condition_variable m_pendingRequestChangedCondition;
Jørgen Lind's avatar
Jørgen Lind committed
228
  std::map<VolumeDataChunk, PendingDownloadRequest> m_pendingDownloadRequests;
229
  std::map<int64_t, PendingUploadRequest> m_pendingUploadRequests;
Jørgen Lind's avatar
Jørgen Lind committed
230
231
  std::vector<std::unique_ptr<UploadError>> m_uploadErrors;
  uint32_t m_currentErrorIndex;
232
233
234
};
}
#endif //VOLUMEDATAACCESSMANAGERIMPL_H