Commit 5bec848f authored by Morten Ofstad's avatar Morten Ofstad
Browse files

Added creation of LayerMetadataManager when you create a new page accessor with AccessMode_Create.

parent b3ebc22e
......@@ -43,7 +43,7 @@ VDSHandle *open(const OpenOptions &options, Error &error)
{
return nullptr;
}
ret->dataAccessManager.reset(new VolumeDataAccessManagerImpl(ret.get()));
ret->dataAccessManager.reset(new VolumeDataAccessManagerImpl(*ret.get()));
ret->requestProcessor.reset(new VolumeDataRequestProcessor(*ret->dataAccessManager.get()));
return ret.release();
}
......@@ -88,6 +88,35 @@ static int32_t getLODCount(const VolumeDataLayoutDescriptor &desc)
return desc.getLODLevels() + 1;
}
std::string getLayerName(VolumeDataLayer const &volumeDataLayer)
{
if(volumeDataLayer.getChannelIndex() == 0)
{
return std::string(DimensionGroupUtil::getDimensionGroupName(volumeDataLayer.getChunkDimensionGroup())) + "LOD" + std::to_string(volumeDataLayer.getLOD());
}
else
{
assert(strlen(volumeDataLayer.getVolumeDataChannelDescriptor().getName()) != 0);
return std::string(volumeDataLayer.getVolumeDataChannelDescriptor().getName()) + std::string(DimensionGroupUtil::getDimensionGroupName(volumeDataLayer.getPrimaryChannelLayer().getChunkDimensionGroup())) + "LOD" + std::to_string(volumeDataLayer.getLOD());
}
}
MetadataManager *findMetadataManager(LayerMetadataContainer const &layerMetadataContainer, std::string const &layerName)
{
std::unique_lock<std::mutex> metadataManagersMutexLock(layerMetadataContainer.mutex);
auto it = layerMetadataContainer.managers.find(layerName);
return (it != layerMetadataContainer.managers.end()) ? it->second.get() : nullptr;
}
MetadataManager *createMetadataManager(VDSHandle &handle, std::string const &layerName, MetadataStatus const &metadataStatus)
{
std::unique_lock<std::mutex> metadataManagersMutexLock(handle.layerMetadataContainer.mutex);
assert(handle.layerMetadataContainer.managers.find(layerName) == handle.layerMetadataContainer.managers.end());
int pageLimit = handle.axisDescriptors.size() <= 3 ? 64 : 1024;
return handle.layerMetadataContainer.managers.insert(std::make_pair(layerName, std::unique_ptr<MetadataManager>(new MetadataManager(handle.ioManager.get(), layerName, metadataStatus, pageLimit)))).first->second.get();
}
void createVolumeDataLayout(VDSHandle &handle)
{
//handle.volumeDataLayout.reset(new VolumeDataLayout(handle.channelDescriptors)
......@@ -180,7 +209,7 @@ VDSHandle* create(const OpenOptions& options, VolumeDataLayoutDescriptor const &
if (!serializeAndUploadVolumeDataLayout(*handle, error))
return nullptr;
handle->dataAccessManager.reset(new VolumeDataAccessManagerImpl(handle.get()));
handle->dataAccessManager.reset(new VolumeDataAccessManagerImpl(*handle.get()));
handle->requestProcessor.reset(new VolumeDataRequestProcessor(*handle->dataAccessManager.get()));
return handle.release();
......
......@@ -43,7 +43,7 @@ namespace OpenVDS
class LayerMetadataContainer
{
public:
std::mutex mutex;
mutable std::mutex mutex;
std::map<std::string, std::unique_ptr<MetadataManager>> managers;
};
......@@ -85,6 +85,11 @@ struct VDSHandle
const char *addDescriptorString(std::string const &descriptorString, VDSHandle &handle);
void createVolumeDataLayout(VDSHandle &handle);
std::string getLayerName(VolumeDataLayer const &volumeDataLayer);
MetadataManager *findMetadataManager(LayerMetadataContainer const &layerMetadataContainer, std::string const &layerName);
MetadataManager *createMetadataManager(VDSHandle &handle, std::string const &layerName, MetadataStatus const &metadataStatus);
}
#endif //OPENVDSHANDLE_H
......@@ -1081,15 +1081,13 @@ Json::Value serializeLayerStatusArray(VolumeDataLayout const &volumeDataLayout,
{
for(VolumeDataLayer *volumeDataLayer = volumeDataLayout.getBaseLayer(dimensionGroup, channel); volumeDataLayer && volumeDataLayer->getLayerType() != VolumeDataLayer::Virtual; volumeDataLayer = volumeDataLayer->getParentLayer())
{
std::string layerName = (channel == 0 ? std::string() : std::string(volumeDataLayer->getVolumeDataChannelDescriptor().getName())) + dimensionGroupName + "LOD" + std::to_string(volumeDataLayer->getLOD());
if(volumeDataLayer->getProduceStatus() != VolumeDataLayer::ProduceStatus_Unavailable)
{
std::string layerName = getLayerName(*volumeDataLayer);
Json::Value layerStatusJson = serializeLayerStatus(*volumeDataLayer, layerName);
auto it = layerMetadataContainer.managers.find(layerName);
if(it != layerMetadataContainer.managers.end())
if(MetadataManager *metadataManager = findMetadataManager(layerMetadataContainer, layerName))
{
Json::Value metadataStatusJson = serializeMetadataStatus(it->second->metadataStatus());
Json::Value metadataStatusJson = serializeMetadataStatus(metadataManager->metadataStatus());
for (const auto& key : metadataStatusJson.getMemberNames())
{
......
......@@ -213,10 +213,9 @@ static VolumeDataLayer *getVolumeDataLayer(VolumeDataLayout const *layout, Dimen
return (layer && layer->getLayerType() != VolumeDataLayer::Virtual) ? layer : nullptr;
}
VolumeDataAccessManagerImpl::VolumeDataAccessManagerImpl(VDSHandle* handle)
: m_layout(handle->volumeDataLayout.get())
, m_ioManager(handle->ioManager.get())
, m_layerMetadataContainer(&handle->layerMetadataContainer)
VolumeDataAccessManagerImpl::VolumeDataAccessManagerImpl(VDSHandle &handle)
: m_handle(handle)
, m_ioManager(handle.ioManager.get())
, m_currentErrorIndex(0)
{
}
......@@ -231,7 +230,7 @@ VolumeDataAccessManagerImpl::~VolumeDataAccessManagerImpl()
VolumeDataLayout const* VolumeDataAccessManagerImpl::getVolumeDataLayout() const
{
return m_layout;
return m_handle.volumeDataLayout.get();
}
VolumeDataPageAccessor* VolumeDataAccessManagerImpl::createVolumeDataPageAccessor(VolumeDataLayout const* volumeDataLayout, DimensionsND dimensionsND, int lod, int channel, int maxPages, AccessMode accessMode)
......@@ -245,6 +244,12 @@ VolumeDataPageAccessor* VolumeDataAccessManagerImpl::createVolumeDataPageAccesso
if(accessMode == VolumeDataAccessManager::AccessMode_Create)
{
layer->setProduceStatus(VolumeDataLayer::ProduceStatus_Normal);
MetadataStatus metadataStatus = {};
metadataStatus.m_chunkMetadataPageSize = 1024;
metadataStatus.m_chunkMetadataByteSize = sizeof(int64_t);
metadataStatus.m_compressionMethod = layer->getEffectiveCompressionMethod();
metadataStatus.m_compressionTolerance = layer->getEffectiveCompressionTolerance();
createMetadataManager(m_handle, getLayerName(*layer), metadataStatus);
}
else if (layer->getProduceStatus() == VolumeDataLayer::ProduceStatus_Unavailable)
{
......@@ -297,14 +302,14 @@ float VolumeDataAccessManagerImpl::getCompletionFactor(int64_t requestID)
return 0.0f;
}
static MetadataManager *getMetadataMangerForLayer(LayerMetadataContainer *container, const std::string &layer)
static MetadataManager *getMetadataMangerForLayer(LayerMetadataContainer const &container, const std::string &layer)
{
std::unique_lock<std::mutex> lock(container->mutex);
std::unique_lock<std::mutex> lock(container.mutex);
MetadataManager *metadataManager = nullptr;
auto metadataManagerIterator = container->managers.find(layer);
auto metadataManagerIterator = container.managers.find(layer);
if(metadataManagerIterator != container->managers.end())
if(metadataManagerIterator != container.managers.end())
{
metadataManager = metadataManagerIterator->second.get();
}
......@@ -335,7 +340,7 @@ static std::string createBaseUrl(const VolumeDataLayer *layer)
bool VolumeDataAccessManagerImpl::prepareReadChunkData(const VolumeDataChunk &chunk, bool verbose, Error &error)
{
std::string layerURL = createBaseUrl(chunk.layer);
auto metadataManager = getMetadataMangerForLayer(m_layerMetadataContainer, layerURL);
auto metadataManager = getMetadataMangerForLayer(m_handle.layerMetadataContainer, layerURL);
//do fallback
if (!metadataManager)
{
......@@ -461,14 +466,10 @@ void VolumeDataAccessManagerImpl::pageTransferCompleted(MetadataPage* metadataPa
if (metadataPage->IsValid())
{
uint8_t const *metadata = metadataManager->getPageEntry(metadataPage, entryIndex);
ParsedMetadata parsedMetadata = parseMetadata(metadataManager->metadataStatus().m_chunkMetadataByteSize, metadata);
metadataManager->unlockPage(metadataPage);
pendingRequest.m_lockedMetadataPage = nullptr;
int adaptiveLevel;
IORange ioRange = calculateRangeHeaderImpl(parsedMetadata, metadataManager->metadataStatus(), &adaptiveLevel);
......@@ -479,6 +480,9 @@ void VolumeDataAccessManagerImpl::pageTransferCompleted(MetadataPage* metadataPa
pendingRequest.m_activeTransfer = m_ioManager->downloadObject(url, transferHandler, ioRange);
pendingRequest.m_transferHandle = transferHandler;
}
metadataManager->unlockPage(metadataPage);
pendingRequest.m_lockedMetadataPage = nullptr;
}
}
m_pendingRequestChangedCondition.notify_all();
......@@ -492,6 +496,7 @@ static int64_t gen_upload_jobid()
int64_t VolumeDataAccessManagerImpl::requestWriteChunk(const VolumeDataChunk& chunk, std::shared_ptr<std::vector<uint8_t>> data)
{
return 0;
std::string url = createUrlForChunk(createBaseUrl(chunk.layer), chunk.chunkIndex);
m_pendingUploadRequests.erase(std::remove_if(m_pendingUploadRequests.begin(), m_pendingUploadRequests.end(), [this](PendingUploadRequest &request){
Error error;
......@@ -520,7 +525,7 @@ void VolumeDataAccessManagerImpl::flushUploadQueue()
m_pendingUploadRequests.clear();
error = Error();
serializeAndUploadLayerStatus(m_layout->getHandle(), error);
serializeAndUploadLayerStatus(m_handle, error);
if(error.code != 0)
{
m_uploadErrors.emplace_back(new UploadError(error, "LayerStatus"));
......
......@@ -140,7 +140,7 @@ struct UploadError
class VolumeDataAccessManagerImpl : public VolumeDataAccessManager
{
public:
VolumeDataAccessManagerImpl(VDSHandle *handle);
VolumeDataAccessManagerImpl(VDSHandle &handle);
~VolumeDataAccessManagerImpl() override;
VolumeDataLayout const *getVolumeDataLayout() const override;
VolumeDataPageAccessor *createVolumeDataPageAccessor(VolumeDataLayout const *volumeDataLayout, DimensionsND dimensionsND, int lod, int channel, int maxPages, AccessMode accessMode) override;
......@@ -212,7 +212,7 @@ public:
void addUploadError(const Error &error, VolumeDataLayer *layer, uint64_t chunk);
private:
VolumeDataLayout *m_layout;
VDSHandle &m_handle;
IOManager *m_ioManager;
LayerMetadataContainer *m_layerMetadataContainer;
IntrusiveList<VolumeDataPageAccessorImpl, &VolumeDataPageAccessorImpl::m_volumeDataPageAccessorListNode> m_volumeDataPageAccessorList;
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment