From 91e2b785cf9526d4113b2e046c270d000b826b91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Lind?= Date: Fri, 11 Mar 2022 10:37:54 +0100 Subject: [PATCH] VolumeDataChannelDescriptor::NoLossyCompressionUseZip is not persisted by inspecting the compression format stored in the MetaData and setting the flag on the ChannelDescriptor --- src/OpenVDS/CMakeLists.txt | 1 + src/OpenVDS/OpenVDS.cpp | 2 +- src/OpenVDS/VDS/LayerMetadataContainer.h | 31 ++++++++++++++++++++ src/OpenVDS/VDS/MetadataManager.cpp | 3 +- src/OpenVDS/VDS/MetadataManager.h | 4 ++- src/OpenVDS/VDS/ParseVDSJson.cpp | 17 +++++++---- src/OpenVDS/VDS/ParseVDSJson.h | 2 +- src/OpenVDS/VDS/VDS.h | 7 ----- src/OpenVDS/VDS/VolumeDataStore.h | 4 +++ src/OpenVDS/VDS/VolumeDataStoreIOManager.cpp | 23 +++++++++++++-- src/OpenVDS/VDS/VolumeDataStoreIOManager.h | 6 +++- src/OpenVDS/VDS/VolumeDataStoreVDSFile.cpp | 27 ++++++++++++++++- src/OpenVDS/VDS/VolumeDataStoreVDSFile.h | 6 +++- tests/VDS/ParseVDSJsonTest.cpp | 12 ++++---- 14 files changed, 118 insertions(+), 27 deletions(-) create mode 100644 src/OpenVDS/VDS/LayerMetadataContainer.h diff --git a/src/OpenVDS/CMakeLists.txt b/src/OpenVDS/CMakeLists.txt index 07cfa9f4..2136550e 100644 --- a/src/OpenVDS/CMakeLists.txt +++ b/src/OpenVDS/CMakeLists.txt @@ -126,6 +126,7 @@ else() VDS/ConnectionStringParser.h VDS/GlobalStateImpl.h VDS/StringToDouble.h + VDS/LayerMetadataContainer.h ${WAVELET_HEADERS} ${COMMON_DIR}/Base64/Base64.h ${COMMON_DIR}/ThreadPool/ThreadPool.h diff --git a/src/OpenVDS/OpenVDS.cpp b/src/OpenVDS/OpenVDS.cpp index 32f78539..f53edf25 100644 --- a/src/OpenVDS/OpenVDS.cpp +++ b/src/OpenVDS/OpenVDS.cpp @@ -612,7 +612,7 @@ static bool Init(VDS *vds, VolumeDataStore *volumeDataStore, Error& error) { return false; } - if(!ParseVolumeDataLayout(serializedVolumeDataLayout, vds->layoutDescriptor, vds->axisDescriptors, vds->channelDescriptors, vds->descriptorStrings, vds->metadataContainer, error)) + if(!ParseVolumeDataLayout(serializedVolumeDataLayout, vds->layoutDescriptor, vds->axisDescriptors, vds->channelDescriptors, vds->descriptorStrings, vds->metadataContainer, vds->volumeDataStore->GetLayerMetadataContainer(), error)) { return false; } diff --git a/src/OpenVDS/VDS/LayerMetadataContainer.h b/src/OpenVDS/VDS/LayerMetadataContainer.h new file mode 100644 index 00000000..866f534a --- /dev/null +++ b/src/OpenVDS/VDS/LayerMetadataContainer.h @@ -0,0 +1,31 @@ +/**************************************************************************** +** Copyright 2022 The Open Group +** Copyright 2022 Bluware, Inc. +** +** Licensed under the Apache License, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** http://www.apache.org/licenses/LICENSE-2.0 +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +****************************************************************************/ + +#ifndef LAYERMETADATACONTAINER_H +#define LAYERMETADATACONTAINER_H + +namespace OpenVDS +{ +class LayerMetadataContainer +{ +public: + virtual bool GetMetadataStatus(std::string const &layerName, MetadataStatus &metadataStatus) const = 0; + virtual bool IsChannelZipped(std::string const& channelName, bool isPrimary) const = 0; + virtual void SetMetadataStatus(std::string const &layerName, std::string const &channelName, MetadataStatus &metadataStatus, int pageLimit) = 0; +}; +} +#endif diff --git a/src/OpenVDS/VDS/MetadataManager.cpp b/src/OpenVDS/VDS/MetadataManager.cpp index 8c2cf678..94cf2ed4 100644 --- a/src/OpenVDS/VDS/MetadataManager.cpp +++ b/src/OpenVDS/VDS/MetadataManager.cpp @@ -69,9 +69,10 @@ MetadataPage *metadataPage; std::vector metadata; }; -MetadataManager::MetadataManager(IOManager* iomanager, std::string const& layerUrl, MetadataStatus const& metadataStatus, int pageLimit) +MetadataManager::MetadataManager(IOManager *iomanager, std::string const &layerUrl, std::string const &channelName, MetadataStatus const &metadataStatus, int pageLimit) : m_iomanager(iomanager) , m_layerUrl(layerUrl) + , m_channelName(channelName) , m_metadataStatus(metadataStatus) , m_pageLimit(pageLimit) { diff --git a/src/OpenVDS/VDS/MetadataManager.h b/src/OpenVDS/VDS/MetadataManager.h index a068cabe..3b6e7120 100644 --- a/src/OpenVDS/VDS/MetadataManager.h +++ b/src/OpenVDS/VDS/MetadataManager.h @@ -94,6 +94,7 @@ namespace OpenVDS { IOManager *m_iomanager; std::string m_layerUrl; + std::string m_channelName; MetadataStatus m_metadataStatus; @@ -109,11 +110,12 @@ namespace OpenVDS void LimitPages(); public: - MetadataManager(IOManager *iomanager, std::string const &layerURL, MetadataStatus const &MetadataStatus, int pageLimit); + MetadataManager(IOManager *iomanager, std::string const &layerURL, std::string const &channelName, MetadataStatus const &MetadataStatus, int pageLimit); ~MetadataManager(); const char *LayerUrl() const { return m_layerUrl.c_str(); } const std::string &LayerUrlStr() const { return m_layerUrl; } + const std::string &ChannelName() const { return m_channelName; } MetadataPage *LockPage(int pageIndex, bool *InitiateTransfer); diff --git a/src/OpenVDS/VDS/ParseVDSJson.cpp b/src/OpenVDS/VDS/ParseVDSJson.cpp index 85de4eb5..f6766166 100644 --- a/src/OpenVDS/VDS/ParseVDSJson.cpp +++ b/src/OpenVDS/VDS/ParseVDSJson.cpp @@ -910,7 +910,7 @@ bool DownloadAndParseVolumeDataLayoutAndLayerStatus(VDS& vds, Error& error) try { - if (!ParseVolumeDataLayout(serializedVolumeDataLayout, vds.layoutDescriptor, vds.axisDescriptors, vds.channelDescriptors, vds.descriptorStrings, vds.metadataContainer, error)) + if (!ParseVolumeDataLayout(serializedVolumeDataLayout, vds.layoutDescriptor, vds.axisDescriptors, vds.channelDescriptors, vds.descriptorStrings, vds.metadataContainer, vds.volumeDataStore->GetLayerMetadataContainer(), error)) return false; } @@ -926,7 +926,7 @@ bool DownloadAndParseVolumeDataLayoutAndLayerStatus(VDS& vds, Error& error) return true; } -bool ParseVolumeDataLayout(const std::vector &json, VolumeDataLayoutDescriptor &layoutDescriptor, std::vector &axisDescriptors, std::vector &channelDescriptors, DescriptorStringContainer &descriptorStrings, MetadataContainer &metadataContainer, Error &error) +bool ParseVolumeDataLayout(const std::vector &json, VolumeDataLayoutDescriptor &layoutDescriptor, std::vector &axisDescriptors, std::vector &channelDescriptors, DescriptorStringContainer &descriptorStrings, MetadataContainer &metadataContainer, const LayerMetadataContainer &layerMetadaContainer, Error &error) { Json::Value root; @@ -968,8 +968,11 @@ bool ParseVolumeDataLayout(const std::vector &json, VolumeDataLayoutDes axisDescriptors.push_back(axisDescriptor); } + bool primary = true; for (const Json::Value &channelDescriptorJson : root["channelDescriptors"]) { + bool noLossyCompressionUseZip = layerMetadaContainer.IsChannelZipped(channelDescriptorJson["name"].asString(), primary) && !channelDescriptorJson["allowLossyCompression"].asBool(); + primary = false; if (channelDescriptorJson["useNoValue"].asBool()) { VolumeDataChannelDescriptor @@ -983,7 +986,8 @@ bool ParseVolumeDataLayout(const std::vector &json, VolumeDataLayoutDes channelDescriptorJson["mappedValues"].asInt(), (channelDescriptorJson["discrete"].asBool() ? VolumeDataChannelDescriptor::DiscreteData : VolumeDataChannelDescriptor::Default) | (channelDescriptorJson["renderable"].asBool() ? VolumeDataChannelDescriptor::Default : VolumeDataChannelDescriptor::NotRenderable) | - (channelDescriptorJson["allowLossyCompression"].asBool() ? VolumeDataChannelDescriptor::Default : VolumeDataChannelDescriptor::NoLossyCompression), + (channelDescriptorJson["allowLossyCompression"].asBool() ? VolumeDataChannelDescriptor::Default : VolumeDataChannelDescriptor::NoLossyCompression) | + (noLossyCompressionUseZip ? VolumeDataChannelDescriptor::NoLossyCompressionUseZip : VolumeDataChannelDescriptor::Default), channelDescriptorJson["noValue"].asFloat(), channelDescriptorJson["integerScale"].asFloat(), channelDescriptorJson["integerOffset"].asFloat()); @@ -1003,7 +1007,8 @@ bool ParseVolumeDataLayout(const std::vector &json, VolumeDataLayoutDes channelDescriptorJson["mappedValues"].asInt(), (channelDescriptorJson["discrete"].asBool() ? VolumeDataChannelDescriptor::DiscreteData : VolumeDataChannelDescriptor::Default) | (channelDescriptorJson["renderable"].asBool() ? VolumeDataChannelDescriptor::Default : VolumeDataChannelDescriptor::NotRenderable) | - (channelDescriptorJson["allowLossyCompression"].asBool() ? VolumeDataChannelDescriptor::Default : VolumeDataChannelDescriptor::NoLossyCompression), + (channelDescriptorJson["allowLossyCompression"].asBool() ? VolumeDataChannelDescriptor::Default : VolumeDataChannelDescriptor::NoLossyCompression) | + (noLossyCompressionUseZip ? VolumeDataChannelDescriptor::NoLossyCompressionUseZip : VolumeDataChannelDescriptor::Default), channelDescriptorJson["integerScale"].asFloat(), channelDescriptorJson["integerOffset"].asFloat()); @@ -1159,12 +1164,14 @@ bool ParseLayerStatus(const std::vector &json, VDS &vds, LayerMetadataC std::string layerName = layerStatus["layerName"].asString(); + std::string + channelName = layerStatus["channelName"].asString(); if (hasChunkMetadataPages) { int pageLimit = vds.axisDescriptors.size() <= 3 ? 64 : 1024; - layerMetadataContainer.SetMetadataStatus(layerName, metadataStatus, pageLimit); + layerMetadataContainer.SetMetadataStatus(layerName, channelName, metadataStatus, pageLimit); } } } diff --git a/src/OpenVDS/VDS/ParseVDSJson.h b/src/OpenVDS/VDS/ParseVDSJson.h index 0c9cb0a4..f3d5ed47 100644 --- a/src/OpenVDS/VDS/ParseVDSJson.h +++ b/src/OpenVDS/VDS/ParseVDSJson.h @@ -23,7 +23,7 @@ namespace OpenVDS { -bool ParseVolumeDataLayout(const std::vector &json, VolumeDataLayoutDescriptor &layoutDescriptor, std::vector &axisDescriptors, std::vector &channelDescriptors, DescriptorStringContainer &descriptorStrings, MetadataContainer &metadataContainer, Error &error); +bool ParseVolumeDataLayout(const std::vector &json, VolumeDataLayoutDescriptor &layoutDescriptor, std::vector &axisDescriptors, std::vector &channelDescriptors, DescriptorStringContainer &descriptorStrings, MetadataContainer &metadataContainer, const LayerMetadataContainer &layerMetadaContainer, Error &error); bool ParseLayerStatus(const std::vector &json, VDS &vds, LayerMetadataContainer &layerMetadataContainer, Error &error); std::vector SerializeVolumeDataLayout(VDS& vds); std::vector SerializeLayerStatus(VDS& vds, LayerMetadataContainer const &layerMetadataContainer); diff --git a/src/OpenVDS/VDS/VDS.h b/src/OpenVDS/VDS/VDS.h index c6866920..05892d4a 100644 --- a/src/OpenVDS/VDS/VDS.h +++ b/src/OpenVDS/VDS/VDS.h @@ -40,13 +40,6 @@ namespace OpenVDS { -class LayerMetadataContainer -{ -public: - virtual bool GetMetadataStatus(std::string const &layerName, MetadataStatus &metadataStatus) const = 0; - virtual void SetMetadataStatus(std::string const &layerName, MetadataStatus &metadataStatus, int pageLimit) = 0; -}; - class DescriptorStringContainer { std::vector> m_descriptorStrings; diff --git a/src/OpenVDS/VDS/VolumeDataStore.h b/src/OpenVDS/VDS/VolumeDataStore.h index 43be9572..1e823ca2 100644 --- a/src/OpenVDS/VDS/VolumeDataStore.h +++ b/src/OpenVDS/VDS/VolumeDataStore.h @@ -28,6 +28,7 @@ #include "VolumeDataHash.h" #include "ParsedMetadata.h" #include "GlobalStateImpl.h" +#include "LayerMetadataContainer.h" #include #include @@ -72,6 +73,9 @@ public: virtual bool AddLayer(VolumeDataLayer* volumeDataLayer, int chunkMetadataPageSize) = 0; virtual bool RemoveLayer(VolumeDataLayer* volumeDataLayer) = 0; virtual bool Close(Error &error) = 0; + virtual const LayerMetadataContainer & + GetLayerMetadataContainer() const = 0; + bool DeserializeVolumeData(const VolumeDataChunk &volumeDataChunk, const std::vector& serializedData, const std::vector& metadata, CompressionMethod compressionMethod, int32_t adaptiveLevel, VolumeDataChannelDescriptor::Format loadFormat, DataBlock &dataBlock, std::vector& target, uint64_t& targetHash, Error& error); diff --git a/src/OpenVDS/VDS/VolumeDataStoreIOManager.cpp b/src/OpenVDS/VDS/VolumeDataStoreIOManager.cpp index 6a264922..8e072565 100644 --- a/src/OpenVDS/VDS/VolumeDataStoreIOManager.cpp +++ b/src/OpenVDS/VDS/VolumeDataStoreIOManager.cpp @@ -304,7 +304,8 @@ VolumeDataStoreIOManager::AddLayer(VolumeDataLayer* volumeDataLayer, int chunkMe int pageLimit = volumeDataLayer->GetLayout()->GetDimensionality() <= 3 ? 64 : 1024; - SetMetadataStatus(GetLayerName(*volumeDataLayer), metadataStatus, pageLimit); + std::string channelName = volumeDataLayer->GetLayout()->GetChannelName(volumeDataLayer->GetChannelIndex());; + SetMetadataStatus(GetLayerName(*volumeDataLayer), channelName, metadataStatus, pageLimit); return true; } @@ -859,14 +860,30 @@ VolumeDataStoreIOManager::GetMetadataStatus(std::string const &layerName, Metada } } +bool +VolumeDataStoreIOManager::IsChannelZipped(std::string const& channelName, bool isPrimaryChannel) const +{ + (void)isPrimaryChannel; + std::unique_lock lock(m_mutex); + for (auto& metadataManager : m_metadataManagers) + { + const std::string &metaChannelName = metadataManager.second->ChannelName(); + if (metaChannelName == channelName) + { + return metadataManager.second->GetMetadataStatus().m_compressionMethod == CompressionMethod::Zip; + } + } + return false; +} + void -VolumeDataStoreIOManager::SetMetadataStatus(std::string const &layerName, MetadataStatus &metadataStatus, int pageLimit) +VolumeDataStoreIOManager::SetMetadataStatus(std::string const &layerName, std::string const &channelName, MetadataStatus &metadataStatus, int pageLimit) { std::unique_lock lock(m_mutex); if (m_metadataManagers.find(layerName) == m_metadataManagers.end()) { - m_metadataManagers.insert(std::make_pair(layerName, std::unique_ptr(new MetadataManager(m_ioManager.get(), layerName, metadataStatus, pageLimit)))); + m_metadataManagers.insert(std::make_pair(layerName, std::unique_ptr(new MetadataManager(m_ioManager.get(), layerName, channelName, metadataStatus, pageLimit)))); } } diff --git a/src/OpenVDS/VDS/VolumeDataStoreIOManager.h b/src/OpenVDS/VDS/VolumeDataStoreIOManager.h index 855a911d..84f0944d 100644 --- a/src/OpenVDS/VDS/VolumeDataStoreIOManager.h +++ b/src/OpenVDS/VDS/VolumeDataStoreIOManager.h @@ -136,7 +136,11 @@ public: bool Close(Error &error) override { return m_ioManager->Close(error); } bool GetMetadataStatus(std::string const &layerName, MetadataStatus &metadataStatus) const override; - void SetMetadataStatus(std::string const &layerName, MetadataStatus &metadataStatus, int pageLimit) override; + bool IsChannelZipped(std::string const& channelName, bool isPrimaryChannel) const override; + void SetMetadataStatus(std::string const &layerName, std::string const &channelName, MetadataStatus &metadataStatus, int pageLimit) override; + + const LayerMetadataContainer & + GetLayerMetadataContainer() const override { return *this; } VolumeDataStoreIOManager(VDS &vds, IOManager *ioManager); ~VolumeDataStoreIOManager(); diff --git a/src/OpenVDS/VDS/VolumeDataStoreVDSFile.cpp b/src/OpenVDS/VDS/VolumeDataStoreVDSFile.cpp index 19f54271..d6904c5f 100644 --- a/src/OpenVDS/VDS/VolumeDataStoreVDSFile.cpp +++ b/src/OpenVDS/VDS/VolumeDataStoreVDSFile.cpp @@ -504,7 +504,32 @@ bool VolumeDataStoreVDSFile::GetMetadataStatus(std::string const &layerName, Met return true; } -void VolumeDataStoreVDSFile::SetMetadataStatus(std::string const &layerName, MetadataStatus &metadataStatus, int pageLimit) +bool VolumeDataStoreVDSFile::IsChannelZipped(std::string const& channelName, bool isPrimary) const +{ + for(auto &layerFileEntry : m_layerFiles) + { + const LayerFile *layerFile = &layerFileEntry.second; + std::string layerName = layerFile->fileInterface->GetFileName(); + + size_t lodIndex = layerName.rfind("LOD"); + size_t dimensionsIndex = layerName.rfind("Dimensions_"); + + if(lodIndex == std::string::npos || dimensionsIndex == std::string::npos || lodIndex < dimensionsIndex) + { + continue; + } + + std::string cName = layerName.substr(0, dimensionsIndex); + + if ((isPrimary && cName.empty()) || cName == channelName) + { + return CompressionMethod(layerFile->layerMetadata.m_compressionMethod) == CompressionMethod::Zip; + } + } + return false; +} + +void VolumeDataStoreVDSFile::SetMetadataStatus(std::string const &layerName, std::string const &channelName, MetadataStatus &metadataStatus, int pageLimit) { assert(0 && "Not implemented"); } diff --git a/src/OpenVDS/VDS/VolumeDataStoreVDSFile.h b/src/OpenVDS/VDS/VolumeDataStoreVDSFile.h index 9d0afa39..ffe25735 100644 --- a/src/OpenVDS/VDS/VolumeDataStoreVDSFile.h +++ b/src/OpenVDS/VDS/VolumeDataStoreVDSFile.h @@ -81,7 +81,11 @@ public: bool Close(Error &error) override; bool GetMetadataStatus(std::string const &layerName, MetadataStatus &metadataStatus) const override; - void SetMetadataStatus(std::string const &layerName, MetadataStatus &metadataStatus, int pageLimit) override; + bool IsChannelZipped(std::string const& channelName, bool isPrimary) const override; + void SetMetadataStatus(std::string const &layerName, std::string const &channelName, MetadataStatus &metadataStatus, int pageLimit) override; + + const LayerMetadataContainer & + GetLayerMetadataContainer() const override { return *this; } VolumeDataStoreVDSFile(VDS &vds, const std::string &fileName, Mode mode, Error &error); ~VolumeDataStoreVDSFile(); diff --git a/tests/VDS/ParseVDSJsonTest.cpp b/tests/VDS/ParseVDSJsonTest.cpp index 528cc57b..73b8117c 100644 --- a/tests/VDS/ParseVDSJsonTest.cpp +++ b/tests/VDS/ParseVDSJsonTest.cpp @@ -68,8 +68,10 @@ public: return false; } } - void SetMetadataStatus(std::string const &layerName, OpenVDS::MetadataStatus &metadataStatus, int /*pageLimit*/) override + bool IsChannelZipped(std::string const&, bool) const override { return false; } + void SetMetadataStatus(std::string const &layerName, std::string const &channelName, OpenVDS::MetadataStatus &metadataStatus, int /*pageLimit*/) override { + (void)channelName; m_metadataStatusMap[layerName] = metadataStatus; } }; @@ -130,14 +132,14 @@ GTEST_TEST(VDS_integration, ParseVolumeDataLayoutAndLayerStatus) // Clear error error = OpenVDS::Error(); - // Parse volume data layout - OpenVDS::ParseVolumeDataLayout(serializedVolumeDataLayoutReference, handle.layoutDescriptor, handle.axisDescriptors, handle.channelDescriptors, handle.descriptorStrings, handle.metadataContainer, error); - EXPECT_EQ(error.code, 0); - // Parse layer status OpenVDS::ParseLayerStatus(serializedLayerStatusReference, handle, layerMetadataContainer, error); EXPECT_EQ(error.code, 0); + // Parse volume data layout + OpenVDS::ParseVolumeDataLayout(serializedVolumeDataLayoutReference, handle.layoutDescriptor, handle.axisDescriptors, handle.channelDescriptors, handle.descriptorStrings, handle.metadataContainer, layerMetadataContainer, error); + EXPECT_EQ(error.code, 0); + // Create volume data layout from descriptors CreateVolumeDataLayout(handle); -- GitLab