Commit 8afa4e0f authored by Jørgen Lind's avatar Jørgen Lind
Browse files

Merge branch 'feature/jorgen.lind/adjust_NoLossyCompressionUseZip' into 'master'

VolumeDataChannelDescriptor::NoLossyCompressionUseZip is not persisted

See merge request !577
parents 52a9bc53 91e2b785
Pipeline #98349 passed with stages
in 156 minutes and 42 seconds
......@@ -126,6 +126,7 @@ else()
VDS/ConnectionStringParser.h
VDS/GlobalStateImpl.h
VDS/StringToDouble.h
VDS/LayerMetadataContainer.h
${WAVELET_HEADERS}
${COMMON_DIR}/Base64/Base64.h
${COMMON_DIR}/ThreadPool/ThreadPool.h
......
......@@ -612,7 +612,7 @@ static bool Init(VDS *vds, VolumeDataStore *volumeDataStore, Error& error)
{
return false;
}
if(!ParseVolumeDataLayout(serializedVolumeDataLayout, vds->layoutDescriptor, vds->axisDescriptors, vds->channelDescriptors, vds->descriptorStrings, vds->metadataContainer, error))
if(!ParseVolumeDataLayout(serializedVolumeDataLayout, vds->layoutDescriptor, vds->axisDescriptors, vds->channelDescriptors, vds->descriptorStrings, vds->metadataContainer, vds->volumeDataStore->GetLayerMetadataContainer(), error))
{
return false;
}
......
/****************************************************************************
** Copyright 2022 The Open Group
** Copyright 2022 Bluware, Inc.
**
** Licensed under the Apache License, Version 2.0 (the "License");
** you may not use this file except in compliance with the License.
** You may obtain a copy of the License at
**
** http://www.apache.org/licenses/LICENSE-2.0
**
** Unless required by applicable law or agreed to in writing, software
** distributed under the License is distributed on an "AS IS" BASIS,
** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
** See the License for the specific language governing permissions and
** limitations under the License.
****************************************************************************/
#ifndef LAYERMETADATACONTAINER_H
#define LAYERMETADATACONTAINER_H
namespace OpenVDS
{
class LayerMetadataContainer
{
public:
virtual bool GetMetadataStatus(std::string const &layerName, MetadataStatus &metadataStatus) const = 0;
virtual bool IsChannelZipped(std::string const& channelName, bool isPrimary) const = 0;
virtual void SetMetadataStatus(std::string const &layerName, std::string const &channelName, MetadataStatus &metadataStatus, int pageLimit) = 0;
};
}
#endif
......@@ -69,9 +69,10 @@ MetadataPage *metadataPage;
std::vector<uint8_t> metadata;
};
MetadataManager::MetadataManager(IOManager* iomanager, std::string const& layerUrl, MetadataStatus const& metadataStatus, int pageLimit)
MetadataManager::MetadataManager(IOManager *iomanager, std::string const &layerUrl, std::string const &channelName, MetadataStatus const &metadataStatus, int pageLimit)
: m_iomanager(iomanager)
, m_layerUrl(layerUrl)
, m_channelName(channelName)
, m_metadataStatus(metadataStatus)
, m_pageLimit(pageLimit)
{
......
......@@ -94,6 +94,7 @@ namespace OpenVDS
{
IOManager *m_iomanager;
std::string m_layerUrl;
std::string m_channelName;
MetadataStatus m_metadataStatus;
......@@ -109,11 +110,12 @@ namespace OpenVDS
void LimitPages();
public:
MetadataManager(IOManager *iomanager, std::string const &layerURL, MetadataStatus const &MetadataStatus, int pageLimit);
MetadataManager(IOManager *iomanager, std::string const &layerURL, std::string const &channelName, MetadataStatus const &MetadataStatus, int pageLimit);
~MetadataManager();
const char *LayerUrl() const { return m_layerUrl.c_str(); }
const std::string &LayerUrlStr() const { return m_layerUrl; }
const std::string &ChannelName() const { return m_channelName; }
MetadataPage *LockPage(int pageIndex, bool *InitiateTransfer);
......
......@@ -910,7 +910,7 @@ bool DownloadAndParseVolumeDataLayoutAndLayerStatus(VDS& vds, Error& error)
try
{
if (!ParseVolumeDataLayout(serializedVolumeDataLayout, vds.layoutDescriptor, vds.axisDescriptors, vds.channelDescriptors, vds.descriptorStrings, vds.metadataContainer, error))
if (!ParseVolumeDataLayout(serializedVolumeDataLayout, vds.layoutDescriptor, vds.axisDescriptors, vds.channelDescriptors, vds.descriptorStrings, vds.metadataContainer, vds.volumeDataStore->GetLayerMetadataContainer(), error))
return false;
}
......@@ -926,7 +926,7 @@ bool DownloadAndParseVolumeDataLayoutAndLayerStatus(VDS& vds, Error& error)
return true;
}
bool ParseVolumeDataLayout(const std::vector<uint8_t> &json, VolumeDataLayoutDescriptor &layoutDescriptor, std::vector<VolumeDataAxisDescriptor> &axisDescriptors, std::vector<VolumeDataChannelDescriptor> &channelDescriptors, DescriptorStringContainer &descriptorStrings, MetadataContainer &metadataContainer, Error &error)
bool ParseVolumeDataLayout(const std::vector<uint8_t> &json, VolumeDataLayoutDescriptor &layoutDescriptor, std::vector<VolumeDataAxisDescriptor> &axisDescriptors, std::vector<VolumeDataChannelDescriptor> &channelDescriptors, DescriptorStringContainer &descriptorStrings, MetadataContainer &metadataContainer, const LayerMetadataContainer &layerMetadaContainer, Error &error)
{
Json::Value root;
......@@ -968,8 +968,11 @@ bool ParseVolumeDataLayout(const std::vector<uint8_t> &json, VolumeDataLayoutDes
axisDescriptors.push_back(axisDescriptor);
}
bool primary = true;
for (const Json::Value &channelDescriptorJson : root["channelDescriptors"])
{
bool noLossyCompressionUseZip = layerMetadaContainer.IsChannelZipped(channelDescriptorJson["name"].asString(), primary) && !channelDescriptorJson["allowLossyCompression"].asBool();
primary = false;
if (channelDescriptorJson["useNoValue"].asBool())
{
VolumeDataChannelDescriptor
......@@ -983,7 +986,8 @@ bool ParseVolumeDataLayout(const std::vector<uint8_t> &json, VolumeDataLayoutDes
channelDescriptorJson["mappedValues"].asInt(),
(channelDescriptorJson["discrete"].asBool() ? VolumeDataChannelDescriptor::DiscreteData : VolumeDataChannelDescriptor::Default) |
(channelDescriptorJson["renderable"].asBool() ? VolumeDataChannelDescriptor::Default : VolumeDataChannelDescriptor::NotRenderable) |
(channelDescriptorJson["allowLossyCompression"].asBool() ? VolumeDataChannelDescriptor::Default : VolumeDataChannelDescriptor::NoLossyCompression),
(channelDescriptorJson["allowLossyCompression"].asBool() ? VolumeDataChannelDescriptor::Default : VolumeDataChannelDescriptor::NoLossyCompression) |
(noLossyCompressionUseZip ? VolumeDataChannelDescriptor::NoLossyCompressionUseZip : VolumeDataChannelDescriptor::Default),
channelDescriptorJson["noValue"].asFloat(),
channelDescriptorJson["integerScale"].asFloat(),
channelDescriptorJson["integerOffset"].asFloat());
......@@ -1003,7 +1007,8 @@ bool ParseVolumeDataLayout(const std::vector<uint8_t> &json, VolumeDataLayoutDes
channelDescriptorJson["mappedValues"].asInt(),
(channelDescriptorJson["discrete"].asBool() ? VolumeDataChannelDescriptor::DiscreteData : VolumeDataChannelDescriptor::Default) |
(channelDescriptorJson["renderable"].asBool() ? VolumeDataChannelDescriptor::Default : VolumeDataChannelDescriptor::NotRenderable) |
(channelDescriptorJson["allowLossyCompression"].asBool() ? VolumeDataChannelDescriptor::Default : VolumeDataChannelDescriptor::NoLossyCompression),
(channelDescriptorJson["allowLossyCompression"].asBool() ? VolumeDataChannelDescriptor::Default : VolumeDataChannelDescriptor::NoLossyCompression) |
(noLossyCompressionUseZip ? VolumeDataChannelDescriptor::NoLossyCompressionUseZip : VolumeDataChannelDescriptor::Default),
channelDescriptorJson["integerScale"].asFloat(),
channelDescriptorJson["integerOffset"].asFloat());
......@@ -1159,12 +1164,14 @@ bool ParseLayerStatus(const std::vector<uint8_t> &json, VDS &vds, LayerMetadataC
std::string
layerName = layerStatus["layerName"].asString();
std::string
channelName = layerStatus["channelName"].asString();
if (hasChunkMetadataPages)
{
int pageLimit = vds.axisDescriptors.size() <= 3 ? 64 : 1024;
layerMetadataContainer.SetMetadataStatus(layerName, metadataStatus, pageLimit);
layerMetadataContainer.SetMetadataStatus(layerName, channelName, metadataStatus, pageLimit);
}
}
}
......
......@@ -23,7 +23,7 @@
namespace OpenVDS
{
bool ParseVolumeDataLayout(const std::vector<uint8_t> &json, VolumeDataLayoutDescriptor &layoutDescriptor, std::vector<VolumeDataAxisDescriptor> &axisDescriptors, std::vector<VolumeDataChannelDescriptor> &channelDescriptors, DescriptorStringContainer &descriptorStrings, MetadataContainer &metadataContainer, Error &error);
bool ParseVolumeDataLayout(const std::vector<uint8_t> &json, VolumeDataLayoutDescriptor &layoutDescriptor, std::vector<VolumeDataAxisDescriptor> &axisDescriptors, std::vector<VolumeDataChannelDescriptor> &channelDescriptors, DescriptorStringContainer &descriptorStrings, MetadataContainer &metadataContainer, const LayerMetadataContainer &layerMetadaContainer, Error &error);
bool ParseLayerStatus(const std::vector<uint8_t> &json, VDS &vds, LayerMetadataContainer &layerMetadataContainer, Error &error);
std::vector<uint8_t> SerializeVolumeDataLayout(VDS& vds);
std::vector<uint8_t> SerializeLayerStatus(VDS& vds, LayerMetadataContainer const &layerMetadataContainer);
......
......@@ -40,13 +40,6 @@
namespace OpenVDS
{
class LayerMetadataContainer
{
public:
virtual bool GetMetadataStatus(std::string const &layerName, MetadataStatus &metadataStatus) const = 0;
virtual void SetMetadataStatus(std::string const &layerName, MetadataStatus &metadataStatus, int pageLimit) = 0;
};
class DescriptorStringContainer
{
std::vector<std::unique_ptr<char[]>> m_descriptorStrings;
......
......@@ -28,6 +28,7 @@
#include "VolumeDataHash.h"
#include "ParsedMetadata.h"
#include "GlobalStateImpl.h"
#include "LayerMetadataContainer.h"
#include <vector>
#include <map>
......@@ -72,6 +73,9 @@ public:
virtual bool AddLayer(VolumeDataLayer* volumeDataLayer, int chunkMetadataPageSize) = 0;
virtual bool RemoveLayer(VolumeDataLayer* volumeDataLayer) = 0;
virtual bool Close(Error &error) = 0;
virtual const LayerMetadataContainer &
GetLayerMetadataContainer() const = 0;
bool DeserializeVolumeData(const VolumeDataChunk &volumeDataChunk, const std::vector<uint8_t>& serializedData, const std::vector<uint8_t>& metadata, CompressionMethod compressionMethod, int32_t adaptiveLevel, VolumeDataChannelDescriptor::Format loadFormat, DataBlock &dataBlock, std::vector<uint8_t>& target, uint64_t& targetHash, Error& error);
......
......@@ -304,7 +304,8 @@ VolumeDataStoreIOManager::AddLayer(VolumeDataLayer* volumeDataLayer, int chunkMe
int pageLimit = volumeDataLayer->GetLayout()->GetDimensionality() <= 3 ? 64 : 1024;
SetMetadataStatus(GetLayerName(*volumeDataLayer), metadataStatus, pageLimit);
std::string channelName = volumeDataLayer->GetLayout()->GetChannelName(volumeDataLayer->GetChannelIndex());;
SetMetadataStatus(GetLayerName(*volumeDataLayer), channelName, metadataStatus, pageLimit);
return true;
}
......@@ -859,14 +860,30 @@ VolumeDataStoreIOManager::GetMetadataStatus(std::string const &layerName, Metada
}
}
bool
VolumeDataStoreIOManager::IsChannelZipped(std::string const& channelName, bool isPrimaryChannel) const
{
(void)isPrimaryChannel;
std::unique_lock<std::mutex> lock(m_mutex);
for (auto& metadataManager : m_metadataManagers)
{
const std::string &metaChannelName = metadataManager.second->ChannelName();
if (metaChannelName == channelName)
{
return metadataManager.second->GetMetadataStatus().m_compressionMethod == CompressionMethod::Zip;
}
}
return false;
}
void
VolumeDataStoreIOManager::SetMetadataStatus(std::string const &layerName, MetadataStatus &metadataStatus, int pageLimit)
VolumeDataStoreIOManager::SetMetadataStatus(std::string const &layerName, std::string const &channelName, MetadataStatus &metadataStatus, int pageLimit)
{
std::unique_lock<std::mutex> lock(m_mutex);
if (m_metadataManagers.find(layerName) == m_metadataManagers.end())
{
m_metadataManagers.insert(std::make_pair(layerName, std::unique_ptr<MetadataManager>(new MetadataManager(m_ioManager.get(), layerName, metadataStatus, pageLimit))));
m_metadataManagers.insert(std::make_pair(layerName, std::unique_ptr<MetadataManager>(new MetadataManager(m_ioManager.get(), layerName, channelName, metadataStatus, pageLimit))));
}
}
......
......@@ -136,7 +136,11 @@ public:
bool Close(Error &error) override { return m_ioManager->Close(error); }
bool GetMetadataStatus(std::string const &layerName, MetadataStatus &metadataStatus) const override;
void SetMetadataStatus(std::string const &layerName, MetadataStatus &metadataStatus, int pageLimit) override;
bool IsChannelZipped(std::string const& channelName, bool isPrimaryChannel) const override;
void SetMetadataStatus(std::string const &layerName, std::string const &channelName, MetadataStatus &metadataStatus, int pageLimit) override;
const LayerMetadataContainer &
GetLayerMetadataContainer() const override { return *this; }
VolumeDataStoreIOManager(VDS &vds, IOManager *ioManager);
~VolumeDataStoreIOManager();
......
......@@ -504,7 +504,32 @@ bool VolumeDataStoreVDSFile::GetMetadataStatus(std::string const &layerName, Met
return true;
}
void VolumeDataStoreVDSFile::SetMetadataStatus(std::string const &layerName, MetadataStatus &metadataStatus, int pageLimit)
bool VolumeDataStoreVDSFile::IsChannelZipped(std::string const& channelName, bool isPrimary) const
{
for(auto &layerFileEntry : m_layerFiles)
{
const LayerFile *layerFile = &layerFileEntry.second;
std::string layerName = layerFile->fileInterface->GetFileName();
size_t lodIndex = layerName.rfind("LOD");
size_t dimensionsIndex = layerName.rfind("Dimensions_");
if(lodIndex == std::string::npos || dimensionsIndex == std::string::npos || lodIndex < dimensionsIndex)
{
continue;
}
std::string cName = layerName.substr(0, dimensionsIndex);
if ((isPrimary && cName.empty()) || cName == channelName)
{
return CompressionMethod(layerFile->layerMetadata.m_compressionMethod) == CompressionMethod::Zip;
}
}
return false;
}
void VolumeDataStoreVDSFile::SetMetadataStatus(std::string const &layerName, std::string const &channelName, MetadataStatus &metadataStatus, int pageLimit)
{
assert(0 && "Not implemented");
}
......
......@@ -81,7 +81,11 @@ public:
bool Close(Error &error) override;
bool GetMetadataStatus(std::string const &layerName, MetadataStatus &metadataStatus) const override;
void SetMetadataStatus(std::string const &layerName, MetadataStatus &metadataStatus, int pageLimit) override;
bool IsChannelZipped(std::string const& channelName, bool isPrimary) const override;
void SetMetadataStatus(std::string const &layerName, std::string const &channelName, MetadataStatus &metadataStatus, int pageLimit) override;
const LayerMetadataContainer &
GetLayerMetadataContainer() const override { return *this; }
VolumeDataStoreVDSFile(VDS &vds, const std::string &fileName, Mode mode, Error &error);
~VolumeDataStoreVDSFile();
......
......@@ -68,8 +68,10 @@ public:
return false;
}
}
void SetMetadataStatus(std::string const &layerName, OpenVDS::MetadataStatus &metadataStatus, int /*pageLimit*/) override
bool IsChannelZipped(std::string const&, bool) const override { return false; }
void SetMetadataStatus(std::string const &layerName, std::string const &channelName, OpenVDS::MetadataStatus &metadataStatus, int /*pageLimit*/) override
{
(void)channelName;
m_metadataStatusMap[layerName] = metadataStatus;
}
};
......@@ -130,14 +132,14 @@ GTEST_TEST(VDS_integration, ParseVolumeDataLayoutAndLayerStatus)
// Clear error
error = OpenVDS::Error();
// Parse volume data layout
OpenVDS::ParseVolumeDataLayout(serializedVolumeDataLayoutReference, handle.layoutDescriptor, handle.axisDescriptors, handle.channelDescriptors, handle.descriptorStrings, handle.metadataContainer, error);
EXPECT_EQ(error.code, 0);
// Parse layer status
OpenVDS::ParseLayerStatus(serializedLayerStatusReference, handle, layerMetadataContainer, error);
EXPECT_EQ(error.code, 0);
// Parse volume data layout
OpenVDS::ParseVolumeDataLayout(serializedVolumeDataLayoutReference, handle.layoutDescriptor, handle.axisDescriptors, handle.channelDescriptors, handle.descriptorStrings, handle.metadataContainer, layerMetadataContainer, error);
EXPECT_EQ(error.code, 0);
// Create volume data layout from descriptors
CreateVolumeDataLayout(handle);
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment