Merge branch 'feature/jorgen.lind/adjust_NoLossyCompressionUseZip' into 'master'

VolumeDataChannelDescriptor::NoLossyCompressionUseZip is not persisted

See merge request !577

src/OpenVDS/CMakeLists.txt

@@ -126,6 +126,7 @@ else()
     VDS/ConnectionStringParser.h
     VDS/GlobalStateImpl.h
     VDS/StringToDouble.h
+    VDS/LayerMetadataContainer.h
     ${WAVELET_HEADERS}
     ${COMMON_DIR}/Base64/Base64.h
     ${COMMON_DIR}/ThreadPool/ThreadPool.h

src/OpenVDS/OpenVDS.cpp

@@ -612,7 +612,7 @@ static bool Init(VDS *vds, VolumeDataStore *volumeDataStore, Error& error)
   {
     return false;
   }
-  if(!ParseVolumeDataLayout(serializedVolumeDataLayout, vds->layoutDescriptor, vds->axisDescriptors, vds->channelDescriptors, vds->descriptorStrings, vds->metadataContainer, error))
+  if(!ParseVolumeDataLayout(serializedVolumeDataLayout, vds->layoutDescriptor, vds->axisDescriptors, vds->channelDescriptors, vds->descriptorStrings, vds->metadataContainer, vds->volumeDataStore->GetLayerMetadataContainer(), error))
   {
     return false;
   }

src/OpenVDS/VDS/LayerMetadataContainer.h

+/****************************************************************************
+** Copyright 2022 The Open Group
+** Copyright 2022 Bluware, Inc.
+**
+** Licensed under the Apache License, Version 2.0 (the "License");
+** you may not use this file except in compliance with the License.
+** You may obtain a copy of the License at
+**
+**     http://www.apache.org/licenses/LICENSE-2.0
+**
+** Unless required by applicable law or agreed to in writing, software
+** distributed under the License is distributed on an "AS IS" BASIS,
+** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+** See the License for the specific language governing permissions and
+** limitations under the License.
+****************************************************************************/
+
+#ifndef LAYERMETADATACONTAINER_H
+#define LAYERMETADATACONTAINER_H
+
+namespace OpenVDS
+{
+class LayerMetadataContainer
+{
+public:
+  virtual bool GetMetadataStatus(std::string const &layerName, MetadataStatus &metadataStatus) const = 0;
+  virtual bool IsChannelZipped(std::string const& channelName, bool isPrimary) const = 0;
+  virtual void SetMetadataStatus(std::string const &layerName, std::string const &channelName, MetadataStatus &metadataStatus, int pageLimit) = 0;
+};
+}
+#endif

src/OpenVDS/VDS/MetadataManager.cpp

@@ -69,9 +69,10 @@ MetadataPage *metadataPage;
 std::vector<uint8_t> metadata;
 };
 
-MetadataManager::MetadataManager(IOManager* iomanager, std::string const& layerUrl, MetadataStatus const& metadataStatus, int pageLimit)
+MetadataManager::MetadataManager(IOManager *iomanager, std::string const &layerUrl, std::string const &channelName, MetadataStatus const &metadataStatus, int pageLimit)
   : m_iomanager(iomanager)
   , m_layerUrl(layerUrl)
+  , m_channelName(channelName)
   , m_metadataStatus(metadataStatus)
   , m_pageLimit(pageLimit)
 {

src/OpenVDS/VDS/MetadataManager.h

@@ -94,6 +94,7 @@ namespace OpenVDS
   {
     IOManager *m_iomanager;
     std::string m_layerUrl;
+    std::string m_channelName;
 
     MetadataStatus m_metadataStatus;
 
@@ -109,11 +110,12 @@ namespace OpenVDS
 
     void LimitPages();
   public:
-    MetadataManager(IOManager *iomanager, std::string const &layerURL, MetadataStatus const &MetadataStatus, int pageLimit);
+    MetadataManager(IOManager *iomanager, std::string const &layerURL, std::string const &channelName, MetadataStatus const &MetadataStatus, int pageLimit);
     ~MetadataManager();
 
     const char *LayerUrl() const { return m_layerUrl.c_str(); }
     const std::string &LayerUrlStr() const { return m_layerUrl; }
+    const std::string &ChannelName() const { return m_channelName; }
 
     MetadataPage *LockPage(int pageIndex, bool *InitiateTransfer);
 

src/OpenVDS/VDS/ParseVDSJson.cpp

@@ -910,7 +910,7 @@ bool DownloadAndParseVolumeDataLayoutAndLayerStatus(VDS& vds, Error& error)
 
   try
   {
-    if (!ParseVolumeDataLayout(serializedVolumeDataLayout, vds.layoutDescriptor, vds.axisDescriptors, vds.channelDescriptors, vds.descriptorStrings, vds.metadataContainer, error))
+    if (!ParseVolumeDataLayout(serializedVolumeDataLayout, vds.layoutDescriptor, vds.axisDescriptors, vds.channelDescriptors, vds.descriptorStrings, vds.metadataContainer, vds.volumeDataStore->GetLayerMetadataContainer(), error))
       return false;
   }
 
@@ -926,7 +926,7 @@ bool DownloadAndParseVolumeDataLayoutAndLayerStatus(VDS& vds, Error& error)
   return true;
 }
 
-bool ParseVolumeDataLayout(const std::vector<uint8_t> &json, VolumeDataLayoutDescriptor &layoutDescriptor, std::vector<VolumeDataAxisDescriptor> &axisDescriptors, std::vector<VolumeDataChannelDescriptor> &channelDescriptors, DescriptorStringContainer &descriptorStrings, MetadataContainer &metadataContainer, Error &error)
+bool ParseVolumeDataLayout(const std::vector<uint8_t> &json, VolumeDataLayoutDescriptor &layoutDescriptor, std::vector<VolumeDataAxisDescriptor> &axisDescriptors, std::vector<VolumeDataChannelDescriptor> &channelDescriptors, DescriptorStringContainer &descriptorStrings, MetadataContainer &metadataContainer, const LayerMetadataContainer &layerMetadaContainer, Error &error)
 {
   Json::Value root;
 
@@ -968,8 +968,11 @@ bool ParseVolumeDataLayout(const std::vector<uint8_t> &json, VolumeDataLayoutDes
       axisDescriptors.push_back(axisDescriptor);
     }
 
+    bool primary = true;
     for (const Json::Value &channelDescriptorJson : root["channelDescriptors"])
     {
+      bool noLossyCompressionUseZip = layerMetadaContainer.IsChannelZipped(channelDescriptorJson["name"].asString(), primary) && !channelDescriptorJson["allowLossyCompression"].asBool();
+      primary = false;
       if (channelDescriptorJson["useNoValue"].asBool())
       {
         VolumeDataChannelDescriptor
@@ -983,7 +986,8 @@ bool ParseVolumeDataLayout(const std::vector<uint8_t> &json, VolumeDataLayoutDes
                             channelDescriptorJson["mappedValues"].asInt(),
                             (channelDescriptorJson["discrete"].asBool() ? VolumeDataChannelDescriptor::DiscreteData : VolumeDataChannelDescriptor::Default) |
                             (channelDescriptorJson["renderable"].asBool() ? VolumeDataChannelDescriptor::Default : VolumeDataChannelDescriptor::NotRenderable) |
-                            (channelDescriptorJson["allowLossyCompression"].asBool() ? VolumeDataChannelDescriptor::Default : VolumeDataChannelDescriptor::NoLossyCompression),
+                            (channelDescriptorJson["allowLossyCompression"].asBool() ? VolumeDataChannelDescriptor::Default : VolumeDataChannelDescriptor::NoLossyCompression) |
+                            (noLossyCompressionUseZip ? VolumeDataChannelDescriptor::NoLossyCompressionUseZip : VolumeDataChannelDescriptor::Default),
                             channelDescriptorJson["noValue"].asFloat(),
                             channelDescriptorJson["integerScale"].asFloat(),
                             channelDescriptorJson["integerOffset"].asFloat());
@@ -1003,7 +1007,8 @@ bool ParseVolumeDataLayout(const std::vector<uint8_t> &json, VolumeDataLayoutDes
                             channelDescriptorJson["mappedValues"].asInt(),
                             (channelDescriptorJson["discrete"].asBool() ? VolumeDataChannelDescriptor::DiscreteData : VolumeDataChannelDescriptor::Default) |
                             (channelDescriptorJson["renderable"].asBool() ? VolumeDataChannelDescriptor::Default : VolumeDataChannelDescriptor::NotRenderable) |
-                            (channelDescriptorJson["allowLossyCompression"].asBool() ? VolumeDataChannelDescriptor::Default : VolumeDataChannelDescriptor::NoLossyCompression),
+                            (channelDescriptorJson["allowLossyCompression"].asBool() ? VolumeDataChannelDescriptor::Default : VolumeDataChannelDescriptor::NoLossyCompression) |
+                            (noLossyCompressionUseZip ? VolumeDataChannelDescriptor::NoLossyCompressionUseZip : VolumeDataChannelDescriptor::Default),
                             channelDescriptorJson["integerScale"].asFloat(),
                             channelDescriptorJson["integerOffset"].asFloat());
 
@@ -1159,12 +1164,14 @@ bool ParseLayerStatus(const std::vector<uint8_t> &json, VDS &vds, LayerMetadataC
 
       std::string
         layerName = layerStatus["layerName"].asString();
+      std::string
+        channelName = layerStatus["channelName"].asString();
 
       if (hasChunkMetadataPages)
       {
         int pageLimit = vds.axisDescriptors.size() <= 3 ? 64 : 1024;
 
-        layerMetadataContainer.SetMetadataStatus(layerName, metadataStatus, pageLimit);
+        layerMetadataContainer.SetMetadataStatus(layerName, channelName, metadataStatus, pageLimit);
       }
     }
   }

src/OpenVDS/VDS/ParseVDSJson.h

@@ -23,7 +23,7 @@
 
 namespace OpenVDS
 {
-bool ParseVolumeDataLayout(const std::vector<uint8_t> &json, VolumeDataLayoutDescriptor &layoutDescriptor, std::vector<VolumeDataAxisDescriptor> &axisDescriptors, std::vector<VolumeDataChannelDescriptor> &channelDescriptors, DescriptorStringContainer &descriptorStrings, MetadataContainer &metadataContainer, Error &error);
+bool ParseVolumeDataLayout(const std::vector<uint8_t> &json, VolumeDataLayoutDescriptor &layoutDescriptor, std::vector<VolumeDataAxisDescriptor> &axisDescriptors, std::vector<VolumeDataChannelDescriptor> &channelDescriptors, DescriptorStringContainer &descriptorStrings, MetadataContainer &metadataContainer, const LayerMetadataContainer &layerMetadaContainer, Error &error);
 bool ParseLayerStatus(const std::vector<uint8_t> &json, VDS &vds, LayerMetadataContainer &layerMetadataContainer, Error &error);
 std::vector<uint8_t> SerializeVolumeDataLayout(VDS& vds);
 std::vector<uint8_t> SerializeLayerStatus(VDS& vds, LayerMetadataContainer const &layerMetadataContainer);

src/OpenVDS/VDS/VDS.h

@@ -40,13 +40,6 @@
 namespace OpenVDS
 {
 
-class LayerMetadataContainer
-{
-public:
-  virtual bool GetMetadataStatus(std::string const &layerName, MetadataStatus &metadataStatus) const = 0;
-  virtual void SetMetadataStatus(std::string const &layerName, MetadataStatus &metadataStatus, int pageLimit) = 0;
-};
-
 class DescriptorStringContainer
 {
   std::vector<std::unique_ptr<char[]>> m_descriptorStrings;

src/OpenVDS/VDS/VolumeDataStore.h

@@ -28,6 +28,7 @@
 #include "VolumeDataHash.h"
 #include "ParsedMetadata.h"
 #include "GlobalStateImpl.h"
+#include "LayerMetadataContainer.h"
 
 #include <vector>
 #include <map>
@@ -72,6 +73,9 @@ public:
   virtual bool          AddLayer(VolumeDataLayer* volumeDataLayer, int chunkMetadataPageSize) = 0;
   virtual bool          RemoveLayer(VolumeDataLayer* volumeDataLayer) = 0;
   virtual bool          Close(Error &error) = 0;
+  virtual const LayerMetadataContainer &
+                        GetLayerMetadataContainer() const = 0;
+
 
   bool DeserializeVolumeData(const VolumeDataChunk &volumeDataChunk, const std::vector<uint8_t>& serializedData, const std::vector<uint8_t>& metadata, CompressionMethod compressionMethod, int32_t adaptiveLevel, VolumeDataChannelDescriptor::Format loadFormat, DataBlock &dataBlock, std::vector<uint8_t>& target, uint64_t& targetHash, Error& error);
 

src/OpenVDS/VDS/VolumeDataStoreIOManager.cpp

@@ -304,7 +304,8 @@ VolumeDataStoreIOManager::AddLayer(VolumeDataLayer* volumeDataLayer, int chunkMe
 
   int pageLimit = volumeDataLayer->GetLayout()->GetDimensionality() <= 3 ? 64 : 1024;
 
-  SetMetadataStatus(GetLayerName(*volumeDataLayer), metadataStatus, pageLimit);
+  std::string channelName = volumeDataLayer->GetLayout()->GetChannelName(volumeDataLayer->GetChannelIndex());;
+  SetMetadataStatus(GetLayerName(*volumeDataLayer), channelName, metadataStatus, pageLimit);
   return true;
 }
 
@@ -859,14 +860,30 @@ VolumeDataStoreIOManager::GetMetadataStatus(std::string const &layerName, Metada
   }
 }
 
+bool
+VolumeDataStoreIOManager::IsChannelZipped(std::string const& channelName, bool isPrimaryChannel) const
+{
+  (void)isPrimaryChannel;
+  std::unique_lock<std::mutex> lock(m_mutex);
+  for (auto& metadataManager : m_metadataManagers)
+  {
+    const std::string &metaChannelName = metadataManager.second->ChannelName();
+    if (metaChannelName == channelName)
+    {
+      return metadataManager.second->GetMetadataStatus().m_compressionMethod == CompressionMethod::Zip;
+    }
+  }
+  return false;
+}
+
 void
-VolumeDataStoreIOManager::SetMetadataStatus(std::string const &layerName, MetadataStatus &metadataStatus, int pageLimit)
+VolumeDataStoreIOManager::SetMetadataStatus(std::string const &layerName, std::string const &channelName, MetadataStatus &metadataStatus, int pageLimit)
 {
   std::unique_lock<std::mutex> lock(m_mutex);
 
   if (m_metadataManagers.find(layerName) == m_metadataManagers.end())
   {
-    m_metadataManagers.insert(std::make_pair(layerName, std::unique_ptr<MetadataManager>(new MetadataManager(m_ioManager.get(), layerName, metadataStatus, pageLimit))));
+    m_metadataManagers.insert(std::make_pair(layerName, std::unique_ptr<MetadataManager>(new MetadataManager(m_ioManager.get(), layerName, channelName, metadataStatus, pageLimit))));
   }
 }
 

src/OpenVDS/VDS/VolumeDataStoreIOManager.h

@@ -136,7 +136,11 @@ public:
   bool          Close(Error &error) override { return m_ioManager->Close(error); }
 
   bool          GetMetadataStatus(std::string const &layerName, MetadataStatus &metadataStatus) const override;
-  void          SetMetadataStatus(std::string const &layerName, MetadataStatus &metadataStatus, int pageLimit) override;
+  bool          IsChannelZipped(std::string const& channelName, bool isPrimaryChannel) const override;
+  void          SetMetadataStatus(std::string const &layerName, std::string const &channelName, MetadataStatus &metadataStatus, int pageLimit) override;
+
+  const LayerMetadataContainer &
+                GetLayerMetadataContainer() const override { return *this;  }
 
   VolumeDataStoreIOManager(VDS &vds, IOManager *ioManager);
  ~VolumeDataStoreIOManager();

src/OpenVDS/VDS/VolumeDataStoreVDSFile.cpp

@@ -504,7 +504,32 @@ bool VolumeDataStoreVDSFile::GetMetadataStatus(std::string const &layerName, Met
   return true;
 }
 
-void VolumeDataStoreVDSFile::SetMetadataStatus(std::string const &layerName, MetadataStatus &metadataStatus, int pageLimit)
+bool VolumeDataStoreVDSFile::IsChannelZipped(std::string const& channelName, bool isPrimary) const
+{
+  for(auto &layerFileEntry : m_layerFiles)
+  {
+    const LayerFile *layerFile = &layerFileEntry.second;
+    std::string layerName = layerFile->fileInterface->GetFileName();
+
+    size_t lodIndex = layerName.rfind("LOD");
+    size_t dimensionsIndex = layerName.rfind("Dimensions_");
+
+    if(lodIndex == std::string::npos || dimensionsIndex == std::string::npos || lodIndex < dimensionsIndex)
+    {
+      continue;
+    }
+
+    std::string cName = layerName.substr(0, dimensionsIndex);
+
+    if ((isPrimary && cName.empty()) || cName == channelName)
+    {
+      return CompressionMethod(layerFile->layerMetadata.m_compressionMethod) == CompressionMethod::Zip;
+    }
+  }
+  return false;
+}
+
+void VolumeDataStoreVDSFile::SetMetadataStatus(std::string const &layerName, std::string const &channelName, MetadataStatus &metadataStatus, int pageLimit)
 {
   assert(0 && "Not implemented");
 }

src/OpenVDS/VDS/VolumeDataStoreVDSFile.h

@@ -81,7 +81,11 @@ public:
   bool          Close(Error &error) override;
 
   bool          GetMetadataStatus(std::string const &layerName, MetadataStatus &metadataStatus) const override;
-  void          SetMetadataStatus(std::string const &layerName, MetadataStatus &metadataStatus, int pageLimit) override;
+  bool          IsChannelZipped(std::string const& channelName, bool isPrimary) const override;
+  void          SetMetadataStatus(std::string const &layerName, std::string const &channelName, MetadataStatus &metadataStatus, int pageLimit) override;
+
+  const LayerMetadataContainer &
+                GetLayerMetadataContainer() const override { return *this;  }
 
   VolumeDataStoreVDSFile(VDS &vds, const std::string &fileName, Mode mode, Error &error);
  ~VolumeDataStoreVDSFile();

tests/VDS/ParseVDSJsonTest.cpp

@@ -68,8 +68,10 @@ public:
       return false;
     }
   }
-  void SetMetadataStatus(std::string const &layerName, OpenVDS::MetadataStatus &metadataStatus, int /*pageLimit*/) override
+  bool IsChannelZipped(std::string const&, bool) const override { return false; }
+  void SetMetadataStatus(std::string const &layerName, std::string const &channelName, OpenVDS::MetadataStatus &metadataStatus, int /*pageLimit*/) override
   {
+    (void)channelName;
     m_metadataStatusMap[layerName] = metadataStatus;
   }
 };
@@ -130,14 +132,14 @@ GTEST_TEST(VDS_integration, ParseVolumeDataLayoutAndLayerStatus)
   // Clear error
   error = OpenVDS::Error();
 
-  // Parse volume data layout
-  OpenVDS::ParseVolumeDataLayout(serializedVolumeDataLayoutReference, handle.layoutDescriptor, handle.axisDescriptors, handle.channelDescriptors, handle.descriptorStrings, handle.metadataContainer, error);
-  EXPECT_EQ(error.code, 0);
-
   // Parse layer status
   OpenVDS::ParseLayerStatus(serializedLayerStatusReference, handle, layerMetadataContainer, error);
   EXPECT_EQ(error.code, 0);
 
+  // Parse volume data layout
+  OpenVDS::ParseVolumeDataLayout(serializedVolumeDataLayoutReference, handle.layoutDescriptor, handle.axisDescriptors, handle.channelDescriptors, handle.descriptorStrings, handle.metadataContainer, layerMetadataContainer, error);
+  EXPECT_EQ(error.code, 0);
+
   // Create volume data layout from descriptors
   CreateVolumeDataLayout(handle);