Commit edf890e8 authored by Jørgen Lind's avatar Jørgen Lind
Browse files

Merge branch feature/jorgen.lind/update_2.1 with refs/heads/2.1 into refs/merge-requests/491/train

parents 480ee163 9057f524
Pipeline #76980 passed with stages
in 20 minutes and 16 seconds
......@@ -104,7 +104,7 @@ function(Fetch3rdParty)
Fetch3rdParty_Package(google-cloud-cpp 1.14.0 https://codeload.github.com/googleapis/google-cloud-cpp/tar.gz/v1.14.0 SHA256=839b2d4dcb36a671734dac6b30ea8c298bbeaafcf7a45ee4a7d7aa5986b16569)
Fetch3rdParty_Package(libxml2 2.9.12a https://codeload.github.com/GNOME/libxml2/tar.gz/v2.9.12 SHA256=8a4ddd706419c210b30b8978a51388937fd9362c34fc9a3d69e4fcc6f8055ee0)
Fetch3rdParty_Package(azure-sdk-for-cpp 12.0.0b11 https://codeload.github.com/Azure/azure-sdk-for-cpp/tar.gz/azure-storage-blobs_12.0.0-beta.11 SHA256=b111636335340e3e7a5675351216dde606b8345b9906ed2f42ff8a794f5f2375)
Fetch3rdParty_Package(dms 70892e2e3 https://community.opengroup.org/osdu/platform/domain-data-mgmt-services/seismic/seismic-dms-suite/seismic-store-cpp-lib/-/archive/70892e2e344693652992cda34e362774952a34cd/seismic-store-cpp-lib-master.tar.gz SHA256=f421b2fd2cbfe116ade2b6b109d9adaa417a1d27516afab0a70f3518b7b93a17)
Fetch3rdParty_Package(dms 5c5220922f https://community.opengroup.org/osdu/platform/domain-data-mgmt-services/seismic/seismic-dms-suite/seismic-store-cpp-lib/-/archive/5c5220922f6a23f3b82405102c3d12b8b540a14e/seismic-store-cpp-lib-master.tar.gz SHA256=c3574fa909e5fbe87116d15bc3a4b6a59887b236e995b9a3b17cf4f879849f26)
#Fetch3rdParty_Git(dms git git@community.opengroup.org:osdu/platform/domain-data-mgmt-services/seismic/seismic-dms-suite/seismic-store-cpp-lib.git master)
Fetch3rdParty_File(testng java 6.14.3 jar https://repo1.maven.org/maven2/org/testng/testng/6.14.3/testng-6.14.3.jar MD5=9f17a8f9e99165e148c42b21f4b63d7c)
Fetch3rdParty_File(jcommander java 1.72 jar https://repo1.maven.org/maven2/com/beust/jcommander/1.72/jcommander-1.72.jar MD5=9fde6bc0ba1032eceb7267fd1ad1657b)
......
......@@ -6,10 +6,12 @@ import sys
import math
parser = argparse.ArgumentParser(description="Converter converting numpy array file to VDS.")
parser.add_argument("--url", help="The url for where to put the VDS")
parser.add_argument("--url", help="The url for where to put the VDS", required=True)
parser.add_argument("--connection", default="", help="The connection string for the url.")
parser.add_argument("--value-range-min", default=0, help="The lower bounds of the value range for the data.")
parser.add_argument("--value-range-max", default=0, help="The upper bounds of the value range for the data.")
parser.add_argument("--npz-array-name", default="data", help="The npz array name.")
parser.add_argument("--npy", default=False, help="The input file is npy and not npz.")
parser.add_argument("npz_file", metavar="file", help="The npz file")
if len(sys.argv) == 1:
......@@ -19,7 +21,8 @@ if len(sys.argv) == 1:
args = parser.parse_args()
print("Opening file {}:".format(args.npz_file))
data = np.load(args.npz_file,mmap_mode='r')['data']
np_file = np.load(args.npz_file,mmap_mode='r')
data = np_file[args.npz_array_name] if isinstance(np_file, np.lib.npyio.NpzFile) else np_file
data= data.transpose()
print("Done reading file {}".format(args.npz_file))
......@@ -53,7 +56,7 @@ if openvds.isCompressionMethodSupported(openvds.CompressionMethod.WaveletLossles
2, 2, 4,
openvds.VolumeDataLayoutDescriptor.LODLevels.LODLevels_None,
openvds.VolumeDataLayoutDescriptor.Options.Options_None)
compressionMethod = openvds.CompresionMethod.WaveletLossless
compressionMethod = openvds.CompressionMethod.WaveletLossless
axisDescriptors = [ openvds.VolumeDataAxisDescriptor(x_samples, openvds.KnownAxisNames.x(), openvds.KnownUnitNames.meter(), 0.0, 2000.0),
openvds.VolumeDataAxisDescriptor(y_samples, openvds.KnownAxisNames.y(), openvds.KnownUnitNames.meter(), 0.0, 2000.0),
......
......@@ -33,6 +33,15 @@ public class OpenVdsDemo {
}
static void process(String[] args) throws Exception {
String url = null;
String connection = null;
for (int i = 1; i < args.length; i+=2)
{
if (args[i].equals("--url"))
url = args[i+1];
if (args[i].equals("--connection"))
connection = args[i+1];
}
int output_width = 1000;
int output_height = 1000;
int axis_position = Integer.MIN_VALUE;
......@@ -41,13 +50,22 @@ public class OpenVdsDemo {
int nXSamples = 64, nYSamples = 64, nZSamples = 64;
VolumeDataChannelDescriptor.Format format = VolumeDataChannelDescriptor.Format.FORMAT_U8;
System.out.println("Create MemoryVdsGenerator...");
MemoryVdsGenerator generator = new MemoryVdsGenerator(nXSamples, nYSamples, nZSamples, format);
VdsHandle vds;
if (url != null && !url.isEmpty())
{
System.out.println("Open existing VDS with: " + url);
vds = OpenVDS.open(url, connection);
}
else
{
System.out.println("Create MemoryVdsGenerator...");
vds = new MemoryVdsGenerator(nXSamples, nYSamples, nZSamples, format);
}
VolumeDataLayout layout = generator.getLayout();
VolumeDataLayout layout = vds.getLayout();
printLayout(layout);
VolumeDataAccessManager accessManager = generator.getAccessManager();
VolumeDataAccessManager accessManager = vds.getAccessManager();
int[] axis_mapper = {0, 1, 2};
int[] sampleCount = new int[3];
......@@ -105,7 +123,7 @@ public class OpenVdsDemo {
// Test of finalization. Normally manual call of release() is not needed:
// it is called from JniPointer.finalize()
System.out.println("Release objects...");
generator.release();
vds.release();
System.out.println("Finished");
}
......
......@@ -15,6 +15,10 @@
** limitations under the License.
****************************************************************************/
#if defined(_MSC_VER) && _MSC_VER > 1929
#include <corecrt.h> // fix visual studio 2022 compile complaining about _invalid_parameter
#endif
#include <pybind11/pybind11.h>
#include <pybind11/stl.h>
#include <pybind11/numpy.h>
......
......@@ -827,6 +827,14 @@ bool Wavelet::DeCompress(bool isTransform, int32_t decompressInfo, float decompr
createTransformData(transformData, m_bandSize, m_transformMask, m_transformIterations);
DecompressAdaptiveMode
decompressAdaptiveMode = DecompressAdaptiveMode::AssumeNoOverwrite;
if(WaveletAdaptiveLL_IsWaveletStreamEncodedWithBug(transformData, m_transformIterations, m_transformMask))
{
decompressAdaptiveMode = isLossless ? DecompressAdaptiveMode::AllowOverwrite : DecompressAdaptiveMode::PreventOverwrite;
}
int cpuTempDecodeSizeNeeded = CalculateBufferSizeNeeded(m_allocatedSizeX * m_allocatedSizeY * m_allocatedSizeZ, m_allocatedHalfSizeX * m_allocatedHalfSizeY * m_allocatedHalfSizeZ);
std::vector<uint8_t> cpuTempData;
......@@ -837,7 +845,7 @@ bool Wavelet::DeCompress(bool isTransform, int32_t decompressInfo, float decompr
m_pixelSetChildren.get(), m_pixelSetChildrenCount, m_pixelSetPixelInSignificant.get(), m_pixelSetPixelInSignificantCount,
m_allocatedHalfSizeX, m_allocatedHalfSizeX * m_allocatedHalfSizeY, cpuTempData.data(), m_allocatedHalfSizeX * m_allocatedHalfSizeY * m_allocatedHalfSizeZ, m_allocatedSizeX * m_allocatedSizeY * m_allocatedSizeZ, decompressLevel, isInteger);
int size = WaveletAdaptiveLL_DecompressAdaptive(decodeIterator);
int size = WaveletAdaptiveLL_DecompressAdaptive(decodeIterator, decompressAdaptiveMode);
(void)size;
if (isLossless)
......
......@@ -921,8 +921,8 @@ static inline void ReadWriteStartValues(const WaveletAdaptiveLL_DecodeIterator&
}
}
template<bool isMultiple, bool isAllNormal, int multiple>
static void DecodeAllBits(const WaveletAdaptiveLL_DecodeIterator& decodeIterator, float threshold, const int* valueEncoding, const int* valuesAtLevel, const int values, const int startDecodeBits, const int maxDecodeLevel)
template<bool isMultiple, bool isAllNormal, int multiple, bool isPreventOverwrite>
static void DecodeAllBits(const WaveletAdaptiveLL_DecodeIterator& decodeIterator, float threshold, const int* valueEncoding, const int* valuesAtLevel, const int values, const int startDecodeBits, const int maxDecodeLevel, int threads)
{
const float minLevelThreshold = threshold * powf(2.0f, (float)maxDecodeLevel);
......@@ -935,15 +935,6 @@ static void DecodeAllBits(const WaveletAdaptiveLL_DecodeIterator& decodeIterator
positions += decodeIterator.maxChildren;
}
// The limit value 8 seems to provide good performance across
// different hardware configurations. This OMP block originally
// had no thread limit, which resulted in poor performance on
// some hardware.
// Alternatively, we want to use fewer than 8 threads if OMP
// max threads is less than that.
int threads = std::min(8, omp_get_max_threads());
(void)threads;
float startValue = 0.0f;
if (!decodeIterator.isInteger)
......@@ -1017,22 +1008,31 @@ static void DecodeAllBits(const WaveletAdaptiveLL_DecodeIterator& decodeIterator
if (isAllNormal)
{
decodeIterator.picture[currentPos + decodeIterator.screenDisplacementAllNormal[child]] = rvalue;
if(!isPreventOverwrite || decodeIterator.picture[currentPos + decodeIterator.screenDisplacementAllNormal[child]] == 0.0f)
{
decodeIterator.picture[currentPos + decodeIterator.screenDisplacementAllNormal[child]] = rvalue;
}
}
else
{
decodeIterator.picture[currentPos + decodeIterator.screenDisplacement[child]] = rvalue;
if(!isPreventOverwrite || decodeIterator.picture[currentPos + decodeIterator.screenDisplacement[child]] == 0.0f)
{
decodeIterator.picture[currentPos + decodeIterator.screenDisplacement[child]] = rvalue;
}
}
}
else
{
decodeIterator.picture[positions[parentValue]] = rvalue;
if(!isPreventOverwrite || decodeIterator.picture[positions[parentValue]] == 0.0f)
{
decodeIterator.picture[positions[parentValue]] = rvalue;
}
}
}
}
}
int32_t WaveletAdaptiveLL_DecompressAdaptive(WaveletAdaptiveLL_DecodeIterator decodeIterator)
int32_t WaveletAdaptiveLL_DecompressAdaptive(WaveletAdaptiveLL_DecodeIterator decodeIterator, DecompressAdaptiveMode decompressAdaptiveMode)
{
int streamSize = 0;
......@@ -1067,16 +1067,18 @@ int32_t WaveletAdaptiveLL_DecompressAdaptive(WaveletAdaptiveLL_DecodeIterator de
}
}
if (decodeIterator.decompressLevel > decodeIterator.decodeBits)
{
;
}
memset(decodeIterator.picture, 0, size_t(decodeIterator.sizeX) * size_t(decodeIterator.sizeY) * size_t(decodeIterator.sizeZ) * sizeof(float));
ReadWriteStartValues(decodeIterator, decodeIterator.pixelSetPixelInSignificant, decodeIterator.pixelSetPixelInsignificantCount, decodeIterator.pixelSetChildren, decodeIterator.pixelSetChildrenCount);
// The limit value 8 seems to provide good performance across
// different hardware configurations. This OMP block originally
// had no thread limit, which resulted in poor performance on
// some hardware.
// Alternatively, we want to use fewer than 8 threads if OMP
// max threads is less than that.
int threads = (decompressAdaptiveMode == DecompressAdaptiveMode::AssumeNoOverwrite) ? std::min(8, omp_get_max_threads()) : 1;
if (decodeIterator.decompressLevel <= decodeIterator.decodeBits)
{
int multiple = decodeIterator.isAllNormal ? decodeIterator.children[1] : 1 << decodeIterator.dimensions;
......@@ -1087,17 +1089,37 @@ int32_t WaveletAdaptiveLL_DecompressAdaptive(WaveletAdaptiveLL_DecodeIterator de
{
if (decodeIterator.isAllNormal)
{
if (multiple == 1) DecodeAllBits<true, true, 1>(decodeIterator, decodeIterator.threshold, decodeIterator.valueEncodingMultiple, decodeIterator.valuesAtLevelMultiple, valuesMultiple, decodeIterator.decodeBits, decodeIterator.decompressLevel);
else if (multiple == 2) DecodeAllBits<true, true, 2>(decodeIterator, decodeIterator.threshold, decodeIterator.valueEncodingMultiple, decodeIterator.valuesAtLevelMultiple, valuesMultiple, decodeIterator.decodeBits, decodeIterator.decompressLevel);
else if (multiple == 4) DecodeAllBits<true, true, 4>(decodeIterator, decodeIterator.threshold, decodeIterator.valueEncodingMultiple, decodeIterator.valuesAtLevelMultiple, valuesMultiple, decodeIterator.decodeBits, decodeIterator.decompressLevel);
else if (multiple == 8) DecodeAllBits<true, true, 8>(decodeIterator, decodeIterator.threshold, decodeIterator.valueEncodingMultiple, decodeIterator.valuesAtLevelMultiple, valuesMultiple, decodeIterator.decodeBits, decodeIterator.decompressLevel);
if(decompressAdaptiveMode != DecompressAdaptiveMode::PreventOverwrite)
{
if (multiple == 1) DecodeAllBits<true, true, 1, false>(decodeIterator, decodeIterator.threshold, decodeIterator.valueEncodingMultiple, decodeIterator.valuesAtLevelMultiple, valuesMultiple, decodeIterator.decodeBits, decodeIterator.decompressLevel, threads);
else if (multiple == 2) DecodeAllBits<true, true, 2, false>(decodeIterator, decodeIterator.threshold, decodeIterator.valueEncodingMultiple, decodeIterator.valuesAtLevelMultiple, valuesMultiple, decodeIterator.decodeBits, decodeIterator.decompressLevel, threads);
else if (multiple == 4) DecodeAllBits<true, true, 4, false>(decodeIterator, decodeIterator.threshold, decodeIterator.valueEncodingMultiple, decodeIterator.valuesAtLevelMultiple, valuesMultiple, decodeIterator.decodeBits, decodeIterator.decompressLevel, threads);
else if (multiple == 8) DecodeAllBits<true, true, 8, false>(decodeIterator, decodeIterator.threshold, decodeIterator.valueEncodingMultiple, decodeIterator.valuesAtLevelMultiple, valuesMultiple, decodeIterator.decodeBits, decodeIterator.decompressLevel, threads);
}
else
{
if (multiple == 1) DecodeAllBits<true, true, 1, true>(decodeIterator, decodeIterator.threshold, decodeIterator.valueEncodingMultiple, decodeIterator.valuesAtLevelMultiple, valuesMultiple, decodeIterator.decodeBits, decodeIterator.decompressLevel, threads);
else if (multiple == 2) DecodeAllBits<true, true, 2, true>(decodeIterator, decodeIterator.threshold, decodeIterator.valueEncodingMultiple, decodeIterator.valuesAtLevelMultiple, valuesMultiple, decodeIterator.decodeBits, decodeIterator.decompressLevel, threads);
else if (multiple == 4) DecodeAllBits<true, true, 4, true>(decodeIterator, decodeIterator.threshold, decodeIterator.valueEncodingMultiple, decodeIterator.valuesAtLevelMultiple, valuesMultiple, decodeIterator.decodeBits, decodeIterator.decompressLevel, threads);
else if (multiple == 8) DecodeAllBits<true, true, 8, true>(decodeIterator, decodeIterator.threshold, decodeIterator.valueEncodingMultiple, decodeIterator.valuesAtLevelMultiple, valuesMultiple, decodeIterator.decodeBits, decodeIterator.decompressLevel, threads);
}
}
else
{
// multiple can't be one if using 1 << nDimensions
if (multiple == 2) DecodeAllBits<true, false, 2>(decodeIterator, decodeIterator.threshold, decodeIterator.valueEncodingMultiple, decodeIterator.valuesAtLevelMultiple, valuesMultiple, decodeIterator.decodeBits, decodeIterator.decompressLevel);
else if (multiple == 4) DecodeAllBits<true, false, 4>(decodeIterator, decodeIterator.threshold, decodeIterator.valueEncodingMultiple, decodeIterator.valuesAtLevelMultiple, valuesMultiple, decodeIterator.decodeBits, decodeIterator.decompressLevel);
else if (multiple == 8) DecodeAllBits<true, false, 8>(decodeIterator, decodeIterator.threshold, decodeIterator.valueEncodingMultiple, decodeIterator.valuesAtLevelMultiple, valuesMultiple, decodeIterator.decodeBits, decodeIterator.decompressLevel);
if(decompressAdaptiveMode != DecompressAdaptiveMode::PreventOverwrite)
{
// multiple can't be one if using 1 << nDimensions
if (multiple == 2) DecodeAllBits<true, false, 2, false>(decodeIterator, decodeIterator.threshold, decodeIterator.valueEncodingMultiple, decodeIterator.valuesAtLevelMultiple, valuesMultiple, decodeIterator.decodeBits, decodeIterator.decompressLevel, threads);
else if (multiple == 4) DecodeAllBits<true, false, 4, false>(decodeIterator, decodeIterator.threshold, decodeIterator.valueEncodingMultiple, decodeIterator.valuesAtLevelMultiple, valuesMultiple, decodeIterator.decodeBits, decodeIterator.decompressLevel, threads);
else if (multiple == 8) DecodeAllBits<true, false, 8, false>(decodeIterator, decodeIterator.threshold, decodeIterator.valueEncodingMultiple, decodeIterator.valuesAtLevelMultiple, valuesMultiple, decodeIterator.decodeBits, decodeIterator.decompressLevel, threads);
}
else
{
// multiple can't be one if using 1 << nDimensions
if (multiple == 2) DecodeAllBits<true, false, 2, true>(decodeIterator, decodeIterator.threshold, decodeIterator.valueEncodingMultiple, decodeIterator.valuesAtLevelMultiple, valuesMultiple, decodeIterator.decodeBits, decodeIterator.decompressLevel, threads);
else if (multiple == 4) DecodeAllBits<true, false, 4, true>(decodeIterator, decodeIterator.threshold, decodeIterator.valueEncodingMultiple, decodeIterator.valuesAtLevelMultiple, valuesMultiple, decodeIterator.decodeBits, decodeIterator.decompressLevel, threads);
else if (multiple == 8) DecodeAllBits<true, false, 8, true>(decodeIterator, decodeIterator.threshold, decodeIterator.valueEncodingMultiple, decodeIterator.valuesAtLevelMultiple, valuesMultiple, decodeIterator.decodeBits, decodeIterator.decompressLevel, threads);
}
}
}
......@@ -1107,7 +1129,14 @@ int32_t WaveletAdaptiveLL_DecompressAdaptive(WaveletAdaptiveLL_DecodeIterator de
if (valuesSingle)
{
DecodeAllBits<false, false, 1>(decodeIterator, decodeIterator.threshold, decodeIterator.valueEncodingSingle, decodeIterator.valuesAtLevelSingle, valuesSingle, decodeIterator.decodeBits, decodeIterator.decompressLevel);
if(decompressAdaptiveMode != DecompressAdaptiveMode::PreventOverwrite)
{
DecodeAllBits<false, false, 1, false>(decodeIterator, decodeIterator.threshold, decodeIterator.valueEncodingSingle, decodeIterator.valuesAtLevelSingle, valuesSingle, decodeIterator.decodeBits, decodeIterator.decompressLevel, threads);
}
else
{
DecodeAllBits<false, false, 1, true>(decodeIterator, decodeIterator.threshold, decodeIterator.valueEncodingSingle, decodeIterator.valuesAtLevelSingle, valuesSingle, decodeIterator.decodeBits, decodeIterator.decompressLevel, threads);
}
}
}
}
......@@ -1209,4 +1238,64 @@ int32_t WaveletAdaptiveLL_DecompressLossless(uint8_t *in, float *pic, int32_t si
return (int)(in + totalSize - start);
}
bool CheckIfWaveletSectorHasStreamBug(Wavelet_TransformData * transformData, int transformIndex, int iSector)
{
if (transformData[transformIndex].isNormal)
{
return false;
}
// check if this band is actually normal!
if (transformData[transformIndex].subBandInfo[iSector].childSubBand == 1)
{
// any extra children?
if (transformData[transformIndex].subBandInfo[iSector].extraChildEdge[0][0] == -1 &&
transformData[transformIndex].subBandInfo[iSector].extraChildEdge[0][1] == -1 &&
transformData[transformIndex].subBandInfo[iSector].extraChildEdge[0][2] == -1)
{
// all dividable by two?
if (!(transformData[transformIndex].subBandInfo[iSector].legalChildEdge[0][0] & 1) &&
!(transformData[transformIndex].subBandInfo[iSector].legalChildEdge[0][1] & 1) &&
(!(transformData[transformIndex].subBandInfo[iSector].legalChildEdge[0][2] & 1) || transformData[transformIndex].child == 4))
{
// Are we in the case of 4 children
if (transformData[transformIndex].child == 4)
{
// Are we odd numbered
if (transformData[transformIndex].subBandInfo[iSector].legalChildEdge[0][2] & 1)
{
// And do we have children
if (transformData[transformIndex].childCount[2] > 1)
{
return true;
}
}
}
}
}
}
return false;
}
bool WaveletAdaptiveLL_IsWaveletStreamEncodedWithBug(Wavelet_TransformData *transformData, int transformDataCount, int *transformMask)
{
for (int transformIndex = 1; transformIndex < transformDataCount; transformIndex++)
{
// go through sectors
for (int iSector = 1; iSector < 8; iSector++)
{
if ((iSector & transformMask[transformIndex]) == iSector)
{
if (CheckIfWaveletSectorHasStreamBug(transformData, transformIndex, iSector))
{
return true;
}
}
}
}
return false;
}
}
......@@ -25,6 +25,13 @@
namespace OpenVDS
{
enum class DecompressAdaptiveMode
{
AssumeNoOverwrite,
AllowOverwrite,
PreventOverwrite
};
struct WaveletAdaptiveLL_DecodeIterator
{
Wavelet_FastDecodeInsig *insig;
......@@ -87,8 +94,9 @@ WaveletAdaptiveLL_DecodeIterator WaveletAdaptiveLL_CreateDecodeIterator(uint8_t
Wavelet_PixelSetChildren *pixelSetChildren, int pixelSetChildrenCount, Wavelet_PixelSetPixel *pixelSetPixelInSignificant, int pixelSetPixelInsignificantCount,
int maxSizeX, int maxSizeXY, uint8_t *tempBufferCPU, int maxChildren, int maxPixels, int decompressLevel, bool isInteger);
int32_t WaveletAdaptiveLL_DecompressAdaptive(WaveletAdaptiveLL_DecodeIterator decodeIterator);
int32_t WaveletAdaptiveLL_DecompressAdaptive(WaveletAdaptiveLL_DecodeIterator decodeIterator, DecompressAdaptiveMode decompressAdaptiveMode);
int32_t WaveletAdaptiveLL_DecompressLossless(uint8_t *in, float *pic, int32_t sizeX, int32_t sizeY, int32_t sizeZ, int32_t allocatedSizeX, int32_t allocatedSizeXY);
bool WaveletAdaptiveLL_IsWaveletStreamEncodedWithBug(Wavelet_TransformData *transformData, int transformDataCount, int *transformMask);
}
#endif
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment