Commit 7f6033f2 authored by Jørgen Lind's avatar Jørgen Lind
Browse files

Implement value conversion for 1 bit source and target

parent a71ffb4b
......@@ -165,7 +165,7 @@ inline uint32_t GetAllocatedByteSize(const DataBlock &block)
inline int32_t GetAllocatedByteSizeForSize(const int32_t size)
{
return (size + 7) & -8;
return size == 1 ? 1 : (size + 7) & -8;
}
template <typename T>
......
......@@ -598,13 +598,14 @@ static int32_t CombineAndReduceDimensions (int32_t (&sourceSize )[DataStoreDime
}
template <typename T, bool isUseNoValue>
static void CopyTo1Bit(uint8_t *target, int64_t targetBit, const QuantizingValueConverterWithNoValue<float, T, isUseNoValue> &valueConverter, const T *source, float noValue)
static void CopyTo1Bit(uint8_t *target, int64_t targetBit, const QuantizingValueConverterWithNoValue<float, T, isUseNoValue> &valueConverter, const T *source, float noValue, int32_t count)
{
uint8_t bits = 0;
target += targetBit / 8;
uint8_t bits = *target;
int32_t mask = 1;
//for(int32_t voxel = 0; voxel < voxelsPerRowSource; voxel++)
for(int32_t voxel = 0; voxel < count; voxel++)
{
float value = valueConverter.ConvertValue(*source++);
if (!isUseNoValue || value != noValue)
......@@ -633,44 +634,26 @@ static void CopyTo1Bit(uint8_t *target, int64_t targetBit, const QuantizingValue
}
*target++ = bits;
}
uint8_t fillBits = (0x80 & (*target - 1)) ? 0xff : 0x00;
// for (int32_t byte = (voxelsPerRowSource + 7) / 8; byte < bytesPerRowTarget; byte++)
// {
// *target++ = fillBits;
// }
}
template <typename T>
static void CopyFrom1Bit(T *target, const uint8_t *source, int32_t bits)
{
// for (int32_t iRow = 0; iRow < nRows; iRow++)
// {
// U8
// uBits = 0;
//
// B32
// bMask = 0x80;
//
// for (I32 iVoxel = 0; iVoxel < nVoxelsPerRowTarget; iVoxel++)
// {
// bMask <<= 1;
//
// if (bMask == 0x100)
// {
// uBits = *puSource++;
// bMask = 1;
// }
//
// *ptTarget++ = (T)((uBits & bMask) ? 1.0f : 0.0f);
// }
//
// I32
// iCurrentByte = (nVoxelsPerRowTarget + 7) / 8;
//
// puSource += nBytesPerRowSource - iCurrentByte;
// }
static void CopyFrom1Bit(T *target, const uint8_t *source, uint64_t bitIndex, int32_t count)
{
source += bitIndex / 8;
uint8_t bits = *source;
int32_t mask = 1 << (bitIndex % 8);
for (int i = 0; i < count; i++)
{
*target = (bits & mask)? T(1) : T(0);
target++;
mask <<= 1;
if (mask == 0x100)
{
source++;
bits = *source;
mask = 1;
}
}
}
static force_inline void CopyBits(void* target, int64_t targetBit, const void* source, int64_t sourceBit, int32_t bits)
......@@ -758,6 +741,7 @@ struct BlockCopy
uint8_t *targetLocalBase = reinterpret_cast<uint8_t *>(target) + targetLocalBaseSize;
QuantizingValueConverterWithNoValue<T, S, noValue> valueConverter = createValueConverter<T,S,noValue>(conversionParamters);
QuantizingValueConverterWithNoValue<float, S, noValue> floatValueConverter = createValueConverter<float,S,noValue>(conversionParamters);
for (int dimension3 = 0; dimension3 < overlapSize[3]; dimension3++)
{
......@@ -771,19 +755,17 @@ struct BlockCopy
{
if (sourceOneBit)
{
//should not reach this path
assert(false);
//mempcy(target, puSource, nVoxels * sizeof(U8));
}
else
{
assert(false);
//CopyTo1Bit(target, targetLocalBase + targetLocal, createValueConverter<T,S,noValue>(conversionParamters), static_cast<const S *>(sourceLocalBase + sourceLocal), overlapSize[0] * (int32_t)sizeof(S));
CopyTo1Bit(static_cast<uint8_t *>(target), targetLocalBaseSize + targetLocal, floatValueConverter, reinterpret_cast<const S *>(sourceLocalBase + sourceLocal), conversionParamters.noValue, overlapSize[0]);
}
}
else if(sourceOneBit)
{
assert(false);
CopyFrom1Bit(target, static_cast<const uint8_t *>(source), 0);
CopyFrom1Bit(reinterpret_cast<T *>(targetLocalBase + targetLocal), static_cast<const uint8_t *>(source), sourceLocalBaseSize + sourceLocal, overlapSize[0]);
} else
{
ConvertAndCopy(reinterpret_cast<T *>(targetLocalBase + targetLocal), reinterpret_cast<const S *>(sourceLocalBase + sourceLocal), valueConverter, overlapSize[0]);
......@@ -833,7 +815,7 @@ static void DispatchBlockCopy3(void *target, const int32_t (&targetOffset)[DataS
void const *source, const int32_t (&sourceOffset)[DataStoreDimensionality_Max], const int32_t (&sourceSize)[DataStoreDimensionality_Max],
const int32_t (&overlapSize) [DataStoreDimensionality_Max], const ConversionParameters &conversionParameters)
{
if (conversionParameters.hasReplacementNoValue)
if (conversionParameters.hasReplacementNoValue && !(targetOneBit && sourceOneBit))
BlockCopy<T, targetOneBit, S, sourceOneBit, true>::Do(target, targetOffset, targetSize, source, sourceOffset, sourceSize, overlapSize, conversionParameters);
else
BlockCopy<T, targetOneBit, S, sourceOneBit, false>::Do(target, targetOffset, targetSize, source, sourceOffset, sourceSize, overlapSize, conversionParameters);
......@@ -1201,7 +1183,7 @@ void ProjectValuesKernel(T *output, const T *input, const ProjectVars &projectVa
voxelCenterInIndex[projectVars.projectionDimension] = zValue;
int32_t voxelInIndexInt[Dimensionality_Max];
int32_t voxelInIndexInt[Dimensionality_Max];
voxelInIndexInt[0] = voxelOutIndex[0];
voxelInIndexInt[1] = voxelOutIndex[1];
voxelInIndexInt[2] = voxelOutIndex[2];
......
......@@ -25,42 +25,162 @@
#include "../utils/GenerateVDS.h"
TEST(OpenVDS_integration, RequestVolumeSubsetFormat)
void setupIntegrationTestHandle(std::unique_ptr<OpenVDS::VDS, decltype(&OpenVDS::Close)> &handle)
{
OpenVDS::Error error;
OpenVDS::AWSOpenOptions options;
options.region = TEST_AWS_REGION;
options.bucket = TEST_AWS_BUCKET;
options.key = TEST_AWS_OBJECTID;
if(options.region.empty() || options.bucket.empty() || options.key.empty())
{
//GTEST_SKIP() << "Environment variables not set";
}
//ASSERT_TRUE(options.region.size() && options.bucket.size() && options.key.size());
handle.reset(OpenVDS::Open(options, error));
//ASSERT_TRUE(handle);
}
void setupNoiceTestHandle(std::unique_ptr<OpenVDS::VDS, decltype(&OpenVDS::Close)> &handle)
{
OpenVDS::Error error;
std::unique_ptr<OpenVDS::VDS, decltype(&OpenVDS::Close)> handle(generateSimpleInMemory3DVDS(60,60,60), &OpenVDS::Close);
ASSERT_TRUE(handle);
handle.reset(generateSimpleInMemory3DVDS(60,60,60));
fill3DVDSWithNoice(handle.get());
}
struct RequestSharedData
{
RequestSharedData()
: handle(nullptr, &OpenVDS::Close)
{
setupNoiceTestHandle(handle);
layout = OpenVDS::GetLayout(handle.get());
accessManager = OpenVDS::GetAccessManager(handle.get());
auto layout = OpenVDS::GetLayout(handle.get());
auto accessManager = OpenVDS::GetAccessManager(handle.get());
minPos[0] = 10; minPos[1] = 10; minPos[2] = 10;
maxPos[0] = 50; maxPos[1] = 50; maxPos[2] = 50;
voxelCount = (maxPos[0] - minPos[0]) * (maxPos[1] - minPos[1]) * (maxPos[2] - minPos[2]);
bufferFloat.resize(voxelCount);
int64_t requestFloat = accessManager->RequestVolumeSubset(bufferFloat.data(), layout, OpenVDS::Dimensions_012, 0, 0, minPos, maxPos, OpenVDS::VolumeDataChannelDescriptor::Format_R32);
accessManager->WaitForCompletion(requestFloat);
}
~RequestSharedData()
{
}
std::unique_ptr<OpenVDS::VDS, decltype(&OpenVDS::Close)> handle;
OpenVDS::VolumeDataLayout *layout;
OpenVDS::VolumeDataAccessManager *accessManager;
int32_t minPos[OpenVDS::Dimensionality_Max];
int32_t maxPos[OpenVDS::Dimensionality_Max];
int32_t voxelCount;
std::vector<float> bufferFloat;
};
class RequestVolumeSubsetFormat : public ::testing::Test
{
protected:
static void SetupTestSuit()
{
}
static void TearDownTestSuit()
{
}
virtual void SetUp() override
{
shared_data.reset(new RequestSharedData());
}
virtual void TearDown() override
{
shared_data.reset();
}
static std::unique_ptr<RequestSharedData> shared_data;
};
std::unique_ptr<RequestSharedData> RequestVolumeSubsetFormat::shared_data;
TEST_F(RequestVolumeSubsetFormat, test1Byte)
{
auto *shared_data = RequestVolumeSubsetFormat::shared_data.get();
std::vector<uint8_t> buffer;
buffer.resize(shared_data->voxelCount);
int64_t request= shared_data->accessManager->RequestVolumeSubset(buffer.data(), shared_data->layout, OpenVDS::Dimensions_012, 0, 0, shared_data->minPos, shared_data->maxPos, OpenVDS::VolumeDataChannelDescriptor::Format_U8);
shared_data->accessManager->WaitForCompletion(request);
float minValue = shared_data->layout->GetChannelValueRangeMin(0);
float maxValue = shared_data->layout->GetChannelValueRangeMax(0);
float intScale = shared_data->layout->GetChannelIntegerScale(0);
float intOffset = shared_data->layout->GetChannelIntegerOffset(0);
OpenVDS::QuantizingValueConverterWithNoValue<uint8_t, float, false> valueConverter(minValue, maxValue, intScale, intOffset, 0.f, 0.f);
for (size_t i = 0; i < buffer.size(); i++)
{
ASSERT_EQ(buffer[i], valueConverter.ConvertValue(shared_data->bufferFloat[i]));
}
}
TEST_F(RequestVolumeSubsetFormat, test1Bit)
{
auto *shared_data = RequestVolumeSubsetFormat::shared_data.get();
std::vector<uint8_t> buffer;
buffer.resize(shared_data->voxelCount / 8 + 1);
int64_t request= shared_data->accessManager->RequestVolumeSubset(buffer.data(), shared_data->layout, OpenVDS::Dimensions_012, 0, 0, shared_data->minPos, shared_data->maxPos, OpenVDS::VolumeDataChannelDescriptor::Format_1Bit);
shared_data->accessManager->WaitForCompletion(request);
for (size_t i = 0; i < shared_data->voxelCount; i++)
{
uint8_t data = (buffer[i/8] & (1 << (i % 8))) >> (i % 8);
uint8_t value = shared_data->bufferFloat[i] != shared_data->layout->GetChannelNoValue(0);
ASSERT_TRUE(bool(data) == bool(value));
}
}
TEST(ReqeustVolumeSubsetFormat, source1Bit)
{
std::unique_ptr<OpenVDS::VDS, decltype(&OpenVDS::Close)> handle(generateSimpleInMemory3DVDS(60,60,60, OpenVDS::VolumeDataChannelDescriptor::Format_1Bit), OpenVDS::Close);
fill3DVDSWithBitNoice(handle.get());
OpenVDS::VolumeDataLayout *layout = OpenVDS::GetLayout(handle.get());
OpenVDS::VolumeDataAccessManager *accessManager = OpenVDS::GetAccessManager(handle.get());
int32_t minPos[OpenVDS::Dimensionality_Max];
int32_t maxPos[OpenVDS::Dimensionality_Max];
int32_t voxelCount;
minPos[0] = 10; minPos[1] = 10; minPos[2] = 10;
maxPos[0] = 50; maxPos[1] = 50; maxPos[2] = 50;
std::vector<uint8_t> bufferByte;
bufferByte.resize(40 * 40 * 40);
int64_t requestByte = accessManager->RequestVolumeSubset(bufferByte.data(), layout, OpenVDS::Dimensions_012, 0, 0, minPos, maxPos, OpenVDS::VolumeDataChannelDescriptor::Format_U8);
voxelCount = (maxPos[0] - minPos[0]) * (maxPos[1] - minPos[1]) * (maxPos[2] - minPos[2]);
std::vector<float> bufferFloat;
bufferFloat.resize(40 * 40 * 40);
bufferFloat.resize(voxelCount);
int64_t requestFloat = accessManager->RequestVolumeSubset(bufferFloat.data(), layout, OpenVDS::Dimensions_012, 0, 0, minPos, maxPos, OpenVDS::VolumeDataChannelDescriptor::Format_R32);
accessManager->WaitForCompletion(requestByte);
accessManager->WaitForCompletion(requestFloat);
float minValue = layout->GetChannelValueRangeMin(0);
float maxValue = layout->GetChannelValueRangeMax(0);
float intScale = layout->GetChannelIntegerScale(0);
float intOffset = layout->GetChannelIntegerOffset(0);
OpenVDS::QuantizingValueConverterWithNoValue<uint8_t, float, false> valueConverter(minValue, maxValue, intScale, intOffset, 0.f, 0.f);
std::vector<uint8_t> bufferByte;
bufferByte.resize(voxelCount / 8);
int64_t requestByte = accessManager->RequestVolumeSubset(bufferByte.data(), layout, OpenVDS::Dimensions_012, 0, 0, minPos, maxPos, OpenVDS::VolumeDataChannelDescriptor::Format_1Bit);
accessManager->WaitForCompletion(requestByte);
for (size_t i = 0; i < bufferByte.size(); i++)
for (int i = 0; i < voxelCount; i++)
{
ASSERT_EQ(bufferByte[i], valueConverter.ConvertValue(bufferFloat[i]));
float floatVal = bufferFloat[i];
bool floatBool = bool(floatVal);
uint8_t byteBool = bufferByte[i / 8] & (uint8_t(1) << (i%8));
bool boolBool = bool(byteBool);
ASSERT_EQ(floatBool, boolBool);
}
}
......@@ -9,7 +9,9 @@
#include <VDS/SimplexNoiceKernel.h>
static OpenVDS::VDS *generateSimpleInMemory3DVDS(int32_t samplesX = 100, int32_t samplesY = 100, int32_t samplesZ = 100)
#include <random>
static OpenVDS::VDS *generateSimpleInMemory3DVDS(int32_t samplesX = 100, int32_t samplesY = 100, int32_t samplesZ = 100, OpenVDS::VolumeDataChannelDescriptor::Format format = OpenVDS::VolumeDataChannelDescriptor::Format_R32)
{
auto brickSize = OpenVDS::VolumeDataLayoutDescriptor::BrickSize_32;
int negativeMargin = 4;
......@@ -25,7 +27,7 @@ static OpenVDS::VDS *generateSimpleInMemory3DVDS(int32_t samplesX = 100, int32_t
axisDescriptors.emplace_back(100, KNOWNMETADATA_SURVEYCOORDINATE_INLINECROSSLINE_AXISNAME_INLINE, "", 9985.f, 10369.f);
std::vector<OpenVDS::VolumeDataChannelDescriptor> channelDescriptors;
channelDescriptors.emplace_back(OpenVDS::VolumeDataChannelDescriptor::Format_R32, OpenVDS::VolumeDataChannelDescriptor::Components_1, AMPLITUDE_ATTRIBUTE_NAME, "", -0.10919982194900513, 0.1099749207496643);
channelDescriptors.emplace_back(format, OpenVDS::VolumeDataChannelDescriptor::Components_1, AMPLITUDE_ATTRIBUTE_NAME, "", -0.10919982194900513, 0.1099749207496643);
OpenVDS::InMemoryOpenOptions options;
OpenVDS::MetadataContainer metadataContainer;
......@@ -52,7 +54,63 @@ static void fill3DVDSWithNoice(OpenVDS::VDS *vds, int32_t channel = 0, const Ope
int pitch[OpenVDS::Dimensionality_Max];
void *buffer = page->GetWritableBuffer(pitch);
OpenVDS::CalculateNoise3D(buffer, OpenVDS::VolumeDataChannelDescriptor::Format_R32, &outputIndexer, frequency, 0.001f, 0.f, false, 345);
OpenVDS::CalculateNoise3D(buffer, OpenVDS::VolumeDataChannelDescriptor::Format_R32, &outputIndexer, frequency, 0.021f, 0.f, true, 345);
page->Release();
}
pageAccessor->Commit();
pageAccessor->SetMaxPages(0);
accessManager->FlushUploadQueue();
accessManager->DestroyVolumeDataPageAccessor(pageAccessor);
}
static void fill3DVDSWithBitNoice(OpenVDS::VDS *vds, int32_t channel = 0)
{
OpenVDS::VolumeDataLayout *layout = OpenVDS::GetLayout(vds);
ASSERT_TRUE(layout);
OpenVDS::VolumeDataAccessManager *accessManager = OpenVDS::GetAccessManager(vds);
ASSERT_TRUE(accessManager);
OpenVDS::VolumeDataPageAccessor *pageAccessor = accessManager->CreateVolumeDataPageAccessor(layout, OpenVDS::Dimensions_012, channel, 0, 100, OpenVDS::VolumeDataAccessManager::AccessMode_Create);
ASSERT_TRUE(pageAccessor);
int32_t chunkCount = int32_t(pageAccessor->GetChunkCount());
std::mt19937 gen(123);
std::bernoulli_distribution dist(0.8);
for (int i = 0; i < chunkCount; i++)
{
OpenVDS::VolumeDataPage *page = pageAccessor->CreatePage(i);
OpenVDS::VolumeIndexer3D outputIndexer(page, 0, 0, OpenVDS::Dimensions_012, layout);
int pitch[OpenVDS::Dimensionality_Max];
uint8_t *buffer = static_cast<uint8_t *>(page->GetWritableBuffer(pitch));
int32_t min[OpenVDS::Dimensionality_Max];
int32_t max[OpenVDS::Dimensionality_Max];
page->GetMinMax(min, max);
int32_t size[OpenVDS::Dimensionality_Max];
for (int i = 0; i < OpenVDS::Dimensionality_Max; i++)
size[i] = max[i] - min[i];
for (int z = 0; z < size[2]; z++)
{
for (int y = 0; y < size[1]; y++)
{
for(int x = 0; x < size[0]; x++)
{
int32_t byteIndex = z * pitch[2] + y * pitch[1] + (x / 8);
int32_t bitIndex = z * size[2] * size[1] * size[0] + y * size[1] * size[0] + x;
uint8_t byte = buffer[byteIndex];
bool value = dist(gen);
if (value)
byte |= uint8_t(1) << (bitIndex % 8);
buffer[byteIndex] = byte;
}
}
}
page->Release();
}
pageAccessor->Commit();
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment