mutability of VDS
Dear developers,
is there a way to update a data chunk of already existing VDS? Is this even possible when considering the low-lvl design of the open VDS format?
use-case: data pipeline workers process pre-defined subsets of the data, and we want to re-upload the results to a single VDS entity. Preferably, we want to avoid, a single worker/step that collects all the chunks and then uploads it from within a single context manager, while creating output VDS on s3/DMS.
Thanks, Filip
PS. I believe the below snippet should not result in seg-fault, even if VDS is immutable after the creation. But it is, tested on 2.1.9 linux version:
import openvds
import numpy as np
def create_vds(
opts,
shape=(100, 100, 100),
format=openvds.VolumeDataChannelDescriptor.Format.Format_R32,
brickSize=openvds.VolumeDataLayoutDescriptor.BrickSize.BrickSize_1024,
):
layout_descriptor = openvds.VolumeDataLayoutDescriptor(
brickSize,
0,
0,
4,
openvds.VolumeDataLayoutDescriptor.LODLevels.LODLevels_None,
openvds.VolumeDataLayoutDescriptor.Options.Options_None,
)
axis_descriptors = [
openvds.VolumeDataAxisDescriptor(shape[2], "Z", "m", 0.0, 2000.0),
openvds.VolumeDataAxisDescriptor(shape[1], "Y", "m", 0.0, 2000.0),
openvds.VolumeDataAxisDescriptor(shape[0], "X", "m", 0.0, 2000.0),
]
channel_descriptors = [
openvds.VolumeDataChannelDescriptor(
format,
openvds.VolumeDataChannelDescriptor.Components.Components_1,
"Value",
"",
0.0,
((shape[2] * 3) * (shape[1] * 2) * shape[0]) - 1.0,
)
]
metadata_container = openvds.MetadataContainer()
vds = openvds.create(
opts[0],
opts[1],
layout_descriptor,
axis_descriptors,
channel_descriptors,
metadata_container,
)
manager = openvds.getAccessManager(vds)
accessor = manager.createVolumeDataPageAccessor(
openvds.DimensionsND.Dimensions_012,
0,
0,
8,
openvds.IVolumeDataAccessManager.AccessMode.AccessMode_Create,
)
for c in range(accessor.getChunkCount()):
page = accessor.createPage(c)
buf = np.array(page.getWritableBuffer(), copy=False)
(min, max) = page.getMinMax()
buf[:, :, :] = np.array([1.0] * buf.size, dtype=float).reshape(buf.shape)
page.release()
accessor.commit()
openvds.close(vds)
create_vds(("error.vds", ""))
with openvds.open("./error.vds", "") as vds:
manager = openvds.getAccessManager(vds)
accessor = manager.createVolumeDataPageAccessor(
openvds.DimensionsND.Dimensions_012,
0,
0,
8,
openvds.IVolumeDataAccessManager.AccessMode.AccessMode_ReadWrite,
)
for c in range(accessor.getChunkCount()):
page = accessor.createPage(c)
buf = np.array(page.getWritableBuffer(), copy=False)
(min, max) = page.getMinMax()
buf[:, :, :] = np.array([1.0] * buf.size, dtype=float).reshape(buf.shape)
page.release()
accessor.commit()