From c58b20b328cde26f73dcb1c53f79dd0bd7c8330e Mon Sep 17 00:00:00 2001 From: Julien Lacoste Date: Tue, 24 Aug 2021 16:22:07 +0200 Subject: [PATCH 01/25] Java bindings : Add methods on VolumeDataAccessManager to get info on Upload/Download error --- java/CMakeLists.txt | 1 + java/cpp/src/VolumeDataAccessManager.cpp | 61 +++++++++++++++++++ .../src/org/opengroup/openvds/ErrorInfo.java | 44 +++++++++++++ .../openvds/VolumeDataAccessManager.java | 22 +++++++ 4 files changed, 128 insertions(+) create mode 100644 java/java/src/org/opengroup/openvds/ErrorInfo.java diff --git a/java/CMakeLists.txt b/java/CMakeLists.txt index c0f69202..81675589 100644 --- a/java/CMakeLists.txt +++ b/java/CMakeLists.txt @@ -20,6 +20,7 @@ set(JAVA_SOURCE_FILES java/src/org/opengroup/openvds/CompressionMethod.java java/src/org/opengroup/openvds/DimensionsND.java java/src/org/opengroup/openvds/experimental/VariousJavaTests.java + java/src/org/opengroup/openvds/ErrorInfo.java java/src/org/opengroup/openvds/GoogleOpenOptions.java java/src/org/opengroup/openvds/IndexRegionFloat.java java/src/org/opengroup/openvds/IndexRegionLong.java diff --git a/java/cpp/src/VolumeDataAccessManager.cpp b/java/cpp/src/VolumeDataAccessManager.cpp index 0968092d..c6f723af 100644 --- a/java/cpp/src/VolumeDataAccessManager.cpp +++ b/java/cpp/src/VolumeDataAccessManager.cpp @@ -583,6 +583,67 @@ JNIEXPORT jint JNICALL Java_org_opengroup_openvds_VolumeDataAccessManager_cpGetC return 0; } +/* + * Class: org_opengroup_openvds_VolumeDataAccessManager + * Method: cpGetCurrentUploadErrorInfo + * Signature: (JLorg/opengroup/openvds/ErrorInfo;)V + */ +JNIEXPORT void JNICALL Java_org_opengroup_openvds_VolumeDataAccessManager_cpGetCurrentUploadErrorInfo + (JNIEnv * env, jclass, jlong handle, jobject errorObj) +{ + try { + const char *pObjectID = nullptr; + const char *pErrorString = nullptr; + int32_t errorCode = 0; + + GetManager(handle)->GetCurrentUploadError(&pObjectID, &errorCode, &pErrorString); + + // Get the class of the given error object + jclass clazz = env->GetObjectClass(errorObj); + + // Get Field references + jfieldID paramCode = env->GetFieldID(clazz, "errorCode", "I"); + jfieldID paramMessage = env->GetFieldID(clazz, "errorMessage", "Ljava/lang/String;"); + jfieldID paramID = env->GetFieldID(clazz, "objectID", "Ljava/lang/String;"); + + // Set fields for object + jstring strMsg = NewJString(env, pErrorString); + jstring strID = NewJString(env, pObjectID); + env->SetIntField(errorObj, paramCode, errorCode); + env->SetObjectField(errorObj, paramMessage, strMsg); + env->SetObjectField(errorObj, paramID, strID); + } + CATCH_EXCEPTIONS_FOR_JAVA; +} + +/* + * Class: org_opengroup_openvds_VolumeDataAccessManager + * Method: cpGetCurrentDownloadErrorInfo + * Signature: (JLorg/opengroup/openvds/ErrorInfo;)V + */ +JNIEXPORT void JNICALL Java_org_opengroup_openvds_VolumeDataAccessManager_cpGetCurrentDownloadErrorInfo + (JNIEnv * env, jclass, jlong handle, jobject errorObj) +{ + try { + const char *pErrorString = nullptr; + int32_t errorCode = 0; + GetManager(handle)->GetCurrentDownloadError(&errorCode, &pErrorString); + + // Get the class of the given error object + jclass clazz = env->GetObjectClass(errorObj); + + // Get Field references + jfieldID paramCode = env->GetFieldID(clazz, "errorCode", "I"); + jfieldID paramMessage = env->GetFieldID(clazz, "errorMessage", "Ljava/lang/String;"); + + // Set fields for object + env->SetIntField(errorObj, paramCode, errorCode); + jstring strMsg = NewJString(env, pErrorString); + env->SetObjectField(errorObj, paramMessage, strMsg); + } + CATCH_EXCEPTIONS_FOR_JAVA; +} + #ifdef __cplusplus } #endif diff --git a/java/java/src/org/opengroup/openvds/ErrorInfo.java b/java/java/src/org/opengroup/openvds/ErrorInfo.java new file mode 100644 index 00000000..4dd9f017 --- /dev/null +++ b/java/java/src/org/opengroup/openvds/ErrorInfo.java @@ -0,0 +1,44 @@ +package org.opengroup.openvds; + +/** + * Simple class holding information on Upload/Download error + */ +public class ErrorInfo { + + private int errorCode; + private String errorMessage; + private String objectID; + + public ErrorInfo() { + + } + public ErrorInfo(int code, String msg, String id) { + errorCode = code; + errorMessage = msg; + objectID = id; + } + + public int getErrorCode() { + return errorCode; + } + + public void setErrorCode(int errorCode) { + this.errorCode = errorCode; + } + + public String getErrorMessage() { + return errorMessage; + } + + public void setErrorMessage(String errorMessage) { + this.errorMessage = errorMessage; + } + + public String getObjectID() { + return objectID; + } + + public void setObjectID(String objectID) { + this.objectID = objectID; + } +} diff --git a/java/java/src/org/opengroup/openvds/VolumeDataAccessManager.java b/java/java/src/org/opengroup/openvds/VolumeDataAccessManager.java index 2af400d2..85622597 100644 --- a/java/java/src/org/opengroup/openvds/VolumeDataAccessManager.java +++ b/java/java/src/org/opengroup/openvds/VolumeDataAccessManager.java @@ -147,6 +147,10 @@ public class VolumeDataAccessManager extends JniPointerWithoutDeletion { private static native int cpGetCurrentUploadErrorCode(long handle); + private static native void cpGetCurrentUploadErrorInfo(long handle, ErrorInfo error); + + private static native void cpGetCurrentDownloadErrorInfo(long handle, ErrorInfo error); + public VolumeDataAccessManager(long handle) { super(handle); } @@ -771,4 +775,22 @@ public class VolumeDataAccessManager extends JniPointerWithoutDeletion { public int getCurrentUploadErrorCode() { return cpGetCurrentUploadErrorCode(_handle); } + + /** + * @return current upload error information (code and message in a simple class) + */ + public ErrorInfo getCurrentUploadErrorInfo() { + ErrorInfo errorInfo = new ErrorInfo(); + cpGetCurrentUploadErrorInfo(_handle, errorInfo); + return errorInfo; + } + + /** + * @return current upload error information (code and message in a simple class) + */ + public ErrorInfo getCurrentDownloadErrorInfo() { + ErrorInfo errorInfo = new ErrorInfo(); + cpGetCurrentUploadErrorInfo(_handle, errorInfo); + return errorInfo; + } } -- GitLab From 3d99e0ea803b9c61ed1168d432bb60ac2963adf6 Mon Sep 17 00:00:00 2001 From: Julien Lacoste Date: Tue, 24 Aug 2021 16:28:16 +0200 Subject: [PATCH 02/25] Java bindings : Add methods on VolumeDataAccessManager to get info on Upload/Download error + fix wrong native call --- .../java/src/org/opengroup/openvds/VolumeDataAccessManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/java/src/org/opengroup/openvds/VolumeDataAccessManager.java b/java/java/src/org/opengroup/openvds/VolumeDataAccessManager.java index 85622597..86d24b8c 100644 --- a/java/java/src/org/opengroup/openvds/VolumeDataAccessManager.java +++ b/java/java/src/org/opengroup/openvds/VolumeDataAccessManager.java @@ -790,7 +790,7 @@ public class VolumeDataAccessManager extends JniPointerWithoutDeletion { */ public ErrorInfo getCurrentDownloadErrorInfo() { ErrorInfo errorInfo = new ErrorInfo(); - cpGetCurrentUploadErrorInfo(_handle, errorInfo); + cpGetCurrentDownloadErrorInfo(_handle, errorInfo); return errorInfo; } } -- GitLab From 6b98d1890e1e5ace517d7cee0b2c198b2ba61fa0 Mon Sep 17 00:00:00 2001 From: Julien Lacoste Date: Thu, 2 Sep 2021 12:17:34 +0200 Subject: [PATCH 03/25] bkp --- java/java/demo/CreateVDS.java | 2 +- java/java/demo/CreateVDSCloud.java | 578 ++++++++++++++++++ .../src/org/opengroup/openvds/ErrorInfo.java | 5 +- .../openvds/PageAccessorByteTest.java | 335 ++++++++++ 4 files changed, 916 insertions(+), 4 deletions(-) create mode 100644 java/java/demo/CreateVDSCloud.java create mode 100644 java/java/test/org/opengroup/openvds/PageAccessorByteTest.java diff --git a/java/java/demo/CreateVDS.java b/java/java/demo/CreateVDS.java index 0931bd12..964a84fa 100644 --- a/java/java/demo/CreateVDS.java +++ b/java/java/demo/CreateVDS.java @@ -157,7 +157,7 @@ public class CreateVDS { true, // is renderable false, // allow lossy compression false, // use zip for lossless compresion - true, // use no value + false, // use no value -999.25f, // no value scaleOffset[0], // integer scale scaleOffset[1]); // integer offset diff --git a/java/java/demo/CreateVDSCloud.java b/java/java/demo/CreateVDSCloud.java new file mode 100644 index 00000000..d31b31e4 --- /dev/null +++ b/java/java/demo/CreateVDSCloud.java @@ -0,0 +1,578 @@ +/* + * Copyright 2021 The Open Group + * Copyright 2021 INT, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.opengroup.openvds.*; + +import java.io.File; +import java.io.IOException; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; + +public class CreateVDSCloud { + + // usage : CreateVDSWithLOD n /path/to/file + // where n is the LOD level wanted + public static void main(String[] args) { + try { + if (!checkParams(args)) { + System.out.println("Bad params, usage : n /path/to/file dimX dimY dimZ"); + System.out.println("n is the LOD level wanted (0 <= lod <= 12), path must be a valid non existing file path"); + System.out.println("dims are cube dimension (must be > 0)"); + } + else { + Instant start = Instant.now(); + process(Integer.parseInt(args[0]), args[1], Integer.parseInt(args[2]), Integer.parseInt(args[3]), Integer.parseInt(args[4])); + Instant end = Instant.now(); + + Duration elapsed = Duration.between(start, end); + long hrs = elapsed.toHours(); + long min = elapsed.toMinutes() - (hrs * 60); + long s = (elapsed.toMillis() - (elapsed.toMinutes() * 60 * 1000)) / 1000; + System.out.println("Write VDS TIME : " + hrs + " hrs " + min + " min " + s + "s (" + elapsed.toMillis() + " ms)"); + + testRead(args[1]); + } + } + catch (Throwable t) { + System.out.println(); + t.printStackTrace(); + } + } + + private static boolean checkParams(String[] args) { + if (args == null || args.length != 5) { + return false; + } + try { + Integer lod = Integer.parseInt(args[0]); + if (lod < 0 || lod > 12) { + System.err.println("Invalid LOD value (must be between 0 and 12) !"); + return false; + } + } + catch (NumberFormatException nfe) { + return false; + } + String path = args[1]; + File file = new File(path); + try { + file.getCanonicalPath(); + } + catch (IOException e) { + System.err.println("VDS File path is invalid !"); + return false; + } + if (!path.endsWith(".vds")) { + System.err.println("VDS File path must have vds extension !"); + return false; + } + if (file.exists()) { + System.err.println("VDS File already exists !"); + return false; + } + try { + Integer dimX = Integer.parseInt(args[2]); + if (dimX < 0) { + System.err.println("Invalid x dimension !"); + return false; + } + Integer dimY = Integer.parseInt(args[3]); + if (dimX < 0) { + System.err.println("Invalid y dimension !"); + return false; + } + Integer dimZ = Integer.parseInt(args[4]); + if (dimX < 0) { + System.err.println("Invalid Z dimension !"); + return false; + } + } + catch (NumberFormatException nfe) { + return false; + } + return true; + } + + static void process(int lodParam, String name, int dimX, int dimY, int dimZ) throws Exception { + + int samplesX = dimZ; // time + int samplesY = dimY; // XL + int samplesZ = dimX; // IL + VolumeDataChannelDescriptor.Format format = VolumeDataChannelDescriptor.Format.FORMAT_R32; + + double sizeX = samplesX; + double sizeY = samplesY; + double sizeZ = samplesZ; + + double distMax = distance3D(0, 0, 0, sizeX, sizeY, sizeZ); + double cycles = Math.PI * 2 * 20; + double midX = samplesX / 2f; + double midY = samplesY / 2f; + double midZ = samplesZ / 2f; + + VolumeDataLayoutDescriptor.BrickSize brickSize = VolumeDataLayoutDescriptor.BrickSize.BRICK_SIZE_64; + int negativeMargin = 4; + int positiveMargin = 4; + int brickSize2DMultiplier = 4; + + VolumeDataLayoutDescriptor.LODLevels lodLevels = VolumeDataLayoutDescriptor.LODLevels.values()[lodParam]; + + //VolumeDataLayoutDescriptor.Options layoutOptions = VolumeDataLayoutDescriptor.Options.NONE; + VolumeDataLayoutDescriptor layoutDescriptor = new VolumeDataLayoutDescriptor(brickSize, negativeMargin, positiveMargin, + brickSize2DMultiplier, lodLevels, false,false,0); + + List axisDescriptors = new ArrayList<>(); + axisDescriptors.add(new VolumeDataAxisDescriptor(samplesX, "Sample", "s", 0.0f,6.0f)); + axisDescriptors.add(new VolumeDataAxisDescriptor(samplesY, "Crossline", "",0f, 0f + samplesY - 1f)); + axisDescriptors.add(new VolumeDataAxisDescriptor(samplesZ, "Inline", "", 0f, 0f + samplesZ - 1f)); + + List channelDescriptors = new ArrayList<>(); + + float rangeMin = -1f; + float rangeMax = 1f; + float[] scaleOffset = getScaleOffsetForFormat(rangeMin, rangeMax, true, format); + + VolumeDataChannelDescriptor channelDescriptor = new VolumeDataChannelDescriptor(format, VolumeDataChannelDescriptor.Components.COMPONENTS_1, + "Amplitude", + "", + rangeMin, // range min + rangeMax, // range max + VolumeDataMapping.DIRECT, // mapping + 1, // mapped value count + false, // is discrete + true, // is renderable + false, // allow lossy compression + false, // use zip for lossless compresion + false, // use no value + -999.25f, // no value + scaleOffset[0], // integer scale + scaleOffset[1]); // integer offset + channelDescriptors.add(channelDescriptor); + + // options AWS + String pBucket = "miniovaap"; // the bucket of the VDS + String pKey = name; // the key prefix of the VDS ? + String pRegion = "eu-west-3"; // the region of the bucket of the VDS + String pEndpointOverride = "https://webvr.int.com/"; // This parameter allows to override the endpoint url + AWSOpenOptions optionsAWS = new AWSOpenOptions(pBucket, pKey, pRegion, pEndpointOverride); + optionsAWS.accessKeyId = "minioadmin"; // ? + optionsAWS.secretKey = "minioadmin"; + optionsAWS.sessionToken = ""; // ? + + // file options + //VDSFileOpenOptions options = new VDSFileOpenOptions(vdsFilePath); + + MetadataContainer metadataContainer = new MetadataContainer(); + StringBuilder builder = new StringBuilder(); + builder.append("CreateVDS Demo "); + builder.append("dimX: "); + builder.append(dimX); + builder.append(" dimY: "); + builder.append(dimY); + builder.append(" dimZ: "); + builder.append(dimZ); + + metadataContainer.setMetadataString("General Info", "Cube layout", builder.toString()); + metadataContainer.setMetadataInt("General Info", "LOD", lodLevels.ordinal()); + metadataContainer.setMetadataString("General Info", "Brick Size", "BRICK_SIZE_64"); + + VdsHandle vds = OpenVDS.create(optionsAWS, layoutDescriptor, + axisDescriptors.toArray(new VolumeDataAxisDescriptor[0]), + channelDescriptors.toArray(new VolumeDataChannelDescriptor[0]), metadataContainer); + + // for compression use +// float compressionTolerance = 0f; // +// VdsHandle vds = OpenVDS.create(options, layoutDescriptor, +// axisDescriptors.toArray(new VolumeDataAxisDescriptor[0]), +// channelDescriptors.toArray(new VolumeDataChannelDescriptor[0]), metadataContainer, CompressionMethod.ZIP, compressionTolerance); + + VolumeDataLayout layout = vds.getLayout(); + VolumeDataAccessManager accessManager = vds.getAccessManager(); + + int channel = 0; + + int lodCount = lodLevels.ordinal(); + + int[] localIndex = new int[3]; + int[] voxelPos = new int[3]; + + int[] pitch = new int[VolumeDataLayout.Dimensionality_Max]; + int[] chunkMin = new int[VolumeDataLayout.Dimensionality_Max]; + int[] chunkMax = new int[VolumeDataLayout.Dimensionality_Max]; + + for (int lod = 0 ; lod <= lodCount ; ++lod) { + VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( + DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND + lod, // lod + channel, // channel + 50, // max pages + VolumeDataPageAccessor.AccessMode.Create.getCode()); // access mode + + + //ASSERT_TRUE(pageAccessor); + int paLOD = pageAccessor.getLOD(); + int paCI = pageAccessor.getChannelIndex(); + int[] paNumSamples = pageAccessor.getNumSamples(); + + int chunkCount = (int) pageAccessor.getChunkCount(); + System.out.println("LOD : " + lod + " Chunk count : " + chunkCount); + + for (int i = 0; i < chunkCount; i++) { + + long time_1 = System.currentTimeMillis(); + + VolumeDataPage page = pageAccessor.createPage(i); + System.out.print("LOD : " + lod + " / Page " + (i + 1) + " / " + chunkCount); + + VolumeIndexer3D outputIndexer = new VolumeIndexer3D(page, 0, lod, DimensionsND.DIMENSIONS_012.ordinal(), layout); + + int[] numSamplesChunk = new int[3]; + int[] numSamplesDB = new int[3]; + + for (int j = 0; j < 3; j++) { + numSamplesChunk[j] = outputIndexer.getLocalChunkNumSamples(j); + numSamplesDB[j] = outputIndexer.getDataBlockNumSamples(j); + } + System.out.print("\tDimensions Chunk : [" + numSamplesChunk[0] + ", " + numSamplesChunk[1] + ", " + numSamplesChunk[2] + "]"); + System.out.print("\tDimensions DataBlock : [" + numSamplesDB[0] + ", " + numSamplesDB[1] + ", " + numSamplesDB[2] + "]"); + + page.getMinMax(chunkMin, chunkMax); + + System.out.print("\tCoords page : " + chunkMin[0] + "/" + chunkMax[0] + " " + chunkMin[1] + "/" + chunkMax[1] + " " + chunkMin[2] + "/" + chunkMax[2]); + int chunkSizeY = chunkMax[2] - chunkMin[2]; + int chunkSizeX = chunkMax[1] - chunkMin[1]; + int chunkSizeZ = chunkMax[0] - chunkMin[0]; + + // get pitch and allocate buffer size + page.getPitch(pitch); + float[] output = new float[page.getAllocatedBufferSize()]; + + // or read buffer of page + //float[] output = page.readFloatBuffer(pitch); + + System.out.print( " Page Buffer Size : " + output.length + " ");/// [" + stats.getMin() + "," + stats.getMax() + "]"); + // dimension of buffer + System.out.println(" Pitch : [" + pitch[0] + ", " + pitch[1] + ", " + pitch[2] + "]"); + + int[] numSamples = numSamplesChunk; + + for (int iDim2 = 0; iDim2 < numSamples[2]; iDim2++) { + localIndex[2] = iDim2; + for (int iDim1 = 0; iDim1 < numSamples[1]; iDim1++) { + localIndex[1] = iDim1; + for (int iDim0 = 0; iDim0 < numSamples[0]; iDim0++) { + localIndex[0] = iDim0; + + int[] voxelIndex = outputIndexer.localIndexToVoxelIndex(localIndex); + for (int vp = 0; vp < 3; ++vp) { + voxelPos[vp] = voxelIndex[vp]; + } + + float value = 0f; + double dist = 0; + if (voxelPos[0] >= midX) { + dist = distance2D(midY, midZ, voxelPos[1], voxelPos[2]); + } else { + dist = distance3D(midX, midY, midZ, voxelPos[0], voxelPos[1], voxelPos[2]); + } + value = (float) Math.sin((dist * cycles) / distMax); + + int iPos = outputIndexer.localIndexToDataIndex(localIndex); + output[iPos] = value; + } + } + } + + // write buffer then release page + page.writeFloatBuffer(output, pitch); + page.pageRelease(); + } + + int errorCount = accessManager.uploadErrorCount(); + if (errorCount > 0) { + ErrorInfo upErrInfo = accessManager.getCurrentUploadErrorInfo(); + System.out.println("Upload error code : " + upErrInfo.getErrorCode()); + if (upErrInfo.getErrorMessage() != null) { + System.out.println("Upload error message : " + upErrInfo.getErrorMessage()); + } + else { + System.out.println("No error message"); + } + if (upErrInfo.getObjectID() != null) { + System.out.println("Upload error object ID : " + upErrInfo.getObjectID()); + } + else { + System.out.println("No Object ID"); + } + } + + pageAccessor.commit(); + pageAccessor.setMaxPages(0); + accessManager.flushUploadQueue(); + accessManager.destroyVolumeDataPageAccessor(pageAccessor); + } + + + + // Adds slice data : additional layouts +// System.out.println("\nCreate 01 Layout"); +// create2DLayout(distMax, cycles, midX, midY, midZ, layout, accessManager, channel, lodCount, DimensionsND.DIMENSIONS_01); +// +// System.out.println("\nCreate 02 Layout"); +// create2DLayout(distMax, cycles, midX, midY, midZ, layout, accessManager, channel, lodCount, DimensionsND.DIMENSIONS_02); +// +// System.out.println("\nCreate 12 Layout"); +// create2DLayout(distMax, cycles, midX, midY, midZ, layout, accessManager, channel, lodCount, DimensionsND.DIMENSIONS_12); + + + vds.close(); + System.out.println("VDS closed"); + } + + + private static void create2DLayout(double distMax, double cycles, double midX, double midY, double midZ, VolumeDataLayout layout, VolumeDataAccessManager accessManager, + int channel, int lodCount, DimensionsND dimLYT) { + + int[] localIndex = new int[3]; + + int[] pitch = new int[VolumeDataLayout.Dimensionality_Max]; + int[] chunkMin = new int[VolumeDataLayout.Dimensionality_Max]; + int[] chunkMax = new int[VolumeDataLayout.Dimensionality_Max]; + for (int lod = 0 ; lod <= lodCount ; ++lod) { + VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( + dimLYT.ordinal(), // dimension ND + lod, // lod + channel, // channel + 200, // max pages + VolumeDataPageAccessor.AccessMode.Create.getCode()); // access mode + + int chunkCount = (int) pageAccessor.getChunkCount(); + System.out.println("2D Layout LOD : " + lod + " Chunk count : " + chunkCount); + + for (int i = 0; i < chunkCount; i++) { + VolumeDataPage page = pageAccessor.createPage(i); + System.out.print("2D Layout LOD : " + lod + " / Page " + (i + 1) + " / " + chunkCount); + + VolumeIndexer2D outputIndexer = new VolumeIndexer2D(page, 0, lod, dimLYT.ordinal(), layout); + float valueRangeScale = outputIndexer.getValueRangeMax() - outputIndexer.getValueRangeMin(); + + int[] numSamplesChunk = new int[3]; + int[] numSamplesDB = new int[3]; + + for (int j = 0; j < 3 ; j++) { + numSamplesChunk[j] = outputIndexer.getLocalChunkNumSamples(j); + numSamplesDB[j] = outputIndexer.getDataBlockNumSamples(j); + } + System.out.print("\t2D Layout Dimensions Chunk : [" + numSamplesChunk[0] + ", " + numSamplesChunk[1] + ", " + numSamplesChunk[2] + "]"); + System.out.print("\t2D Layout Dimensions DataBlock : [" + numSamplesDB[0] + ", " + numSamplesDB[1] + ", " + numSamplesDB[2] + "]"); + + page.getMinMax(chunkMin, chunkMax); + + // determine fixed position index and value + int fixedCoordIndex = -1; + int fixedCoordValue = -1; + for (int pos = 0; pos < 3 && fixedCoordIndex == -1 ; ++pos) { + if (Math.abs(chunkMax[pos] - chunkMin[pos]) == 1) { + fixedCoordIndex = pos; + fixedCoordValue = chunkMin[pos]; + } + } + + System.out.print("\tCoords page : " + chunkMin[0] + "/" + chunkMax[0] + " " + chunkMin[1] + "/" + chunkMax[1] + "/" + chunkMin[2] + "/" + chunkMax[2]); + int chunkSizeY = chunkMax[2] - chunkMin[2]; + int chunkSizeX = chunkMax[1] - chunkMin[1]; + int chunkSizeZ = chunkMax[0] - chunkMin[0]; + + // get pitch and allocate buffer size + page.getPitch(pitch); + float[] output = new float[page.getAllocatedBufferSize()]; + + System.out.print( " Page Buffer Size : " + output.length);// + " / [" + stats.getMin() + "," + stats.getMax() + "]"); + // dimension of buffer + System.out.println(" Pitch : [" + pitch[0] + ", " + pitch[1] + "," + pitch[2] + "]"); + + long time_2 = System.currentTimeMillis(); + + int[] numSamples = numSamplesChunk; + + for (int iDim2 = 0; iDim2 < numSamples[2]; iDim2++) { + localIndex[2] = iDim2; + for (int iDim1 = 0; iDim1 < numSamples[1]; iDim1++) { + localIndex[1] = iDim1; + for (int iDim0 = 0; iDim0 < numSamples[0]; iDim0++) { + localIndex[0] = iDim0; + int[] local2DIndex = convertTo2DLocalIndex(localIndex, fixedCoordIndex); + int[] voxelIndex = outputIndexer.localChunkIndexToLocalIndex(local2DIndex); + int[] voxelPos = convertTo3DCoord(voxelIndex, fixedCoordIndex, fixedCoordValue); + + float value = 0f; + double dist = 0; + if (voxelPos[0] >= midX) { + dist = distance2D(midY, midZ, voxelPos[1], voxelPos[2]); + } else { + dist = distance3D(midX, midY, midZ, voxelPos[0], voxelPos[1], voxelPos[2]); + } + value = (float) Math.sin((dist * cycles) / distMax); + + int iPos = outputIndexer.localIndexToDataIndex(local2DIndex); + output[iPos] = value; + } + } + } + + // write buffer then release page + page.writeFloatBuffer(output, pitch); + page.pageRelease(); + } + + pageAccessor.commit(); + accessManager.flushUploadQueue(); + accessManager.destroyVolumeDataPageAccessor(pageAccessor); + } + } + + static void testRead(String vdsName) throws IOException { + // options AWS + String pBucket = "miniovaap"; // the bucket of the VDS + String pKey = vdsName; // the key prefix of the VDS ? + String pRegion = "eu-west-3"; // the region of the bucket of the VDS + String pEndpointOverride = "https://webvr.int.com/"; // This parameter allows to override the endpoint url + AWSOpenOptions optionsAWS = new AWSOpenOptions(pBucket, pKey, pRegion, pEndpointOverride); + optionsAWS.accessKeyId = "minioadmin"; // ? + optionsAWS.secretKey = "minioadmin"; + optionsAWS.sessionToken = ""; // ? + + OpenVDS vdsHandle = OpenVDS.open(optionsAWS); + VolumeDataAccessManager accessManager = vdsHandle.getAccessManager(); + + int channel = 0; + VolumeDataLayout layout = vdsHandle.getLayout(); + VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( + DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND + 0, // lod + channel, // channel + 20, // max pages + VolumeDataPageAccessor.AccessMode.ReadOnly.getCode()); // access mode + + + + // compares block data + int[] pitchInput = new int[VolumeDataLayout.Dimensionality_Max]; + + long chunkCount = pageAccessor.getChunkCount(); + for (long chunk = 0 ; chunk < chunkCount ; ++chunk) { + VolumeDataPage inputPage = pageAccessor.readPage(chunk); + VolumeDataPage page = pageAccessor.readPage(chunk); + float[] dataIn = inputPage.readFloatBuffer(pitchInput); + + ErrorInfo dwnErrorInfo = accessManager.getCurrentDownloadErrorInfo(); + if (dwnErrorInfo.getErrorCode() != 0) { + System.out.println("Download error code : " + dwnErrorInfo.getErrorCode()); + if (dwnErrorInfo.getErrorMessage() != null) { + System.out.println("Download error message : " + dwnErrorInfo.getErrorMessage()); + } else { + System.out.println("No error message"); + } + } + } + } + + + /** + * Convert 3D index to 2D index + */ + private static int[] convertTo2DLocalIndex(int[] localIndex, int fixedCoordIndex) { + int[] local2DIndex = new int[2]; + if (fixedCoordIndex == 0) { + local2DIndex[0] = localIndex[1]; + local2DIndex[1] = localIndex[2]; + } + else if (fixedCoordIndex == 1) { + local2DIndex[0] = localIndex[0]; + local2DIndex[1] = localIndex[2]; + } + else if (fixedCoordIndex == 2) { + local2DIndex[0] = localIndex[0]; + local2DIndex[1] = localIndex[1]; + } + return local2DIndex; + } + + /** + * Convert 2D index to 3D index + * @param voxelIndex + * @param fixedCoordIndex + * @param fixedCoordValue + */ + private static int[] convertTo3DCoord(int[] voxelIndex, int fixedCoordIndex, int fixedCoordValue) { + int[] voxelPos = new int[3]; + voxelPos[fixedCoordIndex] = fixedCoordValue; + if (fixedCoordIndex == 0) { + voxelPos[1] = voxelIndex[0]; + voxelPos[2] = voxelIndex[1]; + } + else if (fixedCoordIndex == 1) { + voxelPos[0] = voxelIndex[0]; + voxelPos[2] = voxelIndex[1]; + } + else if (fixedCoordIndex == 2) { + voxelPos[0] = voxelIndex[0]; + voxelPos[1] = voxelIndex[1]; + } + return voxelPos; + } + + private static double distance2D(double x1, double y1, double x2, double y2) { + double diffX = x2 - x1; + double diffY = y2 - y1; + return Math.sqrt((diffX * diffX) + (diffY * diffY)); + } + + private static double distance3D(double x1, double y1, double z1, double x2, double y2, double z2) { + double diffX = x2 - x1; + double diffY = y2 - y1; + double diffZ = z2 - z1; + return Math.sqrt((diffX * diffX) + (diffY * diffY) + (diffZ * diffZ)); + } + + static float[] getScaleOffsetForFormat(float min, float max, boolean useNoValue, VolumeDataChannelDescriptor.Format format) { + float res[] = new float[] {1f, 0f}; + float noValueCmp = useNoValue ? 1f : 0f; + switch (format) { + case FORMAT_U8: + res[0] = 1.f / (255.f - noValueCmp) * (max - min); + res[1] = min; + break; + case FORMAT_U16: + res[0] = 1.f / (65535.f - noValueCmp) * (max - min); + res[1] = min; + break; + case FORMAT_R32: + case FORMAT_U32: + case FORMAT_R64: + case FORMAT_U64: + case FORMAT_1BIT: + case FORMAT_ANY: + res[0] = 1.0f; + res[1] = 0.0f; + } + return res; + } + +} diff --git a/java/java/src/org/opengroup/openvds/ErrorInfo.java b/java/java/src/org/opengroup/openvds/ErrorInfo.java index 4dd9f017..fc9b81b7 100644 --- a/java/java/src/org/opengroup/openvds/ErrorInfo.java +++ b/java/java/src/org/opengroup/openvds/ErrorInfo.java @@ -5,13 +5,12 @@ package org.opengroup.openvds; */ public class ErrorInfo { - private int errorCode; + private int errorCode = -1; private String errorMessage; private String objectID; - public ErrorInfo() { + public ErrorInfo() { } - } public ErrorInfo(int code, String msg, String id) { errorCode = code; errorMessage = msg; diff --git a/java/java/test/org/opengroup/openvds/PageAccessorByteTest.java b/java/java/test/org/opengroup/openvds/PageAccessorByteTest.java new file mode 100644 index 00000000..4e84d0a0 --- /dev/null +++ b/java/java/test/org/opengroup/openvds/PageAccessorByteTest.java @@ -0,0 +1,335 @@ +/* + * Copyright 2021 The Open Group + * Copyright 2021 INT, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.openvds; + +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.io.File; + +import static org.testng.Assert.fail; + +public class PageAccessorByteTest { + + private static String TEMP_FILE_NAME_VOL_INDEX = "volIndexer"; + private static String TEMP_FILE_NAME_COPY = "vdsCopy"; + + public String url; + public VolumeDataLayoutDescriptor ld; + public VolumeDataAxisDescriptor[] vda; + public VolumeDataChannelDescriptor[] vdc; + public MetadataReadAccess md; + public MemoryVdsGenerator vds; + public MetadataContainer metadataContainer; + + private String tempVolIndexerFileName; + private String tempVdsCopyFileName; + + @BeforeClass + public void init() { + vds = new MemoryVdsGenerator(200, 200, 200, VolumeDataChannelDescriptor.Format.FORMAT_U8); + url = "inmemory://create_test"; + VolumeDataLayout volumeDataLayout = vds.getLayout(); + + int nbChannel = volumeDataLayout.getChannelCount(); + VolumeDataAccessManager accessManager = vds.getAccessManager(); + + for (VolumeDataLayoutDescriptor.LODLevels l : VolumeDataLayoutDescriptor.LODLevels.values()) { + for (int channel = 0; channel < nbChannel; channel++) { + for (DimensionsND dimGroup : DimensionsND.values()) { + VDSProduceStatus vdsProduceStatus = accessManager.getVDSProduceStatus(dimGroup, l.ordinal(), channel); + } + } + } + + vda = new VolumeDataAxisDescriptor[] { + volumeDataLayout.getAxisDescriptor(0), + volumeDataLayout.getAxisDescriptor(1), + volumeDataLayout.getAxisDescriptor(2)}; + vdc = new VolumeDataChannelDescriptor[] {volumeDataLayout.getChannelDescriptor(0)}; + + md = volumeDataLayout; + ld = volumeDataLayout.getLayoutDescriptor(); + + metadataContainer = new MetadataContainer(); + + long ms = System.currentTimeMillis(); + tempVolIndexerFileName = TEMP_FILE_NAME_VOL_INDEX + "_" + ms + ".vds"; + tempVdsCopyFileName = TEMP_FILE_NAME_COPY + "_" + ms + ".vds"; + } + + @AfterClass + public void cleanFiles() { + String tempDir = System.getProperty("java.io.tmpdir"); + String fileVolIndexPath = tempDir + File.separator + tempVolIndexerFileName; + File fileVolIndex = new File(fileVolIndexPath); + if (fileVolIndex.exists()) { + //fileVolIndex.delete(); + } + + String fileCopyPath = tempDir + File.separator + tempVdsCopyFileName; + File fileCopy = new File(fileCopyPath); + if (fileCopy.exists()) { + //fileCopy.delete(); + } + } + + @Test + public void testVolumeIndexerCreationDeletion() { + try { + // create file in tmp dir + String tmpDir = System.getProperty("java.io.tmpdir"); + String volIndexPath = tmpDir + File.separator + tempVolIndexerFileName; + VDSFileOpenOptions options = new VDSFileOpenOptions(volIndexPath); + VdsHandle vdsTest = OpenVDS.create(options, ld, + vda, + vdc, metadataContainer); + VolumeDataAccessManager accessManager = vdsTest.getAccessManager(); + //ASSERT_TRUE(accessManager); + + int channel = 0; + VolumeDataLayout layout = vdsTest.getLayout(); + VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( + DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND + 0, // lod + channel, // channel + 100, // max pages + VolumeDataPageAccessor.AccessMode.Create.getCode()); // access mode + + VolumeDataPage page = pageAccessor.createPage(0); + VolumeIndexer3D outputIndexer = new VolumeIndexer3D(page, 0, 0, DimensionsND.DIMENSIONS_012.ordinal(), layout); + outputIndexer.finalize(); + + vdsTest.close(); + } + catch (java.io.IOException e) { + System.out.println(e.getMessage()); + fail(); + } + } + + @Test + public void testCopyPageAccessor() { + try { + String tmpDir = System.getProperty("java.io.tmpdir"); + String vdsPath = tmpDir + File.separator + tempVdsCopyFileName; + VDSFileOpenOptions options = new VDSFileOpenOptions(vdsPath); + + // copy information from input VDS + VolumeDataLayoutDescriptor cpLd = new VolumeDataLayoutDescriptor( + ld.getBrickSize(), + ld.getNegativeMargin(), + ld.getPositiveMargin(), + ld.getBrickSizeMultiplier2D(), + ld.getLODLevels(), + ld.isCreate2DLODs(), + ld.isForceFullResolutionDimension(), + ld.getFullResolutionDimension() + ); + + VolumeDataAxisDescriptor[] cpVda = new VolumeDataAxisDescriptor[3]; + for (int i = 0 ; i < 3 ; ++i) { + cpVda[i] = new VolumeDataAxisDescriptor(vda[i].getNumSamples(), + vda[i].getName(), vda[i].getUnit(), + vda[i].getCoordinateMin(), vda[i].getCoordinateMax()); + } + + VolumeDataChannelDescriptor[] cpVdc = new VolumeDataChannelDescriptor[1]; + cpVdc[0] = new VolumeDataChannelDescriptor( + VolumeDataChannelDescriptor.Format.fromCode(vdc[0].getFormat()), + VolumeDataChannelDescriptor.Components.fromCode(vdc[0].getComponents()), + vdc[0].getName(), + vdc[0].getUnit(), + vdc[0].getValueRangeMin(), vdc[0].getValueRangeMax(), + VolumeDataMapping.fromCode(vdc[0].getMapping()), + vdc[0].getMappedValueCount(), + vdc[0].isDiscrete(), + vdc[0].isRenderable(), + vdc[0].isAllowLossyCompression(), + vdc[0].isUseZipForLosslessCompression(), + vdc[0].isUseNoValue(), + vdc[0].getNoValue(), + vdc[0].getIntegerScale(), + vdc[0].getIntegerOffset() + ); + MetadataContainer cpMetadataContainer = new MetadataContainer(); + VdsHandle vdsCopy = OpenVDS.create(options, cpLd, + cpVda, + cpVdc, + cpMetadataContainer); + + VolumeDataAccessManager accessManager = vdsCopy.getAccessManager(); + //ASSERT_TRUE(accessManager); + + int channel = 0; + VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( + DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND + 0, // lod + channel, // channel + 100, // max pages + VolumeDataPageAccessor.AccessMode.Create.getCode()); // access mode + + // get input manager + VolumeDataAccessManager inputAM = vds.getAccessManager(); + VolumeDataPageAccessor pageAccessorInput = inputAM.createVolumeDataPageAccessor( + DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND + 0, // lod + channel, // channel + 100, // max pages + VolumeDataPageAccessor.AccessMode.ReadOnly.getCode()); // access mode + + // copy file + int[] pitch = new int[VolumeDataLayout.Dimensionality_Max]; + long chunkCount = pageAccessorInput.getChunkCount(); + for (long chunk = 0 ; chunk < chunkCount ; ++chunk) { + VolumeDataPage inputPage = pageAccessorInput.readPage(chunk); + VolumeDataPage page = pageAccessor.createPage(chunk); + byte[] data = inputPage.readByteBuffer(pitch); + page.writeByteBuffer(data, pitch); + + inputPage.pageRelease(); + page.pageRelease(); + } + pageAccessor.commit(); + accessManager.flushUploadQueue(); + accessManager.destroyVolumeDataPageAccessor(pageAccessor); + + vdsCopy.close(); + } + catch (java.io.IOException e) { + System.out.println(e.getMessage()); + fail(); + } + } + + /** + * Will test that copied file is the same as the input + * Name of method is the same + Suffix so that it's executed after the copy test + */ + @Test + public void testCopyPageAccessorValidation() { + try { + String tmpDir = System.getProperty("java.io.tmpdir"); + String vdsPath = tmpDir + File.separator + tempVdsCopyFileName; + VDSFileOpenOptions options = new VDSFileOpenOptions(vdsPath); + VdsHandle vdsCopy = OpenVDS.open(options); + VolumeDataAccessManager accessManager = vdsCopy.getAccessManager(); + + int channel = 0; + VolumeDataLayout layout = vdsCopy.getLayout(); + VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( + DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND + 0, // lod + channel, // channel + 20, // max pages + VolumeDataPageAccessor.AccessMode.ReadOnly.getCode()); // access mode + + // get input manager + VolumeDataAccessManager inputAM = vds.getAccessManager(); + VolumeDataPageAccessor pageAccessorInput = inputAM.createVolumeDataPageAccessor( + DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND + 0, // lod + channel, // channel + 20, // max pages + VolumeDataPageAccessor.AccessMode.ReadOnly.getCode()); // access mode + + // compares block data + int[] pitchInput = new int[VolumeDataLayout.Dimensionality_Max]; + int[] pitchOutput = new int[VolumeDataLayout.Dimensionality_Max]; + + long chunkCount = pageAccessorInput.getChunkCount(); + for (long chunk = 0 ; chunk < chunkCount ; ++chunk) { + VolumeDataPage inputPage = pageAccessorInput.readPage(chunk); + VolumeDataPage page = pageAccessor.readPage(chunk); + byte[] dataIn = inputPage.readByteBuffer(pitchInput); + byte[] dataOut = page.readByteBuffer(pitchOutput); + + inputPage.pageRelease(); + page.pageRelease(); + + Assert.assertEquals(pitchInput, pitchOutput); + Assert.assertEquals(dataIn, dataOut); + } + + accessManager.destroyVolumeDataPageAccessor(pageAccessor); + + vdsCopy.close(); + } + catch (java.io.IOException e) { + System.out.println(e.getMessage()); + fail(); + } + } + + /** + * Will test that copied file has the same layout than original (test that 3D positions are in the same chunk index) + * Name of method is the same + Suffix so that it's executed after the copy test + */ + @Test + public void testCopyPageAccessorValidationChunkIndex() { + try { + String tmpDir = System.getProperty("java.io.tmpdir"); + String vdsPath = tmpDir + File.separator + tempVdsCopyFileName; + VDSFileOpenOptions options = new VDSFileOpenOptions(vdsPath); + VdsHandle vdsCopy = OpenVDS.open(options); + VolumeDataAccessManager accessManager = vdsCopy.getAccessManager(); + + int channel = 0; + VolumeDataLayout layout = vdsCopy.getLayout(); + VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( + DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND + 0, // lod + channel, // channel + 20, // max pages + VolumeDataPageAccessor.AccessMode.ReadOnly.getCode()); // access mode + + // compares block data + int[] chunkMin = new int[VolumeDataLayout.Dimensionality_Max]; + int[] chunkMax = new int[VolumeDataLayout.Dimensionality_Max]; + int[] chunkMaxPos = new int[VolumeDataLayout.Dimensionality_Max]; + + long chunkCount = pageAccessor.getChunkCount(); + for (long chunk = 0 ; chunk < chunkCount ; ++chunk) { + VolumeDataPage inputPage = pageAccessor.readPage(chunk); + + // check that chunk index matches current index + inputPage.getMinMaxExcludingMargin(chunkMin, chunkMax); + for (int i = 0 ; i < VolumeDataLayout.Dimensionality_Max ; ++i) { + chunkMaxPos[i] = chunkMax[i] != 0 ? chunkMax[i] - 1 : chunkMax[i]; + } + long idxChMin = pageAccessor.getChunkIndex(chunkMin); + long idxChMax = pageAccessor.getChunkIndex(chunkMaxPos); + + Assert.assertEquals(chunk, idxChMin); + Assert.assertEquals(idxChMin, idxChMax); + + inputPage.pageRelease(); + } + + accessManager.destroyVolumeDataPageAccessor(pageAccessor); + + vdsCopy.close(); + } + catch (java.io.IOException e) { + System.out.println(e.getMessage()); + fail(); + } + } +} -- GitLab From 311ead0300d363106217f23d42bb64eae4f1d1ae Mon Sep 17 00:00:00 2001 From: Julien Lacoste Date: Thu, 9 Sep 2021 17:18:35 +0200 Subject: [PATCH 04/25] Rename ErrorInfo class --- java/CMakeLists.txt | 2 +- ...rrorInfo.java => ConnectionErrorInfo.java} | 6 +++--- .../openvds/VolumeDataAccessManager.java | 20 +++++++++---------- 3 files changed, 14 insertions(+), 14 deletions(-) rename java/java/src/org/opengroup/openvds/{ErrorInfo.java => ConnectionErrorInfo.java} (84%) diff --git a/java/CMakeLists.txt b/java/CMakeLists.txt index 81675589..6b04f6f6 100644 --- a/java/CMakeLists.txt +++ b/java/CMakeLists.txt @@ -18,9 +18,9 @@ set(JAVA_SOURCE_FILES java/src/org/opengroup/openvds/BufferUtils.java java/src/org/opengroup/openvds/Cleaner.java java/src/org/opengroup/openvds/CompressionMethod.java + java/src/org/opengroup/openvds/ConnectionErrorInfo.java java/src/org/opengroup/openvds/DimensionsND.java java/src/org/opengroup/openvds/experimental/VariousJavaTests.java - java/src/org/opengroup/openvds/ErrorInfo.java java/src/org/opengroup/openvds/GoogleOpenOptions.java java/src/org/opengroup/openvds/IndexRegionFloat.java java/src/org/opengroup/openvds/IndexRegionLong.java diff --git a/java/java/src/org/opengroup/openvds/ErrorInfo.java b/java/java/src/org/opengroup/openvds/ConnectionErrorInfo.java similarity index 84% rename from java/java/src/org/opengroup/openvds/ErrorInfo.java rename to java/java/src/org/opengroup/openvds/ConnectionErrorInfo.java index fc9b81b7..c1584d91 100644 --- a/java/java/src/org/opengroup/openvds/ErrorInfo.java +++ b/java/java/src/org/opengroup/openvds/ConnectionErrorInfo.java @@ -3,15 +3,15 @@ package org.opengroup.openvds; /** * Simple class holding information on Upload/Download error */ -public class ErrorInfo { +public class ConnectionErrorInfo { private int errorCode = -1; private String errorMessage; private String objectID; - public ErrorInfo() { } + public ConnectionErrorInfo() { } - public ErrorInfo(int code, String msg, String id) { + public ConnectionErrorInfo(int code, String msg, String id) { errorCode = code; errorMessage = msg; objectID = id; diff --git a/java/java/src/org/opengroup/openvds/VolumeDataAccessManager.java b/java/java/src/org/opengroup/openvds/VolumeDataAccessManager.java index 86d24b8c..506ac63e 100644 --- a/java/java/src/org/opengroup/openvds/VolumeDataAccessManager.java +++ b/java/java/src/org/opengroup/openvds/VolumeDataAccessManager.java @@ -147,9 +147,9 @@ public class VolumeDataAccessManager extends JniPointerWithoutDeletion { private static native int cpGetCurrentUploadErrorCode(long handle); - private static native void cpGetCurrentUploadErrorInfo(long handle, ErrorInfo error); + private static native void cpGetCurrentUploadErrorInfo(long handle, ConnectionErrorInfo error); - private static native void cpGetCurrentDownloadErrorInfo(long handle, ErrorInfo error); + private static native void cpGetCurrentDownloadErrorInfo(long handle, ConnectionErrorInfo error); public VolumeDataAccessManager(long handle) { super(handle); @@ -779,18 +779,18 @@ public class VolumeDataAccessManager extends JniPointerWithoutDeletion { /** * @return current upload error information (code and message in a simple class) */ - public ErrorInfo getCurrentUploadErrorInfo() { - ErrorInfo errorInfo = new ErrorInfo(); - cpGetCurrentUploadErrorInfo(_handle, errorInfo); - return errorInfo; + public ConnectionErrorInfo getCurrentUploadErrorInfo() { + ConnectionErrorInfo connectionErrorInfo = new ConnectionErrorInfo(); + cpGetCurrentUploadErrorInfo(_handle, connectionErrorInfo); + return connectionErrorInfo; } /** * @return current upload error information (code and message in a simple class) */ - public ErrorInfo getCurrentDownloadErrorInfo() { - ErrorInfo errorInfo = new ErrorInfo(); - cpGetCurrentDownloadErrorInfo(_handle, errorInfo); - return errorInfo; + public ConnectionErrorInfo getCurrentDownloadErrorInfo() { + ConnectionErrorInfo connectionErrorInfo = new ConnectionErrorInfo(); + cpGetCurrentDownloadErrorInfo(_handle, connectionErrorInfo); + return connectionErrorInfo; } } -- GitLab From 1c7e5ecdefbde4cfbdab467dd7ace21de555983a Mon Sep 17 00:00:00 2001 From: Julien Lacoste Date: Thu, 9 Sep 2021 17:22:36 +0200 Subject: [PATCH 05/25] Remove temp test --- .../openvds/PageAccessorByteTest.java | 335 ------------------ 1 file changed, 335 deletions(-) delete mode 100644 java/java/test/org/opengroup/openvds/PageAccessorByteTest.java diff --git a/java/java/test/org/opengroup/openvds/PageAccessorByteTest.java b/java/java/test/org/opengroup/openvds/PageAccessorByteTest.java deleted file mode 100644 index 4e84d0a0..00000000 --- a/java/java/test/org/opengroup/openvds/PageAccessorByteTest.java +++ /dev/null @@ -1,335 +0,0 @@ -/* - * Copyright 2021 The Open Group - * Copyright 2021 INT, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opengroup.openvds; - -import org.testng.Assert; -import org.testng.annotations.AfterClass; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.Test; - -import java.io.File; - -import static org.testng.Assert.fail; - -public class PageAccessorByteTest { - - private static String TEMP_FILE_NAME_VOL_INDEX = "volIndexer"; - private static String TEMP_FILE_NAME_COPY = "vdsCopy"; - - public String url; - public VolumeDataLayoutDescriptor ld; - public VolumeDataAxisDescriptor[] vda; - public VolumeDataChannelDescriptor[] vdc; - public MetadataReadAccess md; - public MemoryVdsGenerator vds; - public MetadataContainer metadataContainer; - - private String tempVolIndexerFileName; - private String tempVdsCopyFileName; - - @BeforeClass - public void init() { - vds = new MemoryVdsGenerator(200, 200, 200, VolumeDataChannelDescriptor.Format.FORMAT_U8); - url = "inmemory://create_test"; - VolumeDataLayout volumeDataLayout = vds.getLayout(); - - int nbChannel = volumeDataLayout.getChannelCount(); - VolumeDataAccessManager accessManager = vds.getAccessManager(); - - for (VolumeDataLayoutDescriptor.LODLevels l : VolumeDataLayoutDescriptor.LODLevels.values()) { - for (int channel = 0; channel < nbChannel; channel++) { - for (DimensionsND dimGroup : DimensionsND.values()) { - VDSProduceStatus vdsProduceStatus = accessManager.getVDSProduceStatus(dimGroup, l.ordinal(), channel); - } - } - } - - vda = new VolumeDataAxisDescriptor[] { - volumeDataLayout.getAxisDescriptor(0), - volumeDataLayout.getAxisDescriptor(1), - volumeDataLayout.getAxisDescriptor(2)}; - vdc = new VolumeDataChannelDescriptor[] {volumeDataLayout.getChannelDescriptor(0)}; - - md = volumeDataLayout; - ld = volumeDataLayout.getLayoutDescriptor(); - - metadataContainer = new MetadataContainer(); - - long ms = System.currentTimeMillis(); - tempVolIndexerFileName = TEMP_FILE_NAME_VOL_INDEX + "_" + ms + ".vds"; - tempVdsCopyFileName = TEMP_FILE_NAME_COPY + "_" + ms + ".vds"; - } - - @AfterClass - public void cleanFiles() { - String tempDir = System.getProperty("java.io.tmpdir"); - String fileVolIndexPath = tempDir + File.separator + tempVolIndexerFileName; - File fileVolIndex = new File(fileVolIndexPath); - if (fileVolIndex.exists()) { - //fileVolIndex.delete(); - } - - String fileCopyPath = tempDir + File.separator + tempVdsCopyFileName; - File fileCopy = new File(fileCopyPath); - if (fileCopy.exists()) { - //fileCopy.delete(); - } - } - - @Test - public void testVolumeIndexerCreationDeletion() { - try { - // create file in tmp dir - String tmpDir = System.getProperty("java.io.tmpdir"); - String volIndexPath = tmpDir + File.separator + tempVolIndexerFileName; - VDSFileOpenOptions options = new VDSFileOpenOptions(volIndexPath); - VdsHandle vdsTest = OpenVDS.create(options, ld, - vda, - vdc, metadataContainer); - VolumeDataAccessManager accessManager = vdsTest.getAccessManager(); - //ASSERT_TRUE(accessManager); - - int channel = 0; - VolumeDataLayout layout = vdsTest.getLayout(); - VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( - DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND - 0, // lod - channel, // channel - 100, // max pages - VolumeDataPageAccessor.AccessMode.Create.getCode()); // access mode - - VolumeDataPage page = pageAccessor.createPage(0); - VolumeIndexer3D outputIndexer = new VolumeIndexer3D(page, 0, 0, DimensionsND.DIMENSIONS_012.ordinal(), layout); - outputIndexer.finalize(); - - vdsTest.close(); - } - catch (java.io.IOException e) { - System.out.println(e.getMessage()); - fail(); - } - } - - @Test - public void testCopyPageAccessor() { - try { - String tmpDir = System.getProperty("java.io.tmpdir"); - String vdsPath = tmpDir + File.separator + tempVdsCopyFileName; - VDSFileOpenOptions options = new VDSFileOpenOptions(vdsPath); - - // copy information from input VDS - VolumeDataLayoutDescriptor cpLd = new VolumeDataLayoutDescriptor( - ld.getBrickSize(), - ld.getNegativeMargin(), - ld.getPositiveMargin(), - ld.getBrickSizeMultiplier2D(), - ld.getLODLevels(), - ld.isCreate2DLODs(), - ld.isForceFullResolutionDimension(), - ld.getFullResolutionDimension() - ); - - VolumeDataAxisDescriptor[] cpVda = new VolumeDataAxisDescriptor[3]; - for (int i = 0 ; i < 3 ; ++i) { - cpVda[i] = new VolumeDataAxisDescriptor(vda[i].getNumSamples(), - vda[i].getName(), vda[i].getUnit(), - vda[i].getCoordinateMin(), vda[i].getCoordinateMax()); - } - - VolumeDataChannelDescriptor[] cpVdc = new VolumeDataChannelDescriptor[1]; - cpVdc[0] = new VolumeDataChannelDescriptor( - VolumeDataChannelDescriptor.Format.fromCode(vdc[0].getFormat()), - VolumeDataChannelDescriptor.Components.fromCode(vdc[0].getComponents()), - vdc[0].getName(), - vdc[0].getUnit(), - vdc[0].getValueRangeMin(), vdc[0].getValueRangeMax(), - VolumeDataMapping.fromCode(vdc[0].getMapping()), - vdc[0].getMappedValueCount(), - vdc[0].isDiscrete(), - vdc[0].isRenderable(), - vdc[0].isAllowLossyCompression(), - vdc[0].isUseZipForLosslessCompression(), - vdc[0].isUseNoValue(), - vdc[0].getNoValue(), - vdc[0].getIntegerScale(), - vdc[0].getIntegerOffset() - ); - MetadataContainer cpMetadataContainer = new MetadataContainer(); - VdsHandle vdsCopy = OpenVDS.create(options, cpLd, - cpVda, - cpVdc, - cpMetadataContainer); - - VolumeDataAccessManager accessManager = vdsCopy.getAccessManager(); - //ASSERT_TRUE(accessManager); - - int channel = 0; - VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( - DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND - 0, // lod - channel, // channel - 100, // max pages - VolumeDataPageAccessor.AccessMode.Create.getCode()); // access mode - - // get input manager - VolumeDataAccessManager inputAM = vds.getAccessManager(); - VolumeDataPageAccessor pageAccessorInput = inputAM.createVolumeDataPageAccessor( - DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND - 0, // lod - channel, // channel - 100, // max pages - VolumeDataPageAccessor.AccessMode.ReadOnly.getCode()); // access mode - - // copy file - int[] pitch = new int[VolumeDataLayout.Dimensionality_Max]; - long chunkCount = pageAccessorInput.getChunkCount(); - for (long chunk = 0 ; chunk < chunkCount ; ++chunk) { - VolumeDataPage inputPage = pageAccessorInput.readPage(chunk); - VolumeDataPage page = pageAccessor.createPage(chunk); - byte[] data = inputPage.readByteBuffer(pitch); - page.writeByteBuffer(data, pitch); - - inputPage.pageRelease(); - page.pageRelease(); - } - pageAccessor.commit(); - accessManager.flushUploadQueue(); - accessManager.destroyVolumeDataPageAccessor(pageAccessor); - - vdsCopy.close(); - } - catch (java.io.IOException e) { - System.out.println(e.getMessage()); - fail(); - } - } - - /** - * Will test that copied file is the same as the input - * Name of method is the same + Suffix so that it's executed after the copy test - */ - @Test - public void testCopyPageAccessorValidation() { - try { - String tmpDir = System.getProperty("java.io.tmpdir"); - String vdsPath = tmpDir + File.separator + tempVdsCopyFileName; - VDSFileOpenOptions options = new VDSFileOpenOptions(vdsPath); - VdsHandle vdsCopy = OpenVDS.open(options); - VolumeDataAccessManager accessManager = vdsCopy.getAccessManager(); - - int channel = 0; - VolumeDataLayout layout = vdsCopy.getLayout(); - VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( - DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND - 0, // lod - channel, // channel - 20, // max pages - VolumeDataPageAccessor.AccessMode.ReadOnly.getCode()); // access mode - - // get input manager - VolumeDataAccessManager inputAM = vds.getAccessManager(); - VolumeDataPageAccessor pageAccessorInput = inputAM.createVolumeDataPageAccessor( - DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND - 0, // lod - channel, // channel - 20, // max pages - VolumeDataPageAccessor.AccessMode.ReadOnly.getCode()); // access mode - - // compares block data - int[] pitchInput = new int[VolumeDataLayout.Dimensionality_Max]; - int[] pitchOutput = new int[VolumeDataLayout.Dimensionality_Max]; - - long chunkCount = pageAccessorInput.getChunkCount(); - for (long chunk = 0 ; chunk < chunkCount ; ++chunk) { - VolumeDataPage inputPage = pageAccessorInput.readPage(chunk); - VolumeDataPage page = pageAccessor.readPage(chunk); - byte[] dataIn = inputPage.readByteBuffer(pitchInput); - byte[] dataOut = page.readByteBuffer(pitchOutput); - - inputPage.pageRelease(); - page.pageRelease(); - - Assert.assertEquals(pitchInput, pitchOutput); - Assert.assertEquals(dataIn, dataOut); - } - - accessManager.destroyVolumeDataPageAccessor(pageAccessor); - - vdsCopy.close(); - } - catch (java.io.IOException e) { - System.out.println(e.getMessage()); - fail(); - } - } - - /** - * Will test that copied file has the same layout than original (test that 3D positions are in the same chunk index) - * Name of method is the same + Suffix so that it's executed after the copy test - */ - @Test - public void testCopyPageAccessorValidationChunkIndex() { - try { - String tmpDir = System.getProperty("java.io.tmpdir"); - String vdsPath = tmpDir + File.separator + tempVdsCopyFileName; - VDSFileOpenOptions options = new VDSFileOpenOptions(vdsPath); - VdsHandle vdsCopy = OpenVDS.open(options); - VolumeDataAccessManager accessManager = vdsCopy.getAccessManager(); - - int channel = 0; - VolumeDataLayout layout = vdsCopy.getLayout(); - VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( - DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND - 0, // lod - channel, // channel - 20, // max pages - VolumeDataPageAccessor.AccessMode.ReadOnly.getCode()); // access mode - - // compares block data - int[] chunkMin = new int[VolumeDataLayout.Dimensionality_Max]; - int[] chunkMax = new int[VolumeDataLayout.Dimensionality_Max]; - int[] chunkMaxPos = new int[VolumeDataLayout.Dimensionality_Max]; - - long chunkCount = pageAccessor.getChunkCount(); - for (long chunk = 0 ; chunk < chunkCount ; ++chunk) { - VolumeDataPage inputPage = pageAccessor.readPage(chunk); - - // check that chunk index matches current index - inputPage.getMinMaxExcludingMargin(chunkMin, chunkMax); - for (int i = 0 ; i < VolumeDataLayout.Dimensionality_Max ; ++i) { - chunkMaxPos[i] = chunkMax[i] != 0 ? chunkMax[i] - 1 : chunkMax[i]; - } - long idxChMin = pageAccessor.getChunkIndex(chunkMin); - long idxChMax = pageAccessor.getChunkIndex(chunkMaxPos); - - Assert.assertEquals(chunk, idxChMin); - Assert.assertEquals(idxChMin, idxChMax); - - inputPage.pageRelease(); - } - - accessManager.destroyVolumeDataPageAccessor(pageAccessor); - - vdsCopy.close(); - } - catch (java.io.IOException e) { - System.out.println(e.getMessage()); - fail(); - } - } -} -- GitLab From 99d87c01c5586957605a486756e8ad271e672b66 Mon Sep 17 00:00:00 2001 From: Julien Lacoste Date: Tue, 24 Aug 2021 16:22:07 +0200 Subject: [PATCH 06/25] Java bindings : Add methods on VolumeDataAccessManager to get info on Upload/Download error --- java/CMakeLists.txt | 1 + java/cpp/src/VolumeDataAccessManager.cpp | 61 +++++++++++++++++++ .../src/org/opengroup/openvds/ErrorInfo.java | 44 +++++++++++++ .../openvds/VolumeDataAccessManager.java | 22 +++++++ 4 files changed, 128 insertions(+) create mode 100644 java/java/src/org/opengroup/openvds/ErrorInfo.java diff --git a/java/CMakeLists.txt b/java/CMakeLists.txt index ee7ca465..1d7e322f 100644 --- a/java/CMakeLists.txt +++ b/java/CMakeLists.txt @@ -20,6 +20,7 @@ set(JAVA_SOURCE_FILES java/src/org/opengroup/openvds/CompressionMethod.java java/src/org/opengroup/openvds/DimensionsND.java java/src/org/opengroup/openvds/experimental/VariousJavaTests.java + java/src/org/opengroup/openvds/ErrorInfo.java java/src/org/opengroup/openvds/GoogleOpenOptions.java java/src/org/opengroup/openvds/IndexRegionFloat.java java/src/org/opengroup/openvds/IndexRegionLong.java diff --git a/java/cpp/src/VolumeDataAccessManager.cpp b/java/cpp/src/VolumeDataAccessManager.cpp index 0968092d..c6f723af 100644 --- a/java/cpp/src/VolumeDataAccessManager.cpp +++ b/java/cpp/src/VolumeDataAccessManager.cpp @@ -583,6 +583,67 @@ JNIEXPORT jint JNICALL Java_org_opengroup_openvds_VolumeDataAccessManager_cpGetC return 0; } +/* + * Class: org_opengroup_openvds_VolumeDataAccessManager + * Method: cpGetCurrentUploadErrorInfo + * Signature: (JLorg/opengroup/openvds/ErrorInfo;)V + */ +JNIEXPORT void JNICALL Java_org_opengroup_openvds_VolumeDataAccessManager_cpGetCurrentUploadErrorInfo + (JNIEnv * env, jclass, jlong handle, jobject errorObj) +{ + try { + const char *pObjectID = nullptr; + const char *pErrorString = nullptr; + int32_t errorCode = 0; + + GetManager(handle)->GetCurrentUploadError(&pObjectID, &errorCode, &pErrorString); + + // Get the class of the given error object + jclass clazz = env->GetObjectClass(errorObj); + + // Get Field references + jfieldID paramCode = env->GetFieldID(clazz, "errorCode", "I"); + jfieldID paramMessage = env->GetFieldID(clazz, "errorMessage", "Ljava/lang/String;"); + jfieldID paramID = env->GetFieldID(clazz, "objectID", "Ljava/lang/String;"); + + // Set fields for object + jstring strMsg = NewJString(env, pErrorString); + jstring strID = NewJString(env, pObjectID); + env->SetIntField(errorObj, paramCode, errorCode); + env->SetObjectField(errorObj, paramMessage, strMsg); + env->SetObjectField(errorObj, paramID, strID); + } + CATCH_EXCEPTIONS_FOR_JAVA; +} + +/* + * Class: org_opengroup_openvds_VolumeDataAccessManager + * Method: cpGetCurrentDownloadErrorInfo + * Signature: (JLorg/opengroup/openvds/ErrorInfo;)V + */ +JNIEXPORT void JNICALL Java_org_opengroup_openvds_VolumeDataAccessManager_cpGetCurrentDownloadErrorInfo + (JNIEnv * env, jclass, jlong handle, jobject errorObj) +{ + try { + const char *pErrorString = nullptr; + int32_t errorCode = 0; + GetManager(handle)->GetCurrentDownloadError(&errorCode, &pErrorString); + + // Get the class of the given error object + jclass clazz = env->GetObjectClass(errorObj); + + // Get Field references + jfieldID paramCode = env->GetFieldID(clazz, "errorCode", "I"); + jfieldID paramMessage = env->GetFieldID(clazz, "errorMessage", "Ljava/lang/String;"); + + // Set fields for object + env->SetIntField(errorObj, paramCode, errorCode); + jstring strMsg = NewJString(env, pErrorString); + env->SetObjectField(errorObj, paramMessage, strMsg); + } + CATCH_EXCEPTIONS_FOR_JAVA; +} + #ifdef __cplusplus } #endif diff --git a/java/java/src/org/opengroup/openvds/ErrorInfo.java b/java/java/src/org/opengroup/openvds/ErrorInfo.java new file mode 100644 index 00000000..4dd9f017 --- /dev/null +++ b/java/java/src/org/opengroup/openvds/ErrorInfo.java @@ -0,0 +1,44 @@ +package org.opengroup.openvds; + +/** + * Simple class holding information on Upload/Download error + */ +public class ErrorInfo { + + private int errorCode; + private String errorMessage; + private String objectID; + + public ErrorInfo() { + + } + public ErrorInfo(int code, String msg, String id) { + errorCode = code; + errorMessage = msg; + objectID = id; + } + + public int getErrorCode() { + return errorCode; + } + + public void setErrorCode(int errorCode) { + this.errorCode = errorCode; + } + + public String getErrorMessage() { + return errorMessage; + } + + public void setErrorMessage(String errorMessage) { + this.errorMessage = errorMessage; + } + + public String getObjectID() { + return objectID; + } + + public void setObjectID(String objectID) { + this.objectID = objectID; + } +} diff --git a/java/java/src/org/opengroup/openvds/VolumeDataAccessManager.java b/java/java/src/org/opengroup/openvds/VolumeDataAccessManager.java index 2af400d2..85622597 100644 --- a/java/java/src/org/opengroup/openvds/VolumeDataAccessManager.java +++ b/java/java/src/org/opengroup/openvds/VolumeDataAccessManager.java @@ -147,6 +147,10 @@ public class VolumeDataAccessManager extends JniPointerWithoutDeletion { private static native int cpGetCurrentUploadErrorCode(long handle); + private static native void cpGetCurrentUploadErrorInfo(long handle, ErrorInfo error); + + private static native void cpGetCurrentDownloadErrorInfo(long handle, ErrorInfo error); + public VolumeDataAccessManager(long handle) { super(handle); } @@ -771,4 +775,22 @@ public class VolumeDataAccessManager extends JniPointerWithoutDeletion { public int getCurrentUploadErrorCode() { return cpGetCurrentUploadErrorCode(_handle); } + + /** + * @return current upload error information (code and message in a simple class) + */ + public ErrorInfo getCurrentUploadErrorInfo() { + ErrorInfo errorInfo = new ErrorInfo(); + cpGetCurrentUploadErrorInfo(_handle, errorInfo); + return errorInfo; + } + + /** + * @return current upload error information (code and message in a simple class) + */ + public ErrorInfo getCurrentDownloadErrorInfo() { + ErrorInfo errorInfo = new ErrorInfo(); + cpGetCurrentUploadErrorInfo(_handle, errorInfo); + return errorInfo; + } } -- GitLab From 0aacb2215a3cf512e0767cd8332efe0c72fca8c3 Mon Sep 17 00:00:00 2001 From: Julien Lacoste Date: Tue, 24 Aug 2021 16:28:16 +0200 Subject: [PATCH 07/25] Java bindings : Add methods on VolumeDataAccessManager to get info on Upload/Download error + fix wrong native call --- .../java/src/org/opengroup/openvds/VolumeDataAccessManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/java/src/org/opengroup/openvds/VolumeDataAccessManager.java b/java/java/src/org/opengroup/openvds/VolumeDataAccessManager.java index 85622597..86d24b8c 100644 --- a/java/java/src/org/opengroup/openvds/VolumeDataAccessManager.java +++ b/java/java/src/org/opengroup/openvds/VolumeDataAccessManager.java @@ -790,7 +790,7 @@ public class VolumeDataAccessManager extends JniPointerWithoutDeletion { */ public ErrorInfo getCurrentDownloadErrorInfo() { ErrorInfo errorInfo = new ErrorInfo(); - cpGetCurrentUploadErrorInfo(_handle, errorInfo); + cpGetCurrentDownloadErrorInfo(_handle, errorInfo); return errorInfo; } } -- GitLab From c07ee5c2154fba3c16805ea816f790426b8e7ed0 Mon Sep 17 00:00:00 2001 From: Julien Lacoste Date: Thu, 2 Sep 2021 12:17:34 +0200 Subject: [PATCH 08/25] bkp --- java/java/demo/CreateVDS.java | 2 +- java/java/demo/CreateVDSCloud.java | 578 ++++++++++++++++++ .../src/org/opengroup/openvds/ErrorInfo.java | 5 +- .../openvds/PageAccessorByteTest.java | 335 ++++++++++ 4 files changed, 916 insertions(+), 4 deletions(-) create mode 100644 java/java/demo/CreateVDSCloud.java create mode 100644 java/java/test/org/opengroup/openvds/PageAccessorByteTest.java diff --git a/java/java/demo/CreateVDS.java b/java/java/demo/CreateVDS.java index 0931bd12..964a84fa 100644 --- a/java/java/demo/CreateVDS.java +++ b/java/java/demo/CreateVDS.java @@ -157,7 +157,7 @@ public class CreateVDS { true, // is renderable false, // allow lossy compression false, // use zip for lossless compresion - true, // use no value + false, // use no value -999.25f, // no value scaleOffset[0], // integer scale scaleOffset[1]); // integer offset diff --git a/java/java/demo/CreateVDSCloud.java b/java/java/demo/CreateVDSCloud.java new file mode 100644 index 00000000..d31b31e4 --- /dev/null +++ b/java/java/demo/CreateVDSCloud.java @@ -0,0 +1,578 @@ +/* + * Copyright 2021 The Open Group + * Copyright 2021 INT, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.opengroup.openvds.*; + +import java.io.File; +import java.io.IOException; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; + +public class CreateVDSCloud { + + // usage : CreateVDSWithLOD n /path/to/file + // where n is the LOD level wanted + public static void main(String[] args) { + try { + if (!checkParams(args)) { + System.out.println("Bad params, usage : n /path/to/file dimX dimY dimZ"); + System.out.println("n is the LOD level wanted (0 <= lod <= 12), path must be a valid non existing file path"); + System.out.println("dims are cube dimension (must be > 0)"); + } + else { + Instant start = Instant.now(); + process(Integer.parseInt(args[0]), args[1], Integer.parseInt(args[2]), Integer.parseInt(args[3]), Integer.parseInt(args[4])); + Instant end = Instant.now(); + + Duration elapsed = Duration.between(start, end); + long hrs = elapsed.toHours(); + long min = elapsed.toMinutes() - (hrs * 60); + long s = (elapsed.toMillis() - (elapsed.toMinutes() * 60 * 1000)) / 1000; + System.out.println("Write VDS TIME : " + hrs + " hrs " + min + " min " + s + "s (" + elapsed.toMillis() + " ms)"); + + testRead(args[1]); + } + } + catch (Throwable t) { + System.out.println(); + t.printStackTrace(); + } + } + + private static boolean checkParams(String[] args) { + if (args == null || args.length != 5) { + return false; + } + try { + Integer lod = Integer.parseInt(args[0]); + if (lod < 0 || lod > 12) { + System.err.println("Invalid LOD value (must be between 0 and 12) !"); + return false; + } + } + catch (NumberFormatException nfe) { + return false; + } + String path = args[1]; + File file = new File(path); + try { + file.getCanonicalPath(); + } + catch (IOException e) { + System.err.println("VDS File path is invalid !"); + return false; + } + if (!path.endsWith(".vds")) { + System.err.println("VDS File path must have vds extension !"); + return false; + } + if (file.exists()) { + System.err.println("VDS File already exists !"); + return false; + } + try { + Integer dimX = Integer.parseInt(args[2]); + if (dimX < 0) { + System.err.println("Invalid x dimension !"); + return false; + } + Integer dimY = Integer.parseInt(args[3]); + if (dimX < 0) { + System.err.println("Invalid y dimension !"); + return false; + } + Integer dimZ = Integer.parseInt(args[4]); + if (dimX < 0) { + System.err.println("Invalid Z dimension !"); + return false; + } + } + catch (NumberFormatException nfe) { + return false; + } + return true; + } + + static void process(int lodParam, String name, int dimX, int dimY, int dimZ) throws Exception { + + int samplesX = dimZ; // time + int samplesY = dimY; // XL + int samplesZ = dimX; // IL + VolumeDataChannelDescriptor.Format format = VolumeDataChannelDescriptor.Format.FORMAT_R32; + + double sizeX = samplesX; + double sizeY = samplesY; + double sizeZ = samplesZ; + + double distMax = distance3D(0, 0, 0, sizeX, sizeY, sizeZ); + double cycles = Math.PI * 2 * 20; + double midX = samplesX / 2f; + double midY = samplesY / 2f; + double midZ = samplesZ / 2f; + + VolumeDataLayoutDescriptor.BrickSize brickSize = VolumeDataLayoutDescriptor.BrickSize.BRICK_SIZE_64; + int negativeMargin = 4; + int positiveMargin = 4; + int brickSize2DMultiplier = 4; + + VolumeDataLayoutDescriptor.LODLevels lodLevels = VolumeDataLayoutDescriptor.LODLevels.values()[lodParam]; + + //VolumeDataLayoutDescriptor.Options layoutOptions = VolumeDataLayoutDescriptor.Options.NONE; + VolumeDataLayoutDescriptor layoutDescriptor = new VolumeDataLayoutDescriptor(brickSize, negativeMargin, positiveMargin, + brickSize2DMultiplier, lodLevels, false,false,0); + + List axisDescriptors = new ArrayList<>(); + axisDescriptors.add(new VolumeDataAxisDescriptor(samplesX, "Sample", "s", 0.0f,6.0f)); + axisDescriptors.add(new VolumeDataAxisDescriptor(samplesY, "Crossline", "",0f, 0f + samplesY - 1f)); + axisDescriptors.add(new VolumeDataAxisDescriptor(samplesZ, "Inline", "", 0f, 0f + samplesZ - 1f)); + + List channelDescriptors = new ArrayList<>(); + + float rangeMin = -1f; + float rangeMax = 1f; + float[] scaleOffset = getScaleOffsetForFormat(rangeMin, rangeMax, true, format); + + VolumeDataChannelDescriptor channelDescriptor = new VolumeDataChannelDescriptor(format, VolumeDataChannelDescriptor.Components.COMPONENTS_1, + "Amplitude", + "", + rangeMin, // range min + rangeMax, // range max + VolumeDataMapping.DIRECT, // mapping + 1, // mapped value count + false, // is discrete + true, // is renderable + false, // allow lossy compression + false, // use zip for lossless compresion + false, // use no value + -999.25f, // no value + scaleOffset[0], // integer scale + scaleOffset[1]); // integer offset + channelDescriptors.add(channelDescriptor); + + // options AWS + String pBucket = "miniovaap"; // the bucket of the VDS + String pKey = name; // the key prefix of the VDS ? + String pRegion = "eu-west-3"; // the region of the bucket of the VDS + String pEndpointOverride = "https://webvr.int.com/"; // This parameter allows to override the endpoint url + AWSOpenOptions optionsAWS = new AWSOpenOptions(pBucket, pKey, pRegion, pEndpointOverride); + optionsAWS.accessKeyId = "minioadmin"; // ? + optionsAWS.secretKey = "minioadmin"; + optionsAWS.sessionToken = ""; // ? + + // file options + //VDSFileOpenOptions options = new VDSFileOpenOptions(vdsFilePath); + + MetadataContainer metadataContainer = new MetadataContainer(); + StringBuilder builder = new StringBuilder(); + builder.append("CreateVDS Demo "); + builder.append("dimX: "); + builder.append(dimX); + builder.append(" dimY: "); + builder.append(dimY); + builder.append(" dimZ: "); + builder.append(dimZ); + + metadataContainer.setMetadataString("General Info", "Cube layout", builder.toString()); + metadataContainer.setMetadataInt("General Info", "LOD", lodLevels.ordinal()); + metadataContainer.setMetadataString("General Info", "Brick Size", "BRICK_SIZE_64"); + + VdsHandle vds = OpenVDS.create(optionsAWS, layoutDescriptor, + axisDescriptors.toArray(new VolumeDataAxisDescriptor[0]), + channelDescriptors.toArray(new VolumeDataChannelDescriptor[0]), metadataContainer); + + // for compression use +// float compressionTolerance = 0f; // +// VdsHandle vds = OpenVDS.create(options, layoutDescriptor, +// axisDescriptors.toArray(new VolumeDataAxisDescriptor[0]), +// channelDescriptors.toArray(new VolumeDataChannelDescriptor[0]), metadataContainer, CompressionMethod.ZIP, compressionTolerance); + + VolumeDataLayout layout = vds.getLayout(); + VolumeDataAccessManager accessManager = vds.getAccessManager(); + + int channel = 0; + + int lodCount = lodLevels.ordinal(); + + int[] localIndex = new int[3]; + int[] voxelPos = new int[3]; + + int[] pitch = new int[VolumeDataLayout.Dimensionality_Max]; + int[] chunkMin = new int[VolumeDataLayout.Dimensionality_Max]; + int[] chunkMax = new int[VolumeDataLayout.Dimensionality_Max]; + + for (int lod = 0 ; lod <= lodCount ; ++lod) { + VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( + DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND + lod, // lod + channel, // channel + 50, // max pages + VolumeDataPageAccessor.AccessMode.Create.getCode()); // access mode + + + //ASSERT_TRUE(pageAccessor); + int paLOD = pageAccessor.getLOD(); + int paCI = pageAccessor.getChannelIndex(); + int[] paNumSamples = pageAccessor.getNumSamples(); + + int chunkCount = (int) pageAccessor.getChunkCount(); + System.out.println("LOD : " + lod + " Chunk count : " + chunkCount); + + for (int i = 0; i < chunkCount; i++) { + + long time_1 = System.currentTimeMillis(); + + VolumeDataPage page = pageAccessor.createPage(i); + System.out.print("LOD : " + lod + " / Page " + (i + 1) + " / " + chunkCount); + + VolumeIndexer3D outputIndexer = new VolumeIndexer3D(page, 0, lod, DimensionsND.DIMENSIONS_012.ordinal(), layout); + + int[] numSamplesChunk = new int[3]; + int[] numSamplesDB = new int[3]; + + for (int j = 0; j < 3; j++) { + numSamplesChunk[j] = outputIndexer.getLocalChunkNumSamples(j); + numSamplesDB[j] = outputIndexer.getDataBlockNumSamples(j); + } + System.out.print("\tDimensions Chunk : [" + numSamplesChunk[0] + ", " + numSamplesChunk[1] + ", " + numSamplesChunk[2] + "]"); + System.out.print("\tDimensions DataBlock : [" + numSamplesDB[0] + ", " + numSamplesDB[1] + ", " + numSamplesDB[2] + "]"); + + page.getMinMax(chunkMin, chunkMax); + + System.out.print("\tCoords page : " + chunkMin[0] + "/" + chunkMax[0] + " " + chunkMin[1] + "/" + chunkMax[1] + " " + chunkMin[2] + "/" + chunkMax[2]); + int chunkSizeY = chunkMax[2] - chunkMin[2]; + int chunkSizeX = chunkMax[1] - chunkMin[1]; + int chunkSizeZ = chunkMax[0] - chunkMin[0]; + + // get pitch and allocate buffer size + page.getPitch(pitch); + float[] output = new float[page.getAllocatedBufferSize()]; + + // or read buffer of page + //float[] output = page.readFloatBuffer(pitch); + + System.out.print( " Page Buffer Size : " + output.length + " ");/// [" + stats.getMin() + "," + stats.getMax() + "]"); + // dimension of buffer + System.out.println(" Pitch : [" + pitch[0] + ", " + pitch[1] + ", " + pitch[2] + "]"); + + int[] numSamples = numSamplesChunk; + + for (int iDim2 = 0; iDim2 < numSamples[2]; iDim2++) { + localIndex[2] = iDim2; + for (int iDim1 = 0; iDim1 < numSamples[1]; iDim1++) { + localIndex[1] = iDim1; + for (int iDim0 = 0; iDim0 < numSamples[0]; iDim0++) { + localIndex[0] = iDim0; + + int[] voxelIndex = outputIndexer.localIndexToVoxelIndex(localIndex); + for (int vp = 0; vp < 3; ++vp) { + voxelPos[vp] = voxelIndex[vp]; + } + + float value = 0f; + double dist = 0; + if (voxelPos[0] >= midX) { + dist = distance2D(midY, midZ, voxelPos[1], voxelPos[2]); + } else { + dist = distance3D(midX, midY, midZ, voxelPos[0], voxelPos[1], voxelPos[2]); + } + value = (float) Math.sin((dist * cycles) / distMax); + + int iPos = outputIndexer.localIndexToDataIndex(localIndex); + output[iPos] = value; + } + } + } + + // write buffer then release page + page.writeFloatBuffer(output, pitch); + page.pageRelease(); + } + + int errorCount = accessManager.uploadErrorCount(); + if (errorCount > 0) { + ErrorInfo upErrInfo = accessManager.getCurrentUploadErrorInfo(); + System.out.println("Upload error code : " + upErrInfo.getErrorCode()); + if (upErrInfo.getErrorMessage() != null) { + System.out.println("Upload error message : " + upErrInfo.getErrorMessage()); + } + else { + System.out.println("No error message"); + } + if (upErrInfo.getObjectID() != null) { + System.out.println("Upload error object ID : " + upErrInfo.getObjectID()); + } + else { + System.out.println("No Object ID"); + } + } + + pageAccessor.commit(); + pageAccessor.setMaxPages(0); + accessManager.flushUploadQueue(); + accessManager.destroyVolumeDataPageAccessor(pageAccessor); + } + + + + // Adds slice data : additional layouts +// System.out.println("\nCreate 01 Layout"); +// create2DLayout(distMax, cycles, midX, midY, midZ, layout, accessManager, channel, lodCount, DimensionsND.DIMENSIONS_01); +// +// System.out.println("\nCreate 02 Layout"); +// create2DLayout(distMax, cycles, midX, midY, midZ, layout, accessManager, channel, lodCount, DimensionsND.DIMENSIONS_02); +// +// System.out.println("\nCreate 12 Layout"); +// create2DLayout(distMax, cycles, midX, midY, midZ, layout, accessManager, channel, lodCount, DimensionsND.DIMENSIONS_12); + + + vds.close(); + System.out.println("VDS closed"); + } + + + private static void create2DLayout(double distMax, double cycles, double midX, double midY, double midZ, VolumeDataLayout layout, VolumeDataAccessManager accessManager, + int channel, int lodCount, DimensionsND dimLYT) { + + int[] localIndex = new int[3]; + + int[] pitch = new int[VolumeDataLayout.Dimensionality_Max]; + int[] chunkMin = new int[VolumeDataLayout.Dimensionality_Max]; + int[] chunkMax = new int[VolumeDataLayout.Dimensionality_Max]; + for (int lod = 0 ; lod <= lodCount ; ++lod) { + VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( + dimLYT.ordinal(), // dimension ND + lod, // lod + channel, // channel + 200, // max pages + VolumeDataPageAccessor.AccessMode.Create.getCode()); // access mode + + int chunkCount = (int) pageAccessor.getChunkCount(); + System.out.println("2D Layout LOD : " + lod + " Chunk count : " + chunkCount); + + for (int i = 0; i < chunkCount; i++) { + VolumeDataPage page = pageAccessor.createPage(i); + System.out.print("2D Layout LOD : " + lod + " / Page " + (i + 1) + " / " + chunkCount); + + VolumeIndexer2D outputIndexer = new VolumeIndexer2D(page, 0, lod, dimLYT.ordinal(), layout); + float valueRangeScale = outputIndexer.getValueRangeMax() - outputIndexer.getValueRangeMin(); + + int[] numSamplesChunk = new int[3]; + int[] numSamplesDB = new int[3]; + + for (int j = 0; j < 3 ; j++) { + numSamplesChunk[j] = outputIndexer.getLocalChunkNumSamples(j); + numSamplesDB[j] = outputIndexer.getDataBlockNumSamples(j); + } + System.out.print("\t2D Layout Dimensions Chunk : [" + numSamplesChunk[0] + ", " + numSamplesChunk[1] + ", " + numSamplesChunk[2] + "]"); + System.out.print("\t2D Layout Dimensions DataBlock : [" + numSamplesDB[0] + ", " + numSamplesDB[1] + ", " + numSamplesDB[2] + "]"); + + page.getMinMax(chunkMin, chunkMax); + + // determine fixed position index and value + int fixedCoordIndex = -1; + int fixedCoordValue = -1; + for (int pos = 0; pos < 3 && fixedCoordIndex == -1 ; ++pos) { + if (Math.abs(chunkMax[pos] - chunkMin[pos]) == 1) { + fixedCoordIndex = pos; + fixedCoordValue = chunkMin[pos]; + } + } + + System.out.print("\tCoords page : " + chunkMin[0] + "/" + chunkMax[0] + " " + chunkMin[1] + "/" + chunkMax[1] + "/" + chunkMin[2] + "/" + chunkMax[2]); + int chunkSizeY = chunkMax[2] - chunkMin[2]; + int chunkSizeX = chunkMax[1] - chunkMin[1]; + int chunkSizeZ = chunkMax[0] - chunkMin[0]; + + // get pitch and allocate buffer size + page.getPitch(pitch); + float[] output = new float[page.getAllocatedBufferSize()]; + + System.out.print( " Page Buffer Size : " + output.length);// + " / [" + stats.getMin() + "," + stats.getMax() + "]"); + // dimension of buffer + System.out.println(" Pitch : [" + pitch[0] + ", " + pitch[1] + "," + pitch[2] + "]"); + + long time_2 = System.currentTimeMillis(); + + int[] numSamples = numSamplesChunk; + + for (int iDim2 = 0; iDim2 < numSamples[2]; iDim2++) { + localIndex[2] = iDim2; + for (int iDim1 = 0; iDim1 < numSamples[1]; iDim1++) { + localIndex[1] = iDim1; + for (int iDim0 = 0; iDim0 < numSamples[0]; iDim0++) { + localIndex[0] = iDim0; + int[] local2DIndex = convertTo2DLocalIndex(localIndex, fixedCoordIndex); + int[] voxelIndex = outputIndexer.localChunkIndexToLocalIndex(local2DIndex); + int[] voxelPos = convertTo3DCoord(voxelIndex, fixedCoordIndex, fixedCoordValue); + + float value = 0f; + double dist = 0; + if (voxelPos[0] >= midX) { + dist = distance2D(midY, midZ, voxelPos[1], voxelPos[2]); + } else { + dist = distance3D(midX, midY, midZ, voxelPos[0], voxelPos[1], voxelPos[2]); + } + value = (float) Math.sin((dist * cycles) / distMax); + + int iPos = outputIndexer.localIndexToDataIndex(local2DIndex); + output[iPos] = value; + } + } + } + + // write buffer then release page + page.writeFloatBuffer(output, pitch); + page.pageRelease(); + } + + pageAccessor.commit(); + accessManager.flushUploadQueue(); + accessManager.destroyVolumeDataPageAccessor(pageAccessor); + } + } + + static void testRead(String vdsName) throws IOException { + // options AWS + String pBucket = "miniovaap"; // the bucket of the VDS + String pKey = vdsName; // the key prefix of the VDS ? + String pRegion = "eu-west-3"; // the region of the bucket of the VDS + String pEndpointOverride = "https://webvr.int.com/"; // This parameter allows to override the endpoint url + AWSOpenOptions optionsAWS = new AWSOpenOptions(pBucket, pKey, pRegion, pEndpointOverride); + optionsAWS.accessKeyId = "minioadmin"; // ? + optionsAWS.secretKey = "minioadmin"; + optionsAWS.sessionToken = ""; // ? + + OpenVDS vdsHandle = OpenVDS.open(optionsAWS); + VolumeDataAccessManager accessManager = vdsHandle.getAccessManager(); + + int channel = 0; + VolumeDataLayout layout = vdsHandle.getLayout(); + VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( + DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND + 0, // lod + channel, // channel + 20, // max pages + VolumeDataPageAccessor.AccessMode.ReadOnly.getCode()); // access mode + + + + // compares block data + int[] pitchInput = new int[VolumeDataLayout.Dimensionality_Max]; + + long chunkCount = pageAccessor.getChunkCount(); + for (long chunk = 0 ; chunk < chunkCount ; ++chunk) { + VolumeDataPage inputPage = pageAccessor.readPage(chunk); + VolumeDataPage page = pageAccessor.readPage(chunk); + float[] dataIn = inputPage.readFloatBuffer(pitchInput); + + ErrorInfo dwnErrorInfo = accessManager.getCurrentDownloadErrorInfo(); + if (dwnErrorInfo.getErrorCode() != 0) { + System.out.println("Download error code : " + dwnErrorInfo.getErrorCode()); + if (dwnErrorInfo.getErrorMessage() != null) { + System.out.println("Download error message : " + dwnErrorInfo.getErrorMessage()); + } else { + System.out.println("No error message"); + } + } + } + } + + + /** + * Convert 3D index to 2D index + */ + private static int[] convertTo2DLocalIndex(int[] localIndex, int fixedCoordIndex) { + int[] local2DIndex = new int[2]; + if (fixedCoordIndex == 0) { + local2DIndex[0] = localIndex[1]; + local2DIndex[1] = localIndex[2]; + } + else if (fixedCoordIndex == 1) { + local2DIndex[0] = localIndex[0]; + local2DIndex[1] = localIndex[2]; + } + else if (fixedCoordIndex == 2) { + local2DIndex[0] = localIndex[0]; + local2DIndex[1] = localIndex[1]; + } + return local2DIndex; + } + + /** + * Convert 2D index to 3D index + * @param voxelIndex + * @param fixedCoordIndex + * @param fixedCoordValue + */ + private static int[] convertTo3DCoord(int[] voxelIndex, int fixedCoordIndex, int fixedCoordValue) { + int[] voxelPos = new int[3]; + voxelPos[fixedCoordIndex] = fixedCoordValue; + if (fixedCoordIndex == 0) { + voxelPos[1] = voxelIndex[0]; + voxelPos[2] = voxelIndex[1]; + } + else if (fixedCoordIndex == 1) { + voxelPos[0] = voxelIndex[0]; + voxelPos[2] = voxelIndex[1]; + } + else if (fixedCoordIndex == 2) { + voxelPos[0] = voxelIndex[0]; + voxelPos[1] = voxelIndex[1]; + } + return voxelPos; + } + + private static double distance2D(double x1, double y1, double x2, double y2) { + double diffX = x2 - x1; + double diffY = y2 - y1; + return Math.sqrt((diffX * diffX) + (diffY * diffY)); + } + + private static double distance3D(double x1, double y1, double z1, double x2, double y2, double z2) { + double diffX = x2 - x1; + double diffY = y2 - y1; + double diffZ = z2 - z1; + return Math.sqrt((diffX * diffX) + (diffY * diffY) + (diffZ * diffZ)); + } + + static float[] getScaleOffsetForFormat(float min, float max, boolean useNoValue, VolumeDataChannelDescriptor.Format format) { + float res[] = new float[] {1f, 0f}; + float noValueCmp = useNoValue ? 1f : 0f; + switch (format) { + case FORMAT_U8: + res[0] = 1.f / (255.f - noValueCmp) * (max - min); + res[1] = min; + break; + case FORMAT_U16: + res[0] = 1.f / (65535.f - noValueCmp) * (max - min); + res[1] = min; + break; + case FORMAT_R32: + case FORMAT_U32: + case FORMAT_R64: + case FORMAT_U64: + case FORMAT_1BIT: + case FORMAT_ANY: + res[0] = 1.0f; + res[1] = 0.0f; + } + return res; + } + +} diff --git a/java/java/src/org/opengroup/openvds/ErrorInfo.java b/java/java/src/org/opengroup/openvds/ErrorInfo.java index 4dd9f017..fc9b81b7 100644 --- a/java/java/src/org/opengroup/openvds/ErrorInfo.java +++ b/java/java/src/org/opengroup/openvds/ErrorInfo.java @@ -5,13 +5,12 @@ package org.opengroup.openvds; */ public class ErrorInfo { - private int errorCode; + private int errorCode = -1; private String errorMessage; private String objectID; - public ErrorInfo() { + public ErrorInfo() { } - } public ErrorInfo(int code, String msg, String id) { errorCode = code; errorMessage = msg; diff --git a/java/java/test/org/opengroup/openvds/PageAccessorByteTest.java b/java/java/test/org/opengroup/openvds/PageAccessorByteTest.java new file mode 100644 index 00000000..4e84d0a0 --- /dev/null +++ b/java/java/test/org/opengroup/openvds/PageAccessorByteTest.java @@ -0,0 +1,335 @@ +/* + * Copyright 2021 The Open Group + * Copyright 2021 INT, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.openvds; + +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.io.File; + +import static org.testng.Assert.fail; + +public class PageAccessorByteTest { + + private static String TEMP_FILE_NAME_VOL_INDEX = "volIndexer"; + private static String TEMP_FILE_NAME_COPY = "vdsCopy"; + + public String url; + public VolumeDataLayoutDescriptor ld; + public VolumeDataAxisDescriptor[] vda; + public VolumeDataChannelDescriptor[] vdc; + public MetadataReadAccess md; + public MemoryVdsGenerator vds; + public MetadataContainer metadataContainer; + + private String tempVolIndexerFileName; + private String tempVdsCopyFileName; + + @BeforeClass + public void init() { + vds = new MemoryVdsGenerator(200, 200, 200, VolumeDataChannelDescriptor.Format.FORMAT_U8); + url = "inmemory://create_test"; + VolumeDataLayout volumeDataLayout = vds.getLayout(); + + int nbChannel = volumeDataLayout.getChannelCount(); + VolumeDataAccessManager accessManager = vds.getAccessManager(); + + for (VolumeDataLayoutDescriptor.LODLevels l : VolumeDataLayoutDescriptor.LODLevels.values()) { + for (int channel = 0; channel < nbChannel; channel++) { + for (DimensionsND dimGroup : DimensionsND.values()) { + VDSProduceStatus vdsProduceStatus = accessManager.getVDSProduceStatus(dimGroup, l.ordinal(), channel); + } + } + } + + vda = new VolumeDataAxisDescriptor[] { + volumeDataLayout.getAxisDescriptor(0), + volumeDataLayout.getAxisDescriptor(1), + volumeDataLayout.getAxisDescriptor(2)}; + vdc = new VolumeDataChannelDescriptor[] {volumeDataLayout.getChannelDescriptor(0)}; + + md = volumeDataLayout; + ld = volumeDataLayout.getLayoutDescriptor(); + + metadataContainer = new MetadataContainer(); + + long ms = System.currentTimeMillis(); + tempVolIndexerFileName = TEMP_FILE_NAME_VOL_INDEX + "_" + ms + ".vds"; + tempVdsCopyFileName = TEMP_FILE_NAME_COPY + "_" + ms + ".vds"; + } + + @AfterClass + public void cleanFiles() { + String tempDir = System.getProperty("java.io.tmpdir"); + String fileVolIndexPath = tempDir + File.separator + tempVolIndexerFileName; + File fileVolIndex = new File(fileVolIndexPath); + if (fileVolIndex.exists()) { + //fileVolIndex.delete(); + } + + String fileCopyPath = tempDir + File.separator + tempVdsCopyFileName; + File fileCopy = new File(fileCopyPath); + if (fileCopy.exists()) { + //fileCopy.delete(); + } + } + + @Test + public void testVolumeIndexerCreationDeletion() { + try { + // create file in tmp dir + String tmpDir = System.getProperty("java.io.tmpdir"); + String volIndexPath = tmpDir + File.separator + tempVolIndexerFileName; + VDSFileOpenOptions options = new VDSFileOpenOptions(volIndexPath); + VdsHandle vdsTest = OpenVDS.create(options, ld, + vda, + vdc, metadataContainer); + VolumeDataAccessManager accessManager = vdsTest.getAccessManager(); + //ASSERT_TRUE(accessManager); + + int channel = 0; + VolumeDataLayout layout = vdsTest.getLayout(); + VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( + DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND + 0, // lod + channel, // channel + 100, // max pages + VolumeDataPageAccessor.AccessMode.Create.getCode()); // access mode + + VolumeDataPage page = pageAccessor.createPage(0); + VolumeIndexer3D outputIndexer = new VolumeIndexer3D(page, 0, 0, DimensionsND.DIMENSIONS_012.ordinal(), layout); + outputIndexer.finalize(); + + vdsTest.close(); + } + catch (java.io.IOException e) { + System.out.println(e.getMessage()); + fail(); + } + } + + @Test + public void testCopyPageAccessor() { + try { + String tmpDir = System.getProperty("java.io.tmpdir"); + String vdsPath = tmpDir + File.separator + tempVdsCopyFileName; + VDSFileOpenOptions options = new VDSFileOpenOptions(vdsPath); + + // copy information from input VDS + VolumeDataLayoutDescriptor cpLd = new VolumeDataLayoutDescriptor( + ld.getBrickSize(), + ld.getNegativeMargin(), + ld.getPositiveMargin(), + ld.getBrickSizeMultiplier2D(), + ld.getLODLevels(), + ld.isCreate2DLODs(), + ld.isForceFullResolutionDimension(), + ld.getFullResolutionDimension() + ); + + VolumeDataAxisDescriptor[] cpVda = new VolumeDataAxisDescriptor[3]; + for (int i = 0 ; i < 3 ; ++i) { + cpVda[i] = new VolumeDataAxisDescriptor(vda[i].getNumSamples(), + vda[i].getName(), vda[i].getUnit(), + vda[i].getCoordinateMin(), vda[i].getCoordinateMax()); + } + + VolumeDataChannelDescriptor[] cpVdc = new VolumeDataChannelDescriptor[1]; + cpVdc[0] = new VolumeDataChannelDescriptor( + VolumeDataChannelDescriptor.Format.fromCode(vdc[0].getFormat()), + VolumeDataChannelDescriptor.Components.fromCode(vdc[0].getComponents()), + vdc[0].getName(), + vdc[0].getUnit(), + vdc[0].getValueRangeMin(), vdc[0].getValueRangeMax(), + VolumeDataMapping.fromCode(vdc[0].getMapping()), + vdc[0].getMappedValueCount(), + vdc[0].isDiscrete(), + vdc[0].isRenderable(), + vdc[0].isAllowLossyCompression(), + vdc[0].isUseZipForLosslessCompression(), + vdc[0].isUseNoValue(), + vdc[0].getNoValue(), + vdc[0].getIntegerScale(), + vdc[0].getIntegerOffset() + ); + MetadataContainer cpMetadataContainer = new MetadataContainer(); + VdsHandle vdsCopy = OpenVDS.create(options, cpLd, + cpVda, + cpVdc, + cpMetadataContainer); + + VolumeDataAccessManager accessManager = vdsCopy.getAccessManager(); + //ASSERT_TRUE(accessManager); + + int channel = 0; + VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( + DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND + 0, // lod + channel, // channel + 100, // max pages + VolumeDataPageAccessor.AccessMode.Create.getCode()); // access mode + + // get input manager + VolumeDataAccessManager inputAM = vds.getAccessManager(); + VolumeDataPageAccessor pageAccessorInput = inputAM.createVolumeDataPageAccessor( + DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND + 0, // lod + channel, // channel + 100, // max pages + VolumeDataPageAccessor.AccessMode.ReadOnly.getCode()); // access mode + + // copy file + int[] pitch = new int[VolumeDataLayout.Dimensionality_Max]; + long chunkCount = pageAccessorInput.getChunkCount(); + for (long chunk = 0 ; chunk < chunkCount ; ++chunk) { + VolumeDataPage inputPage = pageAccessorInput.readPage(chunk); + VolumeDataPage page = pageAccessor.createPage(chunk); + byte[] data = inputPage.readByteBuffer(pitch); + page.writeByteBuffer(data, pitch); + + inputPage.pageRelease(); + page.pageRelease(); + } + pageAccessor.commit(); + accessManager.flushUploadQueue(); + accessManager.destroyVolumeDataPageAccessor(pageAccessor); + + vdsCopy.close(); + } + catch (java.io.IOException e) { + System.out.println(e.getMessage()); + fail(); + } + } + + /** + * Will test that copied file is the same as the input + * Name of method is the same + Suffix so that it's executed after the copy test + */ + @Test + public void testCopyPageAccessorValidation() { + try { + String tmpDir = System.getProperty("java.io.tmpdir"); + String vdsPath = tmpDir + File.separator + tempVdsCopyFileName; + VDSFileOpenOptions options = new VDSFileOpenOptions(vdsPath); + VdsHandle vdsCopy = OpenVDS.open(options); + VolumeDataAccessManager accessManager = vdsCopy.getAccessManager(); + + int channel = 0; + VolumeDataLayout layout = vdsCopy.getLayout(); + VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( + DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND + 0, // lod + channel, // channel + 20, // max pages + VolumeDataPageAccessor.AccessMode.ReadOnly.getCode()); // access mode + + // get input manager + VolumeDataAccessManager inputAM = vds.getAccessManager(); + VolumeDataPageAccessor pageAccessorInput = inputAM.createVolumeDataPageAccessor( + DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND + 0, // lod + channel, // channel + 20, // max pages + VolumeDataPageAccessor.AccessMode.ReadOnly.getCode()); // access mode + + // compares block data + int[] pitchInput = new int[VolumeDataLayout.Dimensionality_Max]; + int[] pitchOutput = new int[VolumeDataLayout.Dimensionality_Max]; + + long chunkCount = pageAccessorInput.getChunkCount(); + for (long chunk = 0 ; chunk < chunkCount ; ++chunk) { + VolumeDataPage inputPage = pageAccessorInput.readPage(chunk); + VolumeDataPage page = pageAccessor.readPage(chunk); + byte[] dataIn = inputPage.readByteBuffer(pitchInput); + byte[] dataOut = page.readByteBuffer(pitchOutput); + + inputPage.pageRelease(); + page.pageRelease(); + + Assert.assertEquals(pitchInput, pitchOutput); + Assert.assertEquals(dataIn, dataOut); + } + + accessManager.destroyVolumeDataPageAccessor(pageAccessor); + + vdsCopy.close(); + } + catch (java.io.IOException e) { + System.out.println(e.getMessage()); + fail(); + } + } + + /** + * Will test that copied file has the same layout than original (test that 3D positions are in the same chunk index) + * Name of method is the same + Suffix so that it's executed after the copy test + */ + @Test + public void testCopyPageAccessorValidationChunkIndex() { + try { + String tmpDir = System.getProperty("java.io.tmpdir"); + String vdsPath = tmpDir + File.separator + tempVdsCopyFileName; + VDSFileOpenOptions options = new VDSFileOpenOptions(vdsPath); + VdsHandle vdsCopy = OpenVDS.open(options); + VolumeDataAccessManager accessManager = vdsCopy.getAccessManager(); + + int channel = 0; + VolumeDataLayout layout = vdsCopy.getLayout(); + VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( + DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND + 0, // lod + channel, // channel + 20, // max pages + VolumeDataPageAccessor.AccessMode.ReadOnly.getCode()); // access mode + + // compares block data + int[] chunkMin = new int[VolumeDataLayout.Dimensionality_Max]; + int[] chunkMax = new int[VolumeDataLayout.Dimensionality_Max]; + int[] chunkMaxPos = new int[VolumeDataLayout.Dimensionality_Max]; + + long chunkCount = pageAccessor.getChunkCount(); + for (long chunk = 0 ; chunk < chunkCount ; ++chunk) { + VolumeDataPage inputPage = pageAccessor.readPage(chunk); + + // check that chunk index matches current index + inputPage.getMinMaxExcludingMargin(chunkMin, chunkMax); + for (int i = 0 ; i < VolumeDataLayout.Dimensionality_Max ; ++i) { + chunkMaxPos[i] = chunkMax[i] != 0 ? chunkMax[i] - 1 : chunkMax[i]; + } + long idxChMin = pageAccessor.getChunkIndex(chunkMin); + long idxChMax = pageAccessor.getChunkIndex(chunkMaxPos); + + Assert.assertEquals(chunk, idxChMin); + Assert.assertEquals(idxChMin, idxChMax); + + inputPage.pageRelease(); + } + + accessManager.destroyVolumeDataPageAccessor(pageAccessor); + + vdsCopy.close(); + } + catch (java.io.IOException e) { + System.out.println(e.getMessage()); + fail(); + } + } +} -- GitLab From 066914c44471e99c0da6e02d5439c546c04089f9 Mon Sep 17 00:00:00 2001 From: Julien Lacoste Date: Thu, 9 Sep 2021 17:18:35 +0200 Subject: [PATCH 09/25] Rename ErrorInfo class --- java/CMakeLists.txt | 2 +- ...rrorInfo.java => ConnectionErrorInfo.java} | 6 +++--- .../openvds/VolumeDataAccessManager.java | 20 +++++++++---------- 3 files changed, 14 insertions(+), 14 deletions(-) rename java/java/src/org/opengroup/openvds/{ErrorInfo.java => ConnectionErrorInfo.java} (84%) diff --git a/java/CMakeLists.txt b/java/CMakeLists.txt index 1d7e322f..7f954fcb 100644 --- a/java/CMakeLists.txt +++ b/java/CMakeLists.txt @@ -18,9 +18,9 @@ set(JAVA_SOURCE_FILES java/src/org/opengroup/openvds/BufferUtils.java java/src/org/opengroup/openvds/Cleaner.java java/src/org/opengroup/openvds/CompressionMethod.java + java/src/org/opengroup/openvds/ConnectionErrorInfo.java java/src/org/opengroup/openvds/DimensionsND.java java/src/org/opengroup/openvds/experimental/VariousJavaTests.java - java/src/org/opengroup/openvds/ErrorInfo.java java/src/org/opengroup/openvds/GoogleOpenOptions.java java/src/org/opengroup/openvds/IndexRegionFloat.java java/src/org/opengroup/openvds/IndexRegionLong.java diff --git a/java/java/src/org/opengroup/openvds/ErrorInfo.java b/java/java/src/org/opengroup/openvds/ConnectionErrorInfo.java similarity index 84% rename from java/java/src/org/opengroup/openvds/ErrorInfo.java rename to java/java/src/org/opengroup/openvds/ConnectionErrorInfo.java index fc9b81b7..c1584d91 100644 --- a/java/java/src/org/opengroup/openvds/ErrorInfo.java +++ b/java/java/src/org/opengroup/openvds/ConnectionErrorInfo.java @@ -3,15 +3,15 @@ package org.opengroup.openvds; /** * Simple class holding information on Upload/Download error */ -public class ErrorInfo { +public class ConnectionErrorInfo { private int errorCode = -1; private String errorMessage; private String objectID; - public ErrorInfo() { } + public ConnectionErrorInfo() { } - public ErrorInfo(int code, String msg, String id) { + public ConnectionErrorInfo(int code, String msg, String id) { errorCode = code; errorMessage = msg; objectID = id; diff --git a/java/java/src/org/opengroup/openvds/VolumeDataAccessManager.java b/java/java/src/org/opengroup/openvds/VolumeDataAccessManager.java index 86d24b8c..506ac63e 100644 --- a/java/java/src/org/opengroup/openvds/VolumeDataAccessManager.java +++ b/java/java/src/org/opengroup/openvds/VolumeDataAccessManager.java @@ -147,9 +147,9 @@ public class VolumeDataAccessManager extends JniPointerWithoutDeletion { private static native int cpGetCurrentUploadErrorCode(long handle); - private static native void cpGetCurrentUploadErrorInfo(long handle, ErrorInfo error); + private static native void cpGetCurrentUploadErrorInfo(long handle, ConnectionErrorInfo error); - private static native void cpGetCurrentDownloadErrorInfo(long handle, ErrorInfo error); + private static native void cpGetCurrentDownloadErrorInfo(long handle, ConnectionErrorInfo error); public VolumeDataAccessManager(long handle) { super(handle); @@ -779,18 +779,18 @@ public class VolumeDataAccessManager extends JniPointerWithoutDeletion { /** * @return current upload error information (code and message in a simple class) */ - public ErrorInfo getCurrentUploadErrorInfo() { - ErrorInfo errorInfo = new ErrorInfo(); - cpGetCurrentUploadErrorInfo(_handle, errorInfo); - return errorInfo; + public ConnectionErrorInfo getCurrentUploadErrorInfo() { + ConnectionErrorInfo connectionErrorInfo = new ConnectionErrorInfo(); + cpGetCurrentUploadErrorInfo(_handle, connectionErrorInfo); + return connectionErrorInfo; } /** * @return current upload error information (code and message in a simple class) */ - public ErrorInfo getCurrentDownloadErrorInfo() { - ErrorInfo errorInfo = new ErrorInfo(); - cpGetCurrentDownloadErrorInfo(_handle, errorInfo); - return errorInfo; + public ConnectionErrorInfo getCurrentDownloadErrorInfo() { + ConnectionErrorInfo connectionErrorInfo = new ConnectionErrorInfo(); + cpGetCurrentDownloadErrorInfo(_handle, connectionErrorInfo); + return connectionErrorInfo; } } -- GitLab From a33c6c1f65735625052b08b45938a30934e397c2 Mon Sep 17 00:00:00 2001 From: Julien Lacoste Date: Thu, 9 Sep 2021 17:22:36 +0200 Subject: [PATCH 10/25] Remove temp test --- .../openvds/PageAccessorByteTest.java | 335 ------------------ 1 file changed, 335 deletions(-) delete mode 100644 java/java/test/org/opengroup/openvds/PageAccessorByteTest.java diff --git a/java/java/test/org/opengroup/openvds/PageAccessorByteTest.java b/java/java/test/org/opengroup/openvds/PageAccessorByteTest.java deleted file mode 100644 index 4e84d0a0..00000000 --- a/java/java/test/org/opengroup/openvds/PageAccessorByteTest.java +++ /dev/null @@ -1,335 +0,0 @@ -/* - * Copyright 2021 The Open Group - * Copyright 2021 INT, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opengroup.openvds; - -import org.testng.Assert; -import org.testng.annotations.AfterClass; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.Test; - -import java.io.File; - -import static org.testng.Assert.fail; - -public class PageAccessorByteTest { - - private static String TEMP_FILE_NAME_VOL_INDEX = "volIndexer"; - private static String TEMP_FILE_NAME_COPY = "vdsCopy"; - - public String url; - public VolumeDataLayoutDescriptor ld; - public VolumeDataAxisDescriptor[] vda; - public VolumeDataChannelDescriptor[] vdc; - public MetadataReadAccess md; - public MemoryVdsGenerator vds; - public MetadataContainer metadataContainer; - - private String tempVolIndexerFileName; - private String tempVdsCopyFileName; - - @BeforeClass - public void init() { - vds = new MemoryVdsGenerator(200, 200, 200, VolumeDataChannelDescriptor.Format.FORMAT_U8); - url = "inmemory://create_test"; - VolumeDataLayout volumeDataLayout = vds.getLayout(); - - int nbChannel = volumeDataLayout.getChannelCount(); - VolumeDataAccessManager accessManager = vds.getAccessManager(); - - for (VolumeDataLayoutDescriptor.LODLevels l : VolumeDataLayoutDescriptor.LODLevels.values()) { - for (int channel = 0; channel < nbChannel; channel++) { - for (DimensionsND dimGroup : DimensionsND.values()) { - VDSProduceStatus vdsProduceStatus = accessManager.getVDSProduceStatus(dimGroup, l.ordinal(), channel); - } - } - } - - vda = new VolumeDataAxisDescriptor[] { - volumeDataLayout.getAxisDescriptor(0), - volumeDataLayout.getAxisDescriptor(1), - volumeDataLayout.getAxisDescriptor(2)}; - vdc = new VolumeDataChannelDescriptor[] {volumeDataLayout.getChannelDescriptor(0)}; - - md = volumeDataLayout; - ld = volumeDataLayout.getLayoutDescriptor(); - - metadataContainer = new MetadataContainer(); - - long ms = System.currentTimeMillis(); - tempVolIndexerFileName = TEMP_FILE_NAME_VOL_INDEX + "_" + ms + ".vds"; - tempVdsCopyFileName = TEMP_FILE_NAME_COPY + "_" + ms + ".vds"; - } - - @AfterClass - public void cleanFiles() { - String tempDir = System.getProperty("java.io.tmpdir"); - String fileVolIndexPath = tempDir + File.separator + tempVolIndexerFileName; - File fileVolIndex = new File(fileVolIndexPath); - if (fileVolIndex.exists()) { - //fileVolIndex.delete(); - } - - String fileCopyPath = tempDir + File.separator + tempVdsCopyFileName; - File fileCopy = new File(fileCopyPath); - if (fileCopy.exists()) { - //fileCopy.delete(); - } - } - - @Test - public void testVolumeIndexerCreationDeletion() { - try { - // create file in tmp dir - String tmpDir = System.getProperty("java.io.tmpdir"); - String volIndexPath = tmpDir + File.separator + tempVolIndexerFileName; - VDSFileOpenOptions options = new VDSFileOpenOptions(volIndexPath); - VdsHandle vdsTest = OpenVDS.create(options, ld, - vda, - vdc, metadataContainer); - VolumeDataAccessManager accessManager = vdsTest.getAccessManager(); - //ASSERT_TRUE(accessManager); - - int channel = 0; - VolumeDataLayout layout = vdsTest.getLayout(); - VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( - DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND - 0, // lod - channel, // channel - 100, // max pages - VolumeDataPageAccessor.AccessMode.Create.getCode()); // access mode - - VolumeDataPage page = pageAccessor.createPage(0); - VolumeIndexer3D outputIndexer = new VolumeIndexer3D(page, 0, 0, DimensionsND.DIMENSIONS_012.ordinal(), layout); - outputIndexer.finalize(); - - vdsTest.close(); - } - catch (java.io.IOException e) { - System.out.println(e.getMessage()); - fail(); - } - } - - @Test - public void testCopyPageAccessor() { - try { - String tmpDir = System.getProperty("java.io.tmpdir"); - String vdsPath = tmpDir + File.separator + tempVdsCopyFileName; - VDSFileOpenOptions options = new VDSFileOpenOptions(vdsPath); - - // copy information from input VDS - VolumeDataLayoutDescriptor cpLd = new VolumeDataLayoutDescriptor( - ld.getBrickSize(), - ld.getNegativeMargin(), - ld.getPositiveMargin(), - ld.getBrickSizeMultiplier2D(), - ld.getLODLevels(), - ld.isCreate2DLODs(), - ld.isForceFullResolutionDimension(), - ld.getFullResolutionDimension() - ); - - VolumeDataAxisDescriptor[] cpVda = new VolumeDataAxisDescriptor[3]; - for (int i = 0 ; i < 3 ; ++i) { - cpVda[i] = new VolumeDataAxisDescriptor(vda[i].getNumSamples(), - vda[i].getName(), vda[i].getUnit(), - vda[i].getCoordinateMin(), vda[i].getCoordinateMax()); - } - - VolumeDataChannelDescriptor[] cpVdc = new VolumeDataChannelDescriptor[1]; - cpVdc[0] = new VolumeDataChannelDescriptor( - VolumeDataChannelDescriptor.Format.fromCode(vdc[0].getFormat()), - VolumeDataChannelDescriptor.Components.fromCode(vdc[0].getComponents()), - vdc[0].getName(), - vdc[0].getUnit(), - vdc[0].getValueRangeMin(), vdc[0].getValueRangeMax(), - VolumeDataMapping.fromCode(vdc[0].getMapping()), - vdc[0].getMappedValueCount(), - vdc[0].isDiscrete(), - vdc[0].isRenderable(), - vdc[0].isAllowLossyCompression(), - vdc[0].isUseZipForLosslessCompression(), - vdc[0].isUseNoValue(), - vdc[0].getNoValue(), - vdc[0].getIntegerScale(), - vdc[0].getIntegerOffset() - ); - MetadataContainer cpMetadataContainer = new MetadataContainer(); - VdsHandle vdsCopy = OpenVDS.create(options, cpLd, - cpVda, - cpVdc, - cpMetadataContainer); - - VolumeDataAccessManager accessManager = vdsCopy.getAccessManager(); - //ASSERT_TRUE(accessManager); - - int channel = 0; - VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( - DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND - 0, // lod - channel, // channel - 100, // max pages - VolumeDataPageAccessor.AccessMode.Create.getCode()); // access mode - - // get input manager - VolumeDataAccessManager inputAM = vds.getAccessManager(); - VolumeDataPageAccessor pageAccessorInput = inputAM.createVolumeDataPageAccessor( - DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND - 0, // lod - channel, // channel - 100, // max pages - VolumeDataPageAccessor.AccessMode.ReadOnly.getCode()); // access mode - - // copy file - int[] pitch = new int[VolumeDataLayout.Dimensionality_Max]; - long chunkCount = pageAccessorInput.getChunkCount(); - for (long chunk = 0 ; chunk < chunkCount ; ++chunk) { - VolumeDataPage inputPage = pageAccessorInput.readPage(chunk); - VolumeDataPage page = pageAccessor.createPage(chunk); - byte[] data = inputPage.readByteBuffer(pitch); - page.writeByteBuffer(data, pitch); - - inputPage.pageRelease(); - page.pageRelease(); - } - pageAccessor.commit(); - accessManager.flushUploadQueue(); - accessManager.destroyVolumeDataPageAccessor(pageAccessor); - - vdsCopy.close(); - } - catch (java.io.IOException e) { - System.out.println(e.getMessage()); - fail(); - } - } - - /** - * Will test that copied file is the same as the input - * Name of method is the same + Suffix so that it's executed after the copy test - */ - @Test - public void testCopyPageAccessorValidation() { - try { - String tmpDir = System.getProperty("java.io.tmpdir"); - String vdsPath = tmpDir + File.separator + tempVdsCopyFileName; - VDSFileOpenOptions options = new VDSFileOpenOptions(vdsPath); - VdsHandle vdsCopy = OpenVDS.open(options); - VolumeDataAccessManager accessManager = vdsCopy.getAccessManager(); - - int channel = 0; - VolumeDataLayout layout = vdsCopy.getLayout(); - VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( - DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND - 0, // lod - channel, // channel - 20, // max pages - VolumeDataPageAccessor.AccessMode.ReadOnly.getCode()); // access mode - - // get input manager - VolumeDataAccessManager inputAM = vds.getAccessManager(); - VolumeDataPageAccessor pageAccessorInput = inputAM.createVolumeDataPageAccessor( - DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND - 0, // lod - channel, // channel - 20, // max pages - VolumeDataPageAccessor.AccessMode.ReadOnly.getCode()); // access mode - - // compares block data - int[] pitchInput = new int[VolumeDataLayout.Dimensionality_Max]; - int[] pitchOutput = new int[VolumeDataLayout.Dimensionality_Max]; - - long chunkCount = pageAccessorInput.getChunkCount(); - for (long chunk = 0 ; chunk < chunkCount ; ++chunk) { - VolumeDataPage inputPage = pageAccessorInput.readPage(chunk); - VolumeDataPage page = pageAccessor.readPage(chunk); - byte[] dataIn = inputPage.readByteBuffer(pitchInput); - byte[] dataOut = page.readByteBuffer(pitchOutput); - - inputPage.pageRelease(); - page.pageRelease(); - - Assert.assertEquals(pitchInput, pitchOutput); - Assert.assertEquals(dataIn, dataOut); - } - - accessManager.destroyVolumeDataPageAccessor(pageAccessor); - - vdsCopy.close(); - } - catch (java.io.IOException e) { - System.out.println(e.getMessage()); - fail(); - } - } - - /** - * Will test that copied file has the same layout than original (test that 3D positions are in the same chunk index) - * Name of method is the same + Suffix so that it's executed after the copy test - */ - @Test - public void testCopyPageAccessorValidationChunkIndex() { - try { - String tmpDir = System.getProperty("java.io.tmpdir"); - String vdsPath = tmpDir + File.separator + tempVdsCopyFileName; - VDSFileOpenOptions options = new VDSFileOpenOptions(vdsPath); - VdsHandle vdsCopy = OpenVDS.open(options); - VolumeDataAccessManager accessManager = vdsCopy.getAccessManager(); - - int channel = 0; - VolumeDataLayout layout = vdsCopy.getLayout(); - VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( - DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND - 0, // lod - channel, // channel - 20, // max pages - VolumeDataPageAccessor.AccessMode.ReadOnly.getCode()); // access mode - - // compares block data - int[] chunkMin = new int[VolumeDataLayout.Dimensionality_Max]; - int[] chunkMax = new int[VolumeDataLayout.Dimensionality_Max]; - int[] chunkMaxPos = new int[VolumeDataLayout.Dimensionality_Max]; - - long chunkCount = pageAccessor.getChunkCount(); - for (long chunk = 0 ; chunk < chunkCount ; ++chunk) { - VolumeDataPage inputPage = pageAccessor.readPage(chunk); - - // check that chunk index matches current index - inputPage.getMinMaxExcludingMargin(chunkMin, chunkMax); - for (int i = 0 ; i < VolumeDataLayout.Dimensionality_Max ; ++i) { - chunkMaxPos[i] = chunkMax[i] != 0 ? chunkMax[i] - 1 : chunkMax[i]; - } - long idxChMin = pageAccessor.getChunkIndex(chunkMin); - long idxChMax = pageAccessor.getChunkIndex(chunkMaxPos); - - Assert.assertEquals(chunk, idxChMin); - Assert.assertEquals(idxChMin, idxChMax); - - inputPage.pageRelease(); - } - - accessManager.destroyVolumeDataPageAccessor(pageAccessor); - - vdsCopy.close(); - } - catch (java.io.IOException e) { - System.out.println(e.getMessage()); - fail(); - } - } -} -- GitLab From 4dac7fc8615a937951b38929cb49f99af2a320b8 Mon Sep 17 00:00:00 2001 From: Julien Lacoste Date: Mon, 20 Sep 2021 10:02:31 +0200 Subject: [PATCH 11/25] Refactor class name --- java/java/demo/CreateVDSCloud.java | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/java/java/demo/CreateVDSCloud.java b/java/java/demo/CreateVDSCloud.java index d31b31e4..32a2faac 100644 --- a/java/java/demo/CreateVDSCloud.java +++ b/java/java/demo/CreateVDSCloud.java @@ -166,13 +166,13 @@ public class CreateVDSCloud { channelDescriptors.add(channelDescriptor); // options AWS - String pBucket = "miniovaap"; // the bucket of the VDS + String pBucket = ""; // the bucket of the VDS String pKey = name; // the key prefix of the VDS ? String pRegion = "eu-west-3"; // the region of the bucket of the VDS - String pEndpointOverride = "https://webvr.int.com/"; // This parameter allows to override the endpoint url + String pEndpointOverride = ""; // This parameter allows to override the endpoint url AWSOpenOptions optionsAWS = new AWSOpenOptions(pBucket, pKey, pRegion, pEndpointOverride); - optionsAWS.accessKeyId = "minioadmin"; // ? - optionsAWS.secretKey = "minioadmin"; + optionsAWS.accessKeyId = ""; // ? + optionsAWS.secretKey = ""; optionsAWS.sessionToken = ""; // ? // file options @@ -306,7 +306,7 @@ public class CreateVDSCloud { int errorCount = accessManager.uploadErrorCount(); if (errorCount > 0) { - ErrorInfo upErrInfo = accessManager.getCurrentUploadErrorInfo(); + ConnectionErrorInfo upErrInfo = accessManager.getCurrentUploadErrorInfo(); System.out.println("Upload error code : " + upErrInfo.getErrorCode()); if (upErrInfo.getErrorMessage() != null) { System.out.println("Upload error message : " + upErrInfo.getErrorMessage()); @@ -481,11 +481,11 @@ public class CreateVDSCloud { VolumeDataPage page = pageAccessor.readPage(chunk); float[] dataIn = inputPage.readFloatBuffer(pitchInput); - ErrorInfo dwnErrorInfo = accessManager.getCurrentDownloadErrorInfo(); - if (dwnErrorInfo.getErrorCode() != 0) { - System.out.println("Download error code : " + dwnErrorInfo.getErrorCode()); - if (dwnErrorInfo.getErrorMessage() != null) { - System.out.println("Download error message : " + dwnErrorInfo.getErrorMessage()); + ConnectionErrorInfo dwnConnectionErrorInfo = accessManager.getCurrentDownloadErrorInfo(); + if (dwnConnectionErrorInfo.getErrorCode() != 0) { + System.out.println("Download error code : " + dwnConnectionErrorInfo.getErrorCode()); + if (dwnConnectionErrorInfo.getErrorMessage() != null) { + System.out.println("Download error message : " + dwnConnectionErrorInfo.getErrorMessage()); } else { System.out.println("No error message"); } -- GitLab From 993435f5a4c5880eb47c69a38e7fe8a428229ac9 Mon Sep 17 00:00:00 2001 From: Julien Lacoste Date: Tue, 24 Aug 2021 16:22:07 +0200 Subject: [PATCH 12/25] Java bindings : Add methods on VolumeDataAccessManager to get info on Upload/Download error --- java/CMakeLists.txt | 1 + java/cpp/src/VolumeDataAccessManager.cpp | 61 +++++++++++++++++++ .../src/org/opengroup/openvds/ErrorInfo.java | 44 +++++++++++++ .../openvds/VolumeDataAccessManager.java | 22 +++++++ 4 files changed, 128 insertions(+) create mode 100644 java/java/src/org/opengroup/openvds/ErrorInfo.java diff --git a/java/CMakeLists.txt b/java/CMakeLists.txt index ee7ca465..1d7e322f 100644 --- a/java/CMakeLists.txt +++ b/java/CMakeLists.txt @@ -20,6 +20,7 @@ set(JAVA_SOURCE_FILES java/src/org/opengroup/openvds/CompressionMethod.java java/src/org/opengroup/openvds/DimensionsND.java java/src/org/opengroup/openvds/experimental/VariousJavaTests.java + java/src/org/opengroup/openvds/ErrorInfo.java java/src/org/opengroup/openvds/GoogleOpenOptions.java java/src/org/opengroup/openvds/IndexRegionFloat.java java/src/org/opengroup/openvds/IndexRegionLong.java diff --git a/java/cpp/src/VolumeDataAccessManager.cpp b/java/cpp/src/VolumeDataAccessManager.cpp index 0968092d..c6f723af 100644 --- a/java/cpp/src/VolumeDataAccessManager.cpp +++ b/java/cpp/src/VolumeDataAccessManager.cpp @@ -583,6 +583,67 @@ JNIEXPORT jint JNICALL Java_org_opengroup_openvds_VolumeDataAccessManager_cpGetC return 0; } +/* + * Class: org_opengroup_openvds_VolumeDataAccessManager + * Method: cpGetCurrentUploadErrorInfo + * Signature: (JLorg/opengroup/openvds/ErrorInfo;)V + */ +JNIEXPORT void JNICALL Java_org_opengroup_openvds_VolumeDataAccessManager_cpGetCurrentUploadErrorInfo + (JNIEnv * env, jclass, jlong handle, jobject errorObj) +{ + try { + const char *pObjectID = nullptr; + const char *pErrorString = nullptr; + int32_t errorCode = 0; + + GetManager(handle)->GetCurrentUploadError(&pObjectID, &errorCode, &pErrorString); + + // Get the class of the given error object + jclass clazz = env->GetObjectClass(errorObj); + + // Get Field references + jfieldID paramCode = env->GetFieldID(clazz, "errorCode", "I"); + jfieldID paramMessage = env->GetFieldID(clazz, "errorMessage", "Ljava/lang/String;"); + jfieldID paramID = env->GetFieldID(clazz, "objectID", "Ljava/lang/String;"); + + // Set fields for object + jstring strMsg = NewJString(env, pErrorString); + jstring strID = NewJString(env, pObjectID); + env->SetIntField(errorObj, paramCode, errorCode); + env->SetObjectField(errorObj, paramMessage, strMsg); + env->SetObjectField(errorObj, paramID, strID); + } + CATCH_EXCEPTIONS_FOR_JAVA; +} + +/* + * Class: org_opengroup_openvds_VolumeDataAccessManager + * Method: cpGetCurrentDownloadErrorInfo + * Signature: (JLorg/opengroup/openvds/ErrorInfo;)V + */ +JNIEXPORT void JNICALL Java_org_opengroup_openvds_VolumeDataAccessManager_cpGetCurrentDownloadErrorInfo + (JNIEnv * env, jclass, jlong handle, jobject errorObj) +{ + try { + const char *pErrorString = nullptr; + int32_t errorCode = 0; + GetManager(handle)->GetCurrentDownloadError(&errorCode, &pErrorString); + + // Get the class of the given error object + jclass clazz = env->GetObjectClass(errorObj); + + // Get Field references + jfieldID paramCode = env->GetFieldID(clazz, "errorCode", "I"); + jfieldID paramMessage = env->GetFieldID(clazz, "errorMessage", "Ljava/lang/String;"); + + // Set fields for object + env->SetIntField(errorObj, paramCode, errorCode); + jstring strMsg = NewJString(env, pErrorString); + env->SetObjectField(errorObj, paramMessage, strMsg); + } + CATCH_EXCEPTIONS_FOR_JAVA; +} + #ifdef __cplusplus } #endif diff --git a/java/java/src/org/opengroup/openvds/ErrorInfo.java b/java/java/src/org/opengroup/openvds/ErrorInfo.java new file mode 100644 index 00000000..4dd9f017 --- /dev/null +++ b/java/java/src/org/opengroup/openvds/ErrorInfo.java @@ -0,0 +1,44 @@ +package org.opengroup.openvds; + +/** + * Simple class holding information on Upload/Download error + */ +public class ErrorInfo { + + private int errorCode; + private String errorMessage; + private String objectID; + + public ErrorInfo() { + + } + public ErrorInfo(int code, String msg, String id) { + errorCode = code; + errorMessage = msg; + objectID = id; + } + + public int getErrorCode() { + return errorCode; + } + + public void setErrorCode(int errorCode) { + this.errorCode = errorCode; + } + + public String getErrorMessage() { + return errorMessage; + } + + public void setErrorMessage(String errorMessage) { + this.errorMessage = errorMessage; + } + + public String getObjectID() { + return objectID; + } + + public void setObjectID(String objectID) { + this.objectID = objectID; + } +} diff --git a/java/java/src/org/opengroup/openvds/VolumeDataAccessManager.java b/java/java/src/org/opengroup/openvds/VolumeDataAccessManager.java index 2af400d2..85622597 100644 --- a/java/java/src/org/opengroup/openvds/VolumeDataAccessManager.java +++ b/java/java/src/org/opengroup/openvds/VolumeDataAccessManager.java @@ -147,6 +147,10 @@ public class VolumeDataAccessManager extends JniPointerWithoutDeletion { private static native int cpGetCurrentUploadErrorCode(long handle); + private static native void cpGetCurrentUploadErrorInfo(long handle, ErrorInfo error); + + private static native void cpGetCurrentDownloadErrorInfo(long handle, ErrorInfo error); + public VolumeDataAccessManager(long handle) { super(handle); } @@ -771,4 +775,22 @@ public class VolumeDataAccessManager extends JniPointerWithoutDeletion { public int getCurrentUploadErrorCode() { return cpGetCurrentUploadErrorCode(_handle); } + + /** + * @return current upload error information (code and message in a simple class) + */ + public ErrorInfo getCurrentUploadErrorInfo() { + ErrorInfo errorInfo = new ErrorInfo(); + cpGetCurrentUploadErrorInfo(_handle, errorInfo); + return errorInfo; + } + + /** + * @return current upload error information (code and message in a simple class) + */ + public ErrorInfo getCurrentDownloadErrorInfo() { + ErrorInfo errorInfo = new ErrorInfo(); + cpGetCurrentUploadErrorInfo(_handle, errorInfo); + return errorInfo; + } } -- GitLab From 9e119a0fc94e14339a247faa703ae7ea23226cc9 Mon Sep 17 00:00:00 2001 From: Julien Lacoste Date: Tue, 24 Aug 2021 16:28:16 +0200 Subject: [PATCH 13/25] Java bindings : Add methods on VolumeDataAccessManager to get info on Upload/Download error + fix wrong native call --- .../java/src/org/opengroup/openvds/VolumeDataAccessManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/java/src/org/opengroup/openvds/VolumeDataAccessManager.java b/java/java/src/org/opengroup/openvds/VolumeDataAccessManager.java index 85622597..86d24b8c 100644 --- a/java/java/src/org/opengroup/openvds/VolumeDataAccessManager.java +++ b/java/java/src/org/opengroup/openvds/VolumeDataAccessManager.java @@ -790,7 +790,7 @@ public class VolumeDataAccessManager extends JniPointerWithoutDeletion { */ public ErrorInfo getCurrentDownloadErrorInfo() { ErrorInfo errorInfo = new ErrorInfo(); - cpGetCurrentUploadErrorInfo(_handle, errorInfo); + cpGetCurrentDownloadErrorInfo(_handle, errorInfo); return errorInfo; } } -- GitLab From 66b1465df54c7dec51780d69924e5a75092de792 Mon Sep 17 00:00:00 2001 From: Julien Lacoste Date: Thu, 2 Sep 2021 12:17:34 +0200 Subject: [PATCH 14/25] bkp --- java/java/demo/CreateVDS.java | 2 +- java/java/demo/CreateVDSCloud.java | 578 ++++++++++++++++++ .../src/org/opengroup/openvds/ErrorInfo.java | 5 +- .../openvds/PageAccessorByteTest.java | 335 ++++++++++ 4 files changed, 916 insertions(+), 4 deletions(-) create mode 100644 java/java/demo/CreateVDSCloud.java create mode 100644 java/java/test/org/opengroup/openvds/PageAccessorByteTest.java diff --git a/java/java/demo/CreateVDS.java b/java/java/demo/CreateVDS.java index 0931bd12..964a84fa 100644 --- a/java/java/demo/CreateVDS.java +++ b/java/java/demo/CreateVDS.java @@ -157,7 +157,7 @@ public class CreateVDS { true, // is renderable false, // allow lossy compression false, // use zip for lossless compresion - true, // use no value + false, // use no value -999.25f, // no value scaleOffset[0], // integer scale scaleOffset[1]); // integer offset diff --git a/java/java/demo/CreateVDSCloud.java b/java/java/demo/CreateVDSCloud.java new file mode 100644 index 00000000..d31b31e4 --- /dev/null +++ b/java/java/demo/CreateVDSCloud.java @@ -0,0 +1,578 @@ +/* + * Copyright 2021 The Open Group + * Copyright 2021 INT, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.opengroup.openvds.*; + +import java.io.File; +import java.io.IOException; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; + +public class CreateVDSCloud { + + // usage : CreateVDSWithLOD n /path/to/file + // where n is the LOD level wanted + public static void main(String[] args) { + try { + if (!checkParams(args)) { + System.out.println("Bad params, usage : n /path/to/file dimX dimY dimZ"); + System.out.println("n is the LOD level wanted (0 <= lod <= 12), path must be a valid non existing file path"); + System.out.println("dims are cube dimension (must be > 0)"); + } + else { + Instant start = Instant.now(); + process(Integer.parseInt(args[0]), args[1], Integer.parseInt(args[2]), Integer.parseInt(args[3]), Integer.parseInt(args[4])); + Instant end = Instant.now(); + + Duration elapsed = Duration.between(start, end); + long hrs = elapsed.toHours(); + long min = elapsed.toMinutes() - (hrs * 60); + long s = (elapsed.toMillis() - (elapsed.toMinutes() * 60 * 1000)) / 1000; + System.out.println("Write VDS TIME : " + hrs + " hrs " + min + " min " + s + "s (" + elapsed.toMillis() + " ms)"); + + testRead(args[1]); + } + } + catch (Throwable t) { + System.out.println(); + t.printStackTrace(); + } + } + + private static boolean checkParams(String[] args) { + if (args == null || args.length != 5) { + return false; + } + try { + Integer lod = Integer.parseInt(args[0]); + if (lod < 0 || lod > 12) { + System.err.println("Invalid LOD value (must be between 0 and 12) !"); + return false; + } + } + catch (NumberFormatException nfe) { + return false; + } + String path = args[1]; + File file = new File(path); + try { + file.getCanonicalPath(); + } + catch (IOException e) { + System.err.println("VDS File path is invalid !"); + return false; + } + if (!path.endsWith(".vds")) { + System.err.println("VDS File path must have vds extension !"); + return false; + } + if (file.exists()) { + System.err.println("VDS File already exists !"); + return false; + } + try { + Integer dimX = Integer.parseInt(args[2]); + if (dimX < 0) { + System.err.println("Invalid x dimension !"); + return false; + } + Integer dimY = Integer.parseInt(args[3]); + if (dimX < 0) { + System.err.println("Invalid y dimension !"); + return false; + } + Integer dimZ = Integer.parseInt(args[4]); + if (dimX < 0) { + System.err.println("Invalid Z dimension !"); + return false; + } + } + catch (NumberFormatException nfe) { + return false; + } + return true; + } + + static void process(int lodParam, String name, int dimX, int dimY, int dimZ) throws Exception { + + int samplesX = dimZ; // time + int samplesY = dimY; // XL + int samplesZ = dimX; // IL + VolumeDataChannelDescriptor.Format format = VolumeDataChannelDescriptor.Format.FORMAT_R32; + + double sizeX = samplesX; + double sizeY = samplesY; + double sizeZ = samplesZ; + + double distMax = distance3D(0, 0, 0, sizeX, sizeY, sizeZ); + double cycles = Math.PI * 2 * 20; + double midX = samplesX / 2f; + double midY = samplesY / 2f; + double midZ = samplesZ / 2f; + + VolumeDataLayoutDescriptor.BrickSize brickSize = VolumeDataLayoutDescriptor.BrickSize.BRICK_SIZE_64; + int negativeMargin = 4; + int positiveMargin = 4; + int brickSize2DMultiplier = 4; + + VolumeDataLayoutDescriptor.LODLevels lodLevels = VolumeDataLayoutDescriptor.LODLevels.values()[lodParam]; + + //VolumeDataLayoutDescriptor.Options layoutOptions = VolumeDataLayoutDescriptor.Options.NONE; + VolumeDataLayoutDescriptor layoutDescriptor = new VolumeDataLayoutDescriptor(brickSize, negativeMargin, positiveMargin, + brickSize2DMultiplier, lodLevels, false,false,0); + + List axisDescriptors = new ArrayList<>(); + axisDescriptors.add(new VolumeDataAxisDescriptor(samplesX, "Sample", "s", 0.0f,6.0f)); + axisDescriptors.add(new VolumeDataAxisDescriptor(samplesY, "Crossline", "",0f, 0f + samplesY - 1f)); + axisDescriptors.add(new VolumeDataAxisDescriptor(samplesZ, "Inline", "", 0f, 0f + samplesZ - 1f)); + + List channelDescriptors = new ArrayList<>(); + + float rangeMin = -1f; + float rangeMax = 1f; + float[] scaleOffset = getScaleOffsetForFormat(rangeMin, rangeMax, true, format); + + VolumeDataChannelDescriptor channelDescriptor = new VolumeDataChannelDescriptor(format, VolumeDataChannelDescriptor.Components.COMPONENTS_1, + "Amplitude", + "", + rangeMin, // range min + rangeMax, // range max + VolumeDataMapping.DIRECT, // mapping + 1, // mapped value count + false, // is discrete + true, // is renderable + false, // allow lossy compression + false, // use zip for lossless compresion + false, // use no value + -999.25f, // no value + scaleOffset[0], // integer scale + scaleOffset[1]); // integer offset + channelDescriptors.add(channelDescriptor); + + // options AWS + String pBucket = "miniovaap"; // the bucket of the VDS + String pKey = name; // the key prefix of the VDS ? + String pRegion = "eu-west-3"; // the region of the bucket of the VDS + String pEndpointOverride = "https://webvr.int.com/"; // This parameter allows to override the endpoint url + AWSOpenOptions optionsAWS = new AWSOpenOptions(pBucket, pKey, pRegion, pEndpointOverride); + optionsAWS.accessKeyId = "minioadmin"; // ? + optionsAWS.secretKey = "minioadmin"; + optionsAWS.sessionToken = ""; // ? + + // file options + //VDSFileOpenOptions options = new VDSFileOpenOptions(vdsFilePath); + + MetadataContainer metadataContainer = new MetadataContainer(); + StringBuilder builder = new StringBuilder(); + builder.append("CreateVDS Demo "); + builder.append("dimX: "); + builder.append(dimX); + builder.append(" dimY: "); + builder.append(dimY); + builder.append(" dimZ: "); + builder.append(dimZ); + + metadataContainer.setMetadataString("General Info", "Cube layout", builder.toString()); + metadataContainer.setMetadataInt("General Info", "LOD", lodLevels.ordinal()); + metadataContainer.setMetadataString("General Info", "Brick Size", "BRICK_SIZE_64"); + + VdsHandle vds = OpenVDS.create(optionsAWS, layoutDescriptor, + axisDescriptors.toArray(new VolumeDataAxisDescriptor[0]), + channelDescriptors.toArray(new VolumeDataChannelDescriptor[0]), metadataContainer); + + // for compression use +// float compressionTolerance = 0f; // +// VdsHandle vds = OpenVDS.create(options, layoutDescriptor, +// axisDescriptors.toArray(new VolumeDataAxisDescriptor[0]), +// channelDescriptors.toArray(new VolumeDataChannelDescriptor[0]), metadataContainer, CompressionMethod.ZIP, compressionTolerance); + + VolumeDataLayout layout = vds.getLayout(); + VolumeDataAccessManager accessManager = vds.getAccessManager(); + + int channel = 0; + + int lodCount = lodLevels.ordinal(); + + int[] localIndex = new int[3]; + int[] voxelPos = new int[3]; + + int[] pitch = new int[VolumeDataLayout.Dimensionality_Max]; + int[] chunkMin = new int[VolumeDataLayout.Dimensionality_Max]; + int[] chunkMax = new int[VolumeDataLayout.Dimensionality_Max]; + + for (int lod = 0 ; lod <= lodCount ; ++lod) { + VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( + DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND + lod, // lod + channel, // channel + 50, // max pages + VolumeDataPageAccessor.AccessMode.Create.getCode()); // access mode + + + //ASSERT_TRUE(pageAccessor); + int paLOD = pageAccessor.getLOD(); + int paCI = pageAccessor.getChannelIndex(); + int[] paNumSamples = pageAccessor.getNumSamples(); + + int chunkCount = (int) pageAccessor.getChunkCount(); + System.out.println("LOD : " + lod + " Chunk count : " + chunkCount); + + for (int i = 0; i < chunkCount; i++) { + + long time_1 = System.currentTimeMillis(); + + VolumeDataPage page = pageAccessor.createPage(i); + System.out.print("LOD : " + lod + " / Page " + (i + 1) + " / " + chunkCount); + + VolumeIndexer3D outputIndexer = new VolumeIndexer3D(page, 0, lod, DimensionsND.DIMENSIONS_012.ordinal(), layout); + + int[] numSamplesChunk = new int[3]; + int[] numSamplesDB = new int[3]; + + for (int j = 0; j < 3; j++) { + numSamplesChunk[j] = outputIndexer.getLocalChunkNumSamples(j); + numSamplesDB[j] = outputIndexer.getDataBlockNumSamples(j); + } + System.out.print("\tDimensions Chunk : [" + numSamplesChunk[0] + ", " + numSamplesChunk[1] + ", " + numSamplesChunk[2] + "]"); + System.out.print("\tDimensions DataBlock : [" + numSamplesDB[0] + ", " + numSamplesDB[1] + ", " + numSamplesDB[2] + "]"); + + page.getMinMax(chunkMin, chunkMax); + + System.out.print("\tCoords page : " + chunkMin[0] + "/" + chunkMax[0] + " " + chunkMin[1] + "/" + chunkMax[1] + " " + chunkMin[2] + "/" + chunkMax[2]); + int chunkSizeY = chunkMax[2] - chunkMin[2]; + int chunkSizeX = chunkMax[1] - chunkMin[1]; + int chunkSizeZ = chunkMax[0] - chunkMin[0]; + + // get pitch and allocate buffer size + page.getPitch(pitch); + float[] output = new float[page.getAllocatedBufferSize()]; + + // or read buffer of page + //float[] output = page.readFloatBuffer(pitch); + + System.out.print( " Page Buffer Size : " + output.length + " ");/// [" + stats.getMin() + "," + stats.getMax() + "]"); + // dimension of buffer + System.out.println(" Pitch : [" + pitch[0] + ", " + pitch[1] + ", " + pitch[2] + "]"); + + int[] numSamples = numSamplesChunk; + + for (int iDim2 = 0; iDim2 < numSamples[2]; iDim2++) { + localIndex[2] = iDim2; + for (int iDim1 = 0; iDim1 < numSamples[1]; iDim1++) { + localIndex[1] = iDim1; + for (int iDim0 = 0; iDim0 < numSamples[0]; iDim0++) { + localIndex[0] = iDim0; + + int[] voxelIndex = outputIndexer.localIndexToVoxelIndex(localIndex); + for (int vp = 0; vp < 3; ++vp) { + voxelPos[vp] = voxelIndex[vp]; + } + + float value = 0f; + double dist = 0; + if (voxelPos[0] >= midX) { + dist = distance2D(midY, midZ, voxelPos[1], voxelPos[2]); + } else { + dist = distance3D(midX, midY, midZ, voxelPos[0], voxelPos[1], voxelPos[2]); + } + value = (float) Math.sin((dist * cycles) / distMax); + + int iPos = outputIndexer.localIndexToDataIndex(localIndex); + output[iPos] = value; + } + } + } + + // write buffer then release page + page.writeFloatBuffer(output, pitch); + page.pageRelease(); + } + + int errorCount = accessManager.uploadErrorCount(); + if (errorCount > 0) { + ErrorInfo upErrInfo = accessManager.getCurrentUploadErrorInfo(); + System.out.println("Upload error code : " + upErrInfo.getErrorCode()); + if (upErrInfo.getErrorMessage() != null) { + System.out.println("Upload error message : " + upErrInfo.getErrorMessage()); + } + else { + System.out.println("No error message"); + } + if (upErrInfo.getObjectID() != null) { + System.out.println("Upload error object ID : " + upErrInfo.getObjectID()); + } + else { + System.out.println("No Object ID"); + } + } + + pageAccessor.commit(); + pageAccessor.setMaxPages(0); + accessManager.flushUploadQueue(); + accessManager.destroyVolumeDataPageAccessor(pageAccessor); + } + + + + // Adds slice data : additional layouts +// System.out.println("\nCreate 01 Layout"); +// create2DLayout(distMax, cycles, midX, midY, midZ, layout, accessManager, channel, lodCount, DimensionsND.DIMENSIONS_01); +// +// System.out.println("\nCreate 02 Layout"); +// create2DLayout(distMax, cycles, midX, midY, midZ, layout, accessManager, channel, lodCount, DimensionsND.DIMENSIONS_02); +// +// System.out.println("\nCreate 12 Layout"); +// create2DLayout(distMax, cycles, midX, midY, midZ, layout, accessManager, channel, lodCount, DimensionsND.DIMENSIONS_12); + + + vds.close(); + System.out.println("VDS closed"); + } + + + private static void create2DLayout(double distMax, double cycles, double midX, double midY, double midZ, VolumeDataLayout layout, VolumeDataAccessManager accessManager, + int channel, int lodCount, DimensionsND dimLYT) { + + int[] localIndex = new int[3]; + + int[] pitch = new int[VolumeDataLayout.Dimensionality_Max]; + int[] chunkMin = new int[VolumeDataLayout.Dimensionality_Max]; + int[] chunkMax = new int[VolumeDataLayout.Dimensionality_Max]; + for (int lod = 0 ; lod <= lodCount ; ++lod) { + VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( + dimLYT.ordinal(), // dimension ND + lod, // lod + channel, // channel + 200, // max pages + VolumeDataPageAccessor.AccessMode.Create.getCode()); // access mode + + int chunkCount = (int) pageAccessor.getChunkCount(); + System.out.println("2D Layout LOD : " + lod + " Chunk count : " + chunkCount); + + for (int i = 0; i < chunkCount; i++) { + VolumeDataPage page = pageAccessor.createPage(i); + System.out.print("2D Layout LOD : " + lod + " / Page " + (i + 1) + " / " + chunkCount); + + VolumeIndexer2D outputIndexer = new VolumeIndexer2D(page, 0, lod, dimLYT.ordinal(), layout); + float valueRangeScale = outputIndexer.getValueRangeMax() - outputIndexer.getValueRangeMin(); + + int[] numSamplesChunk = new int[3]; + int[] numSamplesDB = new int[3]; + + for (int j = 0; j < 3 ; j++) { + numSamplesChunk[j] = outputIndexer.getLocalChunkNumSamples(j); + numSamplesDB[j] = outputIndexer.getDataBlockNumSamples(j); + } + System.out.print("\t2D Layout Dimensions Chunk : [" + numSamplesChunk[0] + ", " + numSamplesChunk[1] + ", " + numSamplesChunk[2] + "]"); + System.out.print("\t2D Layout Dimensions DataBlock : [" + numSamplesDB[0] + ", " + numSamplesDB[1] + ", " + numSamplesDB[2] + "]"); + + page.getMinMax(chunkMin, chunkMax); + + // determine fixed position index and value + int fixedCoordIndex = -1; + int fixedCoordValue = -1; + for (int pos = 0; pos < 3 && fixedCoordIndex == -1 ; ++pos) { + if (Math.abs(chunkMax[pos] - chunkMin[pos]) == 1) { + fixedCoordIndex = pos; + fixedCoordValue = chunkMin[pos]; + } + } + + System.out.print("\tCoords page : " + chunkMin[0] + "/" + chunkMax[0] + " " + chunkMin[1] + "/" + chunkMax[1] + "/" + chunkMin[2] + "/" + chunkMax[2]); + int chunkSizeY = chunkMax[2] - chunkMin[2]; + int chunkSizeX = chunkMax[1] - chunkMin[1]; + int chunkSizeZ = chunkMax[0] - chunkMin[0]; + + // get pitch and allocate buffer size + page.getPitch(pitch); + float[] output = new float[page.getAllocatedBufferSize()]; + + System.out.print( " Page Buffer Size : " + output.length);// + " / [" + stats.getMin() + "," + stats.getMax() + "]"); + // dimension of buffer + System.out.println(" Pitch : [" + pitch[0] + ", " + pitch[1] + "," + pitch[2] + "]"); + + long time_2 = System.currentTimeMillis(); + + int[] numSamples = numSamplesChunk; + + for (int iDim2 = 0; iDim2 < numSamples[2]; iDim2++) { + localIndex[2] = iDim2; + for (int iDim1 = 0; iDim1 < numSamples[1]; iDim1++) { + localIndex[1] = iDim1; + for (int iDim0 = 0; iDim0 < numSamples[0]; iDim0++) { + localIndex[0] = iDim0; + int[] local2DIndex = convertTo2DLocalIndex(localIndex, fixedCoordIndex); + int[] voxelIndex = outputIndexer.localChunkIndexToLocalIndex(local2DIndex); + int[] voxelPos = convertTo3DCoord(voxelIndex, fixedCoordIndex, fixedCoordValue); + + float value = 0f; + double dist = 0; + if (voxelPos[0] >= midX) { + dist = distance2D(midY, midZ, voxelPos[1], voxelPos[2]); + } else { + dist = distance3D(midX, midY, midZ, voxelPos[0], voxelPos[1], voxelPos[2]); + } + value = (float) Math.sin((dist * cycles) / distMax); + + int iPos = outputIndexer.localIndexToDataIndex(local2DIndex); + output[iPos] = value; + } + } + } + + // write buffer then release page + page.writeFloatBuffer(output, pitch); + page.pageRelease(); + } + + pageAccessor.commit(); + accessManager.flushUploadQueue(); + accessManager.destroyVolumeDataPageAccessor(pageAccessor); + } + } + + static void testRead(String vdsName) throws IOException { + // options AWS + String pBucket = "miniovaap"; // the bucket of the VDS + String pKey = vdsName; // the key prefix of the VDS ? + String pRegion = "eu-west-3"; // the region of the bucket of the VDS + String pEndpointOverride = "https://webvr.int.com/"; // This parameter allows to override the endpoint url + AWSOpenOptions optionsAWS = new AWSOpenOptions(pBucket, pKey, pRegion, pEndpointOverride); + optionsAWS.accessKeyId = "minioadmin"; // ? + optionsAWS.secretKey = "minioadmin"; + optionsAWS.sessionToken = ""; // ? + + OpenVDS vdsHandle = OpenVDS.open(optionsAWS); + VolumeDataAccessManager accessManager = vdsHandle.getAccessManager(); + + int channel = 0; + VolumeDataLayout layout = vdsHandle.getLayout(); + VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( + DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND + 0, // lod + channel, // channel + 20, // max pages + VolumeDataPageAccessor.AccessMode.ReadOnly.getCode()); // access mode + + + + // compares block data + int[] pitchInput = new int[VolumeDataLayout.Dimensionality_Max]; + + long chunkCount = pageAccessor.getChunkCount(); + for (long chunk = 0 ; chunk < chunkCount ; ++chunk) { + VolumeDataPage inputPage = pageAccessor.readPage(chunk); + VolumeDataPage page = pageAccessor.readPage(chunk); + float[] dataIn = inputPage.readFloatBuffer(pitchInput); + + ErrorInfo dwnErrorInfo = accessManager.getCurrentDownloadErrorInfo(); + if (dwnErrorInfo.getErrorCode() != 0) { + System.out.println("Download error code : " + dwnErrorInfo.getErrorCode()); + if (dwnErrorInfo.getErrorMessage() != null) { + System.out.println("Download error message : " + dwnErrorInfo.getErrorMessage()); + } else { + System.out.println("No error message"); + } + } + } + } + + + /** + * Convert 3D index to 2D index + */ + private static int[] convertTo2DLocalIndex(int[] localIndex, int fixedCoordIndex) { + int[] local2DIndex = new int[2]; + if (fixedCoordIndex == 0) { + local2DIndex[0] = localIndex[1]; + local2DIndex[1] = localIndex[2]; + } + else if (fixedCoordIndex == 1) { + local2DIndex[0] = localIndex[0]; + local2DIndex[1] = localIndex[2]; + } + else if (fixedCoordIndex == 2) { + local2DIndex[0] = localIndex[0]; + local2DIndex[1] = localIndex[1]; + } + return local2DIndex; + } + + /** + * Convert 2D index to 3D index + * @param voxelIndex + * @param fixedCoordIndex + * @param fixedCoordValue + */ + private static int[] convertTo3DCoord(int[] voxelIndex, int fixedCoordIndex, int fixedCoordValue) { + int[] voxelPos = new int[3]; + voxelPos[fixedCoordIndex] = fixedCoordValue; + if (fixedCoordIndex == 0) { + voxelPos[1] = voxelIndex[0]; + voxelPos[2] = voxelIndex[1]; + } + else if (fixedCoordIndex == 1) { + voxelPos[0] = voxelIndex[0]; + voxelPos[2] = voxelIndex[1]; + } + else if (fixedCoordIndex == 2) { + voxelPos[0] = voxelIndex[0]; + voxelPos[1] = voxelIndex[1]; + } + return voxelPos; + } + + private static double distance2D(double x1, double y1, double x2, double y2) { + double diffX = x2 - x1; + double diffY = y2 - y1; + return Math.sqrt((diffX * diffX) + (diffY * diffY)); + } + + private static double distance3D(double x1, double y1, double z1, double x2, double y2, double z2) { + double diffX = x2 - x1; + double diffY = y2 - y1; + double diffZ = z2 - z1; + return Math.sqrt((diffX * diffX) + (diffY * diffY) + (diffZ * diffZ)); + } + + static float[] getScaleOffsetForFormat(float min, float max, boolean useNoValue, VolumeDataChannelDescriptor.Format format) { + float res[] = new float[] {1f, 0f}; + float noValueCmp = useNoValue ? 1f : 0f; + switch (format) { + case FORMAT_U8: + res[0] = 1.f / (255.f - noValueCmp) * (max - min); + res[1] = min; + break; + case FORMAT_U16: + res[0] = 1.f / (65535.f - noValueCmp) * (max - min); + res[1] = min; + break; + case FORMAT_R32: + case FORMAT_U32: + case FORMAT_R64: + case FORMAT_U64: + case FORMAT_1BIT: + case FORMAT_ANY: + res[0] = 1.0f; + res[1] = 0.0f; + } + return res; + } + +} diff --git a/java/java/src/org/opengroup/openvds/ErrorInfo.java b/java/java/src/org/opengroup/openvds/ErrorInfo.java index 4dd9f017..fc9b81b7 100644 --- a/java/java/src/org/opengroup/openvds/ErrorInfo.java +++ b/java/java/src/org/opengroup/openvds/ErrorInfo.java @@ -5,13 +5,12 @@ package org.opengroup.openvds; */ public class ErrorInfo { - private int errorCode; + private int errorCode = -1; private String errorMessage; private String objectID; - public ErrorInfo() { + public ErrorInfo() { } - } public ErrorInfo(int code, String msg, String id) { errorCode = code; errorMessage = msg; diff --git a/java/java/test/org/opengroup/openvds/PageAccessorByteTest.java b/java/java/test/org/opengroup/openvds/PageAccessorByteTest.java new file mode 100644 index 00000000..4e84d0a0 --- /dev/null +++ b/java/java/test/org/opengroup/openvds/PageAccessorByteTest.java @@ -0,0 +1,335 @@ +/* + * Copyright 2021 The Open Group + * Copyright 2021 INT, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.openvds; + +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.io.File; + +import static org.testng.Assert.fail; + +public class PageAccessorByteTest { + + private static String TEMP_FILE_NAME_VOL_INDEX = "volIndexer"; + private static String TEMP_FILE_NAME_COPY = "vdsCopy"; + + public String url; + public VolumeDataLayoutDescriptor ld; + public VolumeDataAxisDescriptor[] vda; + public VolumeDataChannelDescriptor[] vdc; + public MetadataReadAccess md; + public MemoryVdsGenerator vds; + public MetadataContainer metadataContainer; + + private String tempVolIndexerFileName; + private String tempVdsCopyFileName; + + @BeforeClass + public void init() { + vds = new MemoryVdsGenerator(200, 200, 200, VolumeDataChannelDescriptor.Format.FORMAT_U8); + url = "inmemory://create_test"; + VolumeDataLayout volumeDataLayout = vds.getLayout(); + + int nbChannel = volumeDataLayout.getChannelCount(); + VolumeDataAccessManager accessManager = vds.getAccessManager(); + + for (VolumeDataLayoutDescriptor.LODLevels l : VolumeDataLayoutDescriptor.LODLevels.values()) { + for (int channel = 0; channel < nbChannel; channel++) { + for (DimensionsND dimGroup : DimensionsND.values()) { + VDSProduceStatus vdsProduceStatus = accessManager.getVDSProduceStatus(dimGroup, l.ordinal(), channel); + } + } + } + + vda = new VolumeDataAxisDescriptor[] { + volumeDataLayout.getAxisDescriptor(0), + volumeDataLayout.getAxisDescriptor(1), + volumeDataLayout.getAxisDescriptor(2)}; + vdc = new VolumeDataChannelDescriptor[] {volumeDataLayout.getChannelDescriptor(0)}; + + md = volumeDataLayout; + ld = volumeDataLayout.getLayoutDescriptor(); + + metadataContainer = new MetadataContainer(); + + long ms = System.currentTimeMillis(); + tempVolIndexerFileName = TEMP_FILE_NAME_VOL_INDEX + "_" + ms + ".vds"; + tempVdsCopyFileName = TEMP_FILE_NAME_COPY + "_" + ms + ".vds"; + } + + @AfterClass + public void cleanFiles() { + String tempDir = System.getProperty("java.io.tmpdir"); + String fileVolIndexPath = tempDir + File.separator + tempVolIndexerFileName; + File fileVolIndex = new File(fileVolIndexPath); + if (fileVolIndex.exists()) { + //fileVolIndex.delete(); + } + + String fileCopyPath = tempDir + File.separator + tempVdsCopyFileName; + File fileCopy = new File(fileCopyPath); + if (fileCopy.exists()) { + //fileCopy.delete(); + } + } + + @Test + public void testVolumeIndexerCreationDeletion() { + try { + // create file in tmp dir + String tmpDir = System.getProperty("java.io.tmpdir"); + String volIndexPath = tmpDir + File.separator + tempVolIndexerFileName; + VDSFileOpenOptions options = new VDSFileOpenOptions(volIndexPath); + VdsHandle vdsTest = OpenVDS.create(options, ld, + vda, + vdc, metadataContainer); + VolumeDataAccessManager accessManager = vdsTest.getAccessManager(); + //ASSERT_TRUE(accessManager); + + int channel = 0; + VolumeDataLayout layout = vdsTest.getLayout(); + VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( + DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND + 0, // lod + channel, // channel + 100, // max pages + VolumeDataPageAccessor.AccessMode.Create.getCode()); // access mode + + VolumeDataPage page = pageAccessor.createPage(0); + VolumeIndexer3D outputIndexer = new VolumeIndexer3D(page, 0, 0, DimensionsND.DIMENSIONS_012.ordinal(), layout); + outputIndexer.finalize(); + + vdsTest.close(); + } + catch (java.io.IOException e) { + System.out.println(e.getMessage()); + fail(); + } + } + + @Test + public void testCopyPageAccessor() { + try { + String tmpDir = System.getProperty("java.io.tmpdir"); + String vdsPath = tmpDir + File.separator + tempVdsCopyFileName; + VDSFileOpenOptions options = new VDSFileOpenOptions(vdsPath); + + // copy information from input VDS + VolumeDataLayoutDescriptor cpLd = new VolumeDataLayoutDescriptor( + ld.getBrickSize(), + ld.getNegativeMargin(), + ld.getPositiveMargin(), + ld.getBrickSizeMultiplier2D(), + ld.getLODLevels(), + ld.isCreate2DLODs(), + ld.isForceFullResolutionDimension(), + ld.getFullResolutionDimension() + ); + + VolumeDataAxisDescriptor[] cpVda = new VolumeDataAxisDescriptor[3]; + for (int i = 0 ; i < 3 ; ++i) { + cpVda[i] = new VolumeDataAxisDescriptor(vda[i].getNumSamples(), + vda[i].getName(), vda[i].getUnit(), + vda[i].getCoordinateMin(), vda[i].getCoordinateMax()); + } + + VolumeDataChannelDescriptor[] cpVdc = new VolumeDataChannelDescriptor[1]; + cpVdc[0] = new VolumeDataChannelDescriptor( + VolumeDataChannelDescriptor.Format.fromCode(vdc[0].getFormat()), + VolumeDataChannelDescriptor.Components.fromCode(vdc[0].getComponents()), + vdc[0].getName(), + vdc[0].getUnit(), + vdc[0].getValueRangeMin(), vdc[0].getValueRangeMax(), + VolumeDataMapping.fromCode(vdc[0].getMapping()), + vdc[0].getMappedValueCount(), + vdc[0].isDiscrete(), + vdc[0].isRenderable(), + vdc[0].isAllowLossyCompression(), + vdc[0].isUseZipForLosslessCompression(), + vdc[0].isUseNoValue(), + vdc[0].getNoValue(), + vdc[0].getIntegerScale(), + vdc[0].getIntegerOffset() + ); + MetadataContainer cpMetadataContainer = new MetadataContainer(); + VdsHandle vdsCopy = OpenVDS.create(options, cpLd, + cpVda, + cpVdc, + cpMetadataContainer); + + VolumeDataAccessManager accessManager = vdsCopy.getAccessManager(); + //ASSERT_TRUE(accessManager); + + int channel = 0; + VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( + DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND + 0, // lod + channel, // channel + 100, // max pages + VolumeDataPageAccessor.AccessMode.Create.getCode()); // access mode + + // get input manager + VolumeDataAccessManager inputAM = vds.getAccessManager(); + VolumeDataPageAccessor pageAccessorInput = inputAM.createVolumeDataPageAccessor( + DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND + 0, // lod + channel, // channel + 100, // max pages + VolumeDataPageAccessor.AccessMode.ReadOnly.getCode()); // access mode + + // copy file + int[] pitch = new int[VolumeDataLayout.Dimensionality_Max]; + long chunkCount = pageAccessorInput.getChunkCount(); + for (long chunk = 0 ; chunk < chunkCount ; ++chunk) { + VolumeDataPage inputPage = pageAccessorInput.readPage(chunk); + VolumeDataPage page = pageAccessor.createPage(chunk); + byte[] data = inputPage.readByteBuffer(pitch); + page.writeByteBuffer(data, pitch); + + inputPage.pageRelease(); + page.pageRelease(); + } + pageAccessor.commit(); + accessManager.flushUploadQueue(); + accessManager.destroyVolumeDataPageAccessor(pageAccessor); + + vdsCopy.close(); + } + catch (java.io.IOException e) { + System.out.println(e.getMessage()); + fail(); + } + } + + /** + * Will test that copied file is the same as the input + * Name of method is the same + Suffix so that it's executed after the copy test + */ + @Test + public void testCopyPageAccessorValidation() { + try { + String tmpDir = System.getProperty("java.io.tmpdir"); + String vdsPath = tmpDir + File.separator + tempVdsCopyFileName; + VDSFileOpenOptions options = new VDSFileOpenOptions(vdsPath); + VdsHandle vdsCopy = OpenVDS.open(options); + VolumeDataAccessManager accessManager = vdsCopy.getAccessManager(); + + int channel = 0; + VolumeDataLayout layout = vdsCopy.getLayout(); + VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( + DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND + 0, // lod + channel, // channel + 20, // max pages + VolumeDataPageAccessor.AccessMode.ReadOnly.getCode()); // access mode + + // get input manager + VolumeDataAccessManager inputAM = vds.getAccessManager(); + VolumeDataPageAccessor pageAccessorInput = inputAM.createVolumeDataPageAccessor( + DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND + 0, // lod + channel, // channel + 20, // max pages + VolumeDataPageAccessor.AccessMode.ReadOnly.getCode()); // access mode + + // compares block data + int[] pitchInput = new int[VolumeDataLayout.Dimensionality_Max]; + int[] pitchOutput = new int[VolumeDataLayout.Dimensionality_Max]; + + long chunkCount = pageAccessorInput.getChunkCount(); + for (long chunk = 0 ; chunk < chunkCount ; ++chunk) { + VolumeDataPage inputPage = pageAccessorInput.readPage(chunk); + VolumeDataPage page = pageAccessor.readPage(chunk); + byte[] dataIn = inputPage.readByteBuffer(pitchInput); + byte[] dataOut = page.readByteBuffer(pitchOutput); + + inputPage.pageRelease(); + page.pageRelease(); + + Assert.assertEquals(pitchInput, pitchOutput); + Assert.assertEquals(dataIn, dataOut); + } + + accessManager.destroyVolumeDataPageAccessor(pageAccessor); + + vdsCopy.close(); + } + catch (java.io.IOException e) { + System.out.println(e.getMessage()); + fail(); + } + } + + /** + * Will test that copied file has the same layout than original (test that 3D positions are in the same chunk index) + * Name of method is the same + Suffix so that it's executed after the copy test + */ + @Test + public void testCopyPageAccessorValidationChunkIndex() { + try { + String tmpDir = System.getProperty("java.io.tmpdir"); + String vdsPath = tmpDir + File.separator + tempVdsCopyFileName; + VDSFileOpenOptions options = new VDSFileOpenOptions(vdsPath); + VdsHandle vdsCopy = OpenVDS.open(options); + VolumeDataAccessManager accessManager = vdsCopy.getAccessManager(); + + int channel = 0; + VolumeDataLayout layout = vdsCopy.getLayout(); + VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( + DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND + 0, // lod + channel, // channel + 20, // max pages + VolumeDataPageAccessor.AccessMode.ReadOnly.getCode()); // access mode + + // compares block data + int[] chunkMin = new int[VolumeDataLayout.Dimensionality_Max]; + int[] chunkMax = new int[VolumeDataLayout.Dimensionality_Max]; + int[] chunkMaxPos = new int[VolumeDataLayout.Dimensionality_Max]; + + long chunkCount = pageAccessor.getChunkCount(); + for (long chunk = 0 ; chunk < chunkCount ; ++chunk) { + VolumeDataPage inputPage = pageAccessor.readPage(chunk); + + // check that chunk index matches current index + inputPage.getMinMaxExcludingMargin(chunkMin, chunkMax); + for (int i = 0 ; i < VolumeDataLayout.Dimensionality_Max ; ++i) { + chunkMaxPos[i] = chunkMax[i] != 0 ? chunkMax[i] - 1 : chunkMax[i]; + } + long idxChMin = pageAccessor.getChunkIndex(chunkMin); + long idxChMax = pageAccessor.getChunkIndex(chunkMaxPos); + + Assert.assertEquals(chunk, idxChMin); + Assert.assertEquals(idxChMin, idxChMax); + + inputPage.pageRelease(); + } + + accessManager.destroyVolumeDataPageAccessor(pageAccessor); + + vdsCopy.close(); + } + catch (java.io.IOException e) { + System.out.println(e.getMessage()); + fail(); + } + } +} -- GitLab From e40fb75333df8db9be45e9b6d4be95ca5806e843 Mon Sep 17 00:00:00 2001 From: Julien Lacoste Date: Thu, 9 Sep 2021 17:18:35 +0200 Subject: [PATCH 15/25] Rename ErrorInfo class --- java/CMakeLists.txt | 2 +- ...rrorInfo.java => ConnectionErrorInfo.java} | 6 +++--- .../openvds/VolumeDataAccessManager.java | 20 +++++++++---------- 3 files changed, 14 insertions(+), 14 deletions(-) rename java/java/src/org/opengroup/openvds/{ErrorInfo.java => ConnectionErrorInfo.java} (84%) diff --git a/java/CMakeLists.txt b/java/CMakeLists.txt index 1d7e322f..7f954fcb 100644 --- a/java/CMakeLists.txt +++ b/java/CMakeLists.txt @@ -18,9 +18,9 @@ set(JAVA_SOURCE_FILES java/src/org/opengroup/openvds/BufferUtils.java java/src/org/opengroup/openvds/Cleaner.java java/src/org/opengroup/openvds/CompressionMethod.java + java/src/org/opengroup/openvds/ConnectionErrorInfo.java java/src/org/opengroup/openvds/DimensionsND.java java/src/org/opengroup/openvds/experimental/VariousJavaTests.java - java/src/org/opengroup/openvds/ErrorInfo.java java/src/org/opengroup/openvds/GoogleOpenOptions.java java/src/org/opengroup/openvds/IndexRegionFloat.java java/src/org/opengroup/openvds/IndexRegionLong.java diff --git a/java/java/src/org/opengroup/openvds/ErrorInfo.java b/java/java/src/org/opengroup/openvds/ConnectionErrorInfo.java similarity index 84% rename from java/java/src/org/opengroup/openvds/ErrorInfo.java rename to java/java/src/org/opengroup/openvds/ConnectionErrorInfo.java index fc9b81b7..c1584d91 100644 --- a/java/java/src/org/opengroup/openvds/ErrorInfo.java +++ b/java/java/src/org/opengroup/openvds/ConnectionErrorInfo.java @@ -3,15 +3,15 @@ package org.opengroup.openvds; /** * Simple class holding information on Upload/Download error */ -public class ErrorInfo { +public class ConnectionErrorInfo { private int errorCode = -1; private String errorMessage; private String objectID; - public ErrorInfo() { } + public ConnectionErrorInfo() { } - public ErrorInfo(int code, String msg, String id) { + public ConnectionErrorInfo(int code, String msg, String id) { errorCode = code; errorMessage = msg; objectID = id; diff --git a/java/java/src/org/opengroup/openvds/VolumeDataAccessManager.java b/java/java/src/org/opengroup/openvds/VolumeDataAccessManager.java index 86d24b8c..506ac63e 100644 --- a/java/java/src/org/opengroup/openvds/VolumeDataAccessManager.java +++ b/java/java/src/org/opengroup/openvds/VolumeDataAccessManager.java @@ -147,9 +147,9 @@ public class VolumeDataAccessManager extends JniPointerWithoutDeletion { private static native int cpGetCurrentUploadErrorCode(long handle); - private static native void cpGetCurrentUploadErrorInfo(long handle, ErrorInfo error); + private static native void cpGetCurrentUploadErrorInfo(long handle, ConnectionErrorInfo error); - private static native void cpGetCurrentDownloadErrorInfo(long handle, ErrorInfo error); + private static native void cpGetCurrentDownloadErrorInfo(long handle, ConnectionErrorInfo error); public VolumeDataAccessManager(long handle) { super(handle); @@ -779,18 +779,18 @@ public class VolumeDataAccessManager extends JniPointerWithoutDeletion { /** * @return current upload error information (code and message in a simple class) */ - public ErrorInfo getCurrentUploadErrorInfo() { - ErrorInfo errorInfo = new ErrorInfo(); - cpGetCurrentUploadErrorInfo(_handle, errorInfo); - return errorInfo; + public ConnectionErrorInfo getCurrentUploadErrorInfo() { + ConnectionErrorInfo connectionErrorInfo = new ConnectionErrorInfo(); + cpGetCurrentUploadErrorInfo(_handle, connectionErrorInfo); + return connectionErrorInfo; } /** * @return current upload error information (code and message in a simple class) */ - public ErrorInfo getCurrentDownloadErrorInfo() { - ErrorInfo errorInfo = new ErrorInfo(); - cpGetCurrentDownloadErrorInfo(_handle, errorInfo); - return errorInfo; + public ConnectionErrorInfo getCurrentDownloadErrorInfo() { + ConnectionErrorInfo connectionErrorInfo = new ConnectionErrorInfo(); + cpGetCurrentDownloadErrorInfo(_handle, connectionErrorInfo); + return connectionErrorInfo; } } -- GitLab From a482bfd3b3c1d0e41ce6219137b66ad81f00e78f Mon Sep 17 00:00:00 2001 From: Julien Lacoste Date: Thu, 9 Sep 2021 17:22:36 +0200 Subject: [PATCH 16/25] Remove temp test --- .../openvds/PageAccessorByteTest.java | 335 ------------------ 1 file changed, 335 deletions(-) delete mode 100644 java/java/test/org/opengroup/openvds/PageAccessorByteTest.java diff --git a/java/java/test/org/opengroup/openvds/PageAccessorByteTest.java b/java/java/test/org/opengroup/openvds/PageAccessorByteTest.java deleted file mode 100644 index 4e84d0a0..00000000 --- a/java/java/test/org/opengroup/openvds/PageAccessorByteTest.java +++ /dev/null @@ -1,335 +0,0 @@ -/* - * Copyright 2021 The Open Group - * Copyright 2021 INT, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opengroup.openvds; - -import org.testng.Assert; -import org.testng.annotations.AfterClass; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.Test; - -import java.io.File; - -import static org.testng.Assert.fail; - -public class PageAccessorByteTest { - - private static String TEMP_FILE_NAME_VOL_INDEX = "volIndexer"; - private static String TEMP_FILE_NAME_COPY = "vdsCopy"; - - public String url; - public VolumeDataLayoutDescriptor ld; - public VolumeDataAxisDescriptor[] vda; - public VolumeDataChannelDescriptor[] vdc; - public MetadataReadAccess md; - public MemoryVdsGenerator vds; - public MetadataContainer metadataContainer; - - private String tempVolIndexerFileName; - private String tempVdsCopyFileName; - - @BeforeClass - public void init() { - vds = new MemoryVdsGenerator(200, 200, 200, VolumeDataChannelDescriptor.Format.FORMAT_U8); - url = "inmemory://create_test"; - VolumeDataLayout volumeDataLayout = vds.getLayout(); - - int nbChannel = volumeDataLayout.getChannelCount(); - VolumeDataAccessManager accessManager = vds.getAccessManager(); - - for (VolumeDataLayoutDescriptor.LODLevels l : VolumeDataLayoutDescriptor.LODLevels.values()) { - for (int channel = 0; channel < nbChannel; channel++) { - for (DimensionsND dimGroup : DimensionsND.values()) { - VDSProduceStatus vdsProduceStatus = accessManager.getVDSProduceStatus(dimGroup, l.ordinal(), channel); - } - } - } - - vda = new VolumeDataAxisDescriptor[] { - volumeDataLayout.getAxisDescriptor(0), - volumeDataLayout.getAxisDescriptor(1), - volumeDataLayout.getAxisDescriptor(2)}; - vdc = new VolumeDataChannelDescriptor[] {volumeDataLayout.getChannelDescriptor(0)}; - - md = volumeDataLayout; - ld = volumeDataLayout.getLayoutDescriptor(); - - metadataContainer = new MetadataContainer(); - - long ms = System.currentTimeMillis(); - tempVolIndexerFileName = TEMP_FILE_NAME_VOL_INDEX + "_" + ms + ".vds"; - tempVdsCopyFileName = TEMP_FILE_NAME_COPY + "_" + ms + ".vds"; - } - - @AfterClass - public void cleanFiles() { - String tempDir = System.getProperty("java.io.tmpdir"); - String fileVolIndexPath = tempDir + File.separator + tempVolIndexerFileName; - File fileVolIndex = new File(fileVolIndexPath); - if (fileVolIndex.exists()) { - //fileVolIndex.delete(); - } - - String fileCopyPath = tempDir + File.separator + tempVdsCopyFileName; - File fileCopy = new File(fileCopyPath); - if (fileCopy.exists()) { - //fileCopy.delete(); - } - } - - @Test - public void testVolumeIndexerCreationDeletion() { - try { - // create file in tmp dir - String tmpDir = System.getProperty("java.io.tmpdir"); - String volIndexPath = tmpDir + File.separator + tempVolIndexerFileName; - VDSFileOpenOptions options = new VDSFileOpenOptions(volIndexPath); - VdsHandle vdsTest = OpenVDS.create(options, ld, - vda, - vdc, metadataContainer); - VolumeDataAccessManager accessManager = vdsTest.getAccessManager(); - //ASSERT_TRUE(accessManager); - - int channel = 0; - VolumeDataLayout layout = vdsTest.getLayout(); - VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( - DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND - 0, // lod - channel, // channel - 100, // max pages - VolumeDataPageAccessor.AccessMode.Create.getCode()); // access mode - - VolumeDataPage page = pageAccessor.createPage(0); - VolumeIndexer3D outputIndexer = new VolumeIndexer3D(page, 0, 0, DimensionsND.DIMENSIONS_012.ordinal(), layout); - outputIndexer.finalize(); - - vdsTest.close(); - } - catch (java.io.IOException e) { - System.out.println(e.getMessage()); - fail(); - } - } - - @Test - public void testCopyPageAccessor() { - try { - String tmpDir = System.getProperty("java.io.tmpdir"); - String vdsPath = tmpDir + File.separator + tempVdsCopyFileName; - VDSFileOpenOptions options = new VDSFileOpenOptions(vdsPath); - - // copy information from input VDS - VolumeDataLayoutDescriptor cpLd = new VolumeDataLayoutDescriptor( - ld.getBrickSize(), - ld.getNegativeMargin(), - ld.getPositiveMargin(), - ld.getBrickSizeMultiplier2D(), - ld.getLODLevels(), - ld.isCreate2DLODs(), - ld.isForceFullResolutionDimension(), - ld.getFullResolutionDimension() - ); - - VolumeDataAxisDescriptor[] cpVda = new VolumeDataAxisDescriptor[3]; - for (int i = 0 ; i < 3 ; ++i) { - cpVda[i] = new VolumeDataAxisDescriptor(vda[i].getNumSamples(), - vda[i].getName(), vda[i].getUnit(), - vda[i].getCoordinateMin(), vda[i].getCoordinateMax()); - } - - VolumeDataChannelDescriptor[] cpVdc = new VolumeDataChannelDescriptor[1]; - cpVdc[0] = new VolumeDataChannelDescriptor( - VolumeDataChannelDescriptor.Format.fromCode(vdc[0].getFormat()), - VolumeDataChannelDescriptor.Components.fromCode(vdc[0].getComponents()), - vdc[0].getName(), - vdc[0].getUnit(), - vdc[0].getValueRangeMin(), vdc[0].getValueRangeMax(), - VolumeDataMapping.fromCode(vdc[0].getMapping()), - vdc[0].getMappedValueCount(), - vdc[0].isDiscrete(), - vdc[0].isRenderable(), - vdc[0].isAllowLossyCompression(), - vdc[0].isUseZipForLosslessCompression(), - vdc[0].isUseNoValue(), - vdc[0].getNoValue(), - vdc[0].getIntegerScale(), - vdc[0].getIntegerOffset() - ); - MetadataContainer cpMetadataContainer = new MetadataContainer(); - VdsHandle vdsCopy = OpenVDS.create(options, cpLd, - cpVda, - cpVdc, - cpMetadataContainer); - - VolumeDataAccessManager accessManager = vdsCopy.getAccessManager(); - //ASSERT_TRUE(accessManager); - - int channel = 0; - VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( - DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND - 0, // lod - channel, // channel - 100, // max pages - VolumeDataPageAccessor.AccessMode.Create.getCode()); // access mode - - // get input manager - VolumeDataAccessManager inputAM = vds.getAccessManager(); - VolumeDataPageAccessor pageAccessorInput = inputAM.createVolumeDataPageAccessor( - DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND - 0, // lod - channel, // channel - 100, // max pages - VolumeDataPageAccessor.AccessMode.ReadOnly.getCode()); // access mode - - // copy file - int[] pitch = new int[VolumeDataLayout.Dimensionality_Max]; - long chunkCount = pageAccessorInput.getChunkCount(); - for (long chunk = 0 ; chunk < chunkCount ; ++chunk) { - VolumeDataPage inputPage = pageAccessorInput.readPage(chunk); - VolumeDataPage page = pageAccessor.createPage(chunk); - byte[] data = inputPage.readByteBuffer(pitch); - page.writeByteBuffer(data, pitch); - - inputPage.pageRelease(); - page.pageRelease(); - } - pageAccessor.commit(); - accessManager.flushUploadQueue(); - accessManager.destroyVolumeDataPageAccessor(pageAccessor); - - vdsCopy.close(); - } - catch (java.io.IOException e) { - System.out.println(e.getMessage()); - fail(); - } - } - - /** - * Will test that copied file is the same as the input - * Name of method is the same + Suffix so that it's executed after the copy test - */ - @Test - public void testCopyPageAccessorValidation() { - try { - String tmpDir = System.getProperty("java.io.tmpdir"); - String vdsPath = tmpDir + File.separator + tempVdsCopyFileName; - VDSFileOpenOptions options = new VDSFileOpenOptions(vdsPath); - VdsHandle vdsCopy = OpenVDS.open(options); - VolumeDataAccessManager accessManager = vdsCopy.getAccessManager(); - - int channel = 0; - VolumeDataLayout layout = vdsCopy.getLayout(); - VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( - DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND - 0, // lod - channel, // channel - 20, // max pages - VolumeDataPageAccessor.AccessMode.ReadOnly.getCode()); // access mode - - // get input manager - VolumeDataAccessManager inputAM = vds.getAccessManager(); - VolumeDataPageAccessor pageAccessorInput = inputAM.createVolumeDataPageAccessor( - DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND - 0, // lod - channel, // channel - 20, // max pages - VolumeDataPageAccessor.AccessMode.ReadOnly.getCode()); // access mode - - // compares block data - int[] pitchInput = new int[VolumeDataLayout.Dimensionality_Max]; - int[] pitchOutput = new int[VolumeDataLayout.Dimensionality_Max]; - - long chunkCount = pageAccessorInput.getChunkCount(); - for (long chunk = 0 ; chunk < chunkCount ; ++chunk) { - VolumeDataPage inputPage = pageAccessorInput.readPage(chunk); - VolumeDataPage page = pageAccessor.readPage(chunk); - byte[] dataIn = inputPage.readByteBuffer(pitchInput); - byte[] dataOut = page.readByteBuffer(pitchOutput); - - inputPage.pageRelease(); - page.pageRelease(); - - Assert.assertEquals(pitchInput, pitchOutput); - Assert.assertEquals(dataIn, dataOut); - } - - accessManager.destroyVolumeDataPageAccessor(pageAccessor); - - vdsCopy.close(); - } - catch (java.io.IOException e) { - System.out.println(e.getMessage()); - fail(); - } - } - - /** - * Will test that copied file has the same layout than original (test that 3D positions are in the same chunk index) - * Name of method is the same + Suffix so that it's executed after the copy test - */ - @Test - public void testCopyPageAccessorValidationChunkIndex() { - try { - String tmpDir = System.getProperty("java.io.tmpdir"); - String vdsPath = tmpDir + File.separator + tempVdsCopyFileName; - VDSFileOpenOptions options = new VDSFileOpenOptions(vdsPath); - VdsHandle vdsCopy = OpenVDS.open(options); - VolumeDataAccessManager accessManager = vdsCopy.getAccessManager(); - - int channel = 0; - VolumeDataLayout layout = vdsCopy.getLayout(); - VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( - DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND - 0, // lod - channel, // channel - 20, // max pages - VolumeDataPageAccessor.AccessMode.ReadOnly.getCode()); // access mode - - // compares block data - int[] chunkMin = new int[VolumeDataLayout.Dimensionality_Max]; - int[] chunkMax = new int[VolumeDataLayout.Dimensionality_Max]; - int[] chunkMaxPos = new int[VolumeDataLayout.Dimensionality_Max]; - - long chunkCount = pageAccessor.getChunkCount(); - for (long chunk = 0 ; chunk < chunkCount ; ++chunk) { - VolumeDataPage inputPage = pageAccessor.readPage(chunk); - - // check that chunk index matches current index - inputPage.getMinMaxExcludingMargin(chunkMin, chunkMax); - for (int i = 0 ; i < VolumeDataLayout.Dimensionality_Max ; ++i) { - chunkMaxPos[i] = chunkMax[i] != 0 ? chunkMax[i] - 1 : chunkMax[i]; - } - long idxChMin = pageAccessor.getChunkIndex(chunkMin); - long idxChMax = pageAccessor.getChunkIndex(chunkMaxPos); - - Assert.assertEquals(chunk, idxChMin); - Assert.assertEquals(idxChMin, idxChMax); - - inputPage.pageRelease(); - } - - accessManager.destroyVolumeDataPageAccessor(pageAccessor); - - vdsCopy.close(); - } - catch (java.io.IOException e) { - System.out.println(e.getMessage()); - fail(); - } - } -} -- GitLab From d9ae797d66e151fcedb36294ddc5ae53087c3f59 Mon Sep 17 00:00:00 2001 From: Julien Lacoste Date: Mon, 20 Sep 2021 10:02:31 +0200 Subject: [PATCH 17/25] Refactor class name --- java/java/demo/CreateVDSCloud.java | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/java/java/demo/CreateVDSCloud.java b/java/java/demo/CreateVDSCloud.java index d31b31e4..32a2faac 100644 --- a/java/java/demo/CreateVDSCloud.java +++ b/java/java/demo/CreateVDSCloud.java @@ -166,13 +166,13 @@ public class CreateVDSCloud { channelDescriptors.add(channelDescriptor); // options AWS - String pBucket = "miniovaap"; // the bucket of the VDS + String pBucket = ""; // the bucket of the VDS String pKey = name; // the key prefix of the VDS ? String pRegion = "eu-west-3"; // the region of the bucket of the VDS - String pEndpointOverride = "https://webvr.int.com/"; // This parameter allows to override the endpoint url + String pEndpointOverride = ""; // This parameter allows to override the endpoint url AWSOpenOptions optionsAWS = new AWSOpenOptions(pBucket, pKey, pRegion, pEndpointOverride); - optionsAWS.accessKeyId = "minioadmin"; // ? - optionsAWS.secretKey = "minioadmin"; + optionsAWS.accessKeyId = ""; // ? + optionsAWS.secretKey = ""; optionsAWS.sessionToken = ""; // ? // file options @@ -306,7 +306,7 @@ public class CreateVDSCloud { int errorCount = accessManager.uploadErrorCount(); if (errorCount > 0) { - ErrorInfo upErrInfo = accessManager.getCurrentUploadErrorInfo(); + ConnectionErrorInfo upErrInfo = accessManager.getCurrentUploadErrorInfo(); System.out.println("Upload error code : " + upErrInfo.getErrorCode()); if (upErrInfo.getErrorMessage() != null) { System.out.println("Upload error message : " + upErrInfo.getErrorMessage()); @@ -481,11 +481,11 @@ public class CreateVDSCloud { VolumeDataPage page = pageAccessor.readPage(chunk); float[] dataIn = inputPage.readFloatBuffer(pitchInput); - ErrorInfo dwnErrorInfo = accessManager.getCurrentDownloadErrorInfo(); - if (dwnErrorInfo.getErrorCode() != 0) { - System.out.println("Download error code : " + dwnErrorInfo.getErrorCode()); - if (dwnErrorInfo.getErrorMessage() != null) { - System.out.println("Download error message : " + dwnErrorInfo.getErrorMessage()); + ConnectionErrorInfo dwnConnectionErrorInfo = accessManager.getCurrentDownloadErrorInfo(); + if (dwnConnectionErrorInfo.getErrorCode() != 0) { + System.out.println("Download error code : " + dwnConnectionErrorInfo.getErrorCode()); + if (dwnConnectionErrorInfo.getErrorMessage() != null) { + System.out.println("Download error message : " + dwnConnectionErrorInfo.getErrorMessage()); } else { System.out.println("No error message"); } -- GitLab From c5406bf3d275e08dd47076780935006f7abf2cf3 Mon Sep 17 00:00:00 2001 From: Julien Lacoste Date: Tue, 24 Aug 2021 16:22:07 +0200 Subject: [PATCH 18/25] Java bindings : Add methods on VolumeDataAccessManager to get info on Upload/Download error --- java/CMakeLists.txt | 1 + .../src/org/opengroup/openvds/ErrorInfo.java | 44 +++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 java/java/src/org/opengroup/openvds/ErrorInfo.java diff --git a/java/CMakeLists.txt b/java/CMakeLists.txt index 7f954fcb..673537f5 100644 --- a/java/CMakeLists.txt +++ b/java/CMakeLists.txt @@ -21,6 +21,7 @@ set(JAVA_SOURCE_FILES java/src/org/opengroup/openvds/ConnectionErrorInfo.java java/src/org/opengroup/openvds/DimensionsND.java java/src/org/opengroup/openvds/experimental/VariousJavaTests.java + java/src/org/opengroup/openvds/ErrorInfo.java java/src/org/opengroup/openvds/GoogleOpenOptions.java java/src/org/opengroup/openvds/IndexRegionFloat.java java/src/org/opengroup/openvds/IndexRegionLong.java diff --git a/java/java/src/org/opengroup/openvds/ErrorInfo.java b/java/java/src/org/opengroup/openvds/ErrorInfo.java new file mode 100644 index 00000000..4dd9f017 --- /dev/null +++ b/java/java/src/org/opengroup/openvds/ErrorInfo.java @@ -0,0 +1,44 @@ +package org.opengroup.openvds; + +/** + * Simple class holding information on Upload/Download error + */ +public class ErrorInfo { + + private int errorCode; + private String errorMessage; + private String objectID; + + public ErrorInfo() { + + } + public ErrorInfo(int code, String msg, String id) { + errorCode = code; + errorMessage = msg; + objectID = id; + } + + public int getErrorCode() { + return errorCode; + } + + public void setErrorCode(int errorCode) { + this.errorCode = errorCode; + } + + public String getErrorMessage() { + return errorMessage; + } + + public void setErrorMessage(String errorMessage) { + this.errorMessage = errorMessage; + } + + public String getObjectID() { + return objectID; + } + + public void setObjectID(String objectID) { + this.objectID = objectID; + } +} -- GitLab From 0a8d75a9a77a732142bdaf156f4efee34217fba5 Mon Sep 17 00:00:00 2001 From: Julien Lacoste Date: Tue, 24 Aug 2021 16:28:16 +0200 Subject: [PATCH 19/25] Java bindings : Add methods on VolumeDataAccessManager to get info on Upload/Download error + fix wrong native call --- .../org/opengroup/openvds/VolumeDataAccessManager.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/java/java/src/org/opengroup/openvds/VolumeDataAccessManager.java b/java/java/src/org/opengroup/openvds/VolumeDataAccessManager.java index 506ac63e..f0be1ac7 100644 --- a/java/java/src/org/opengroup/openvds/VolumeDataAccessManager.java +++ b/java/java/src/org/opengroup/openvds/VolumeDataAccessManager.java @@ -779,10 +779,10 @@ public class VolumeDataAccessManager extends JniPointerWithoutDeletion { /** * @return current upload error information (code and message in a simple class) */ - public ConnectionErrorInfo getCurrentUploadErrorInfo() { - ConnectionErrorInfo connectionErrorInfo = new ConnectionErrorInfo(); - cpGetCurrentUploadErrorInfo(_handle, connectionErrorInfo); - return connectionErrorInfo; + public ErrorInfo getCurrentUploadErrorInfo() { + ErrorInfo errorInfo = new ErrorInfo(); + cpGetCurrentUploadErrorInfo(_handle, errorInfo); + return errorInfo; } /** -- GitLab From 535e4a66dabfeead3cd5263a393bc22b147b3c86 Mon Sep 17 00:00:00 2001 From: Julien Lacoste Date: Thu, 2 Sep 2021 12:17:34 +0200 Subject: [PATCH 20/25] bkp --- .../src/org/opengroup/openvds/ErrorInfo.java | 5 +- .../openvds/PageAccessorByteTest.java | 335 ++++++++++++++++++ 2 files changed, 337 insertions(+), 3 deletions(-) create mode 100644 java/java/test/org/opengroup/openvds/PageAccessorByteTest.java diff --git a/java/java/src/org/opengroup/openvds/ErrorInfo.java b/java/java/src/org/opengroup/openvds/ErrorInfo.java index 4dd9f017..fc9b81b7 100644 --- a/java/java/src/org/opengroup/openvds/ErrorInfo.java +++ b/java/java/src/org/opengroup/openvds/ErrorInfo.java @@ -5,13 +5,12 @@ package org.opengroup.openvds; */ public class ErrorInfo { - private int errorCode; + private int errorCode = -1; private String errorMessage; private String objectID; - public ErrorInfo() { + public ErrorInfo() { } - } public ErrorInfo(int code, String msg, String id) { errorCode = code; errorMessage = msg; diff --git a/java/java/test/org/opengroup/openvds/PageAccessorByteTest.java b/java/java/test/org/opengroup/openvds/PageAccessorByteTest.java new file mode 100644 index 00000000..4e84d0a0 --- /dev/null +++ b/java/java/test/org/opengroup/openvds/PageAccessorByteTest.java @@ -0,0 +1,335 @@ +/* + * Copyright 2021 The Open Group + * Copyright 2021 INT, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.openvds; + +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.io.File; + +import static org.testng.Assert.fail; + +public class PageAccessorByteTest { + + private static String TEMP_FILE_NAME_VOL_INDEX = "volIndexer"; + private static String TEMP_FILE_NAME_COPY = "vdsCopy"; + + public String url; + public VolumeDataLayoutDescriptor ld; + public VolumeDataAxisDescriptor[] vda; + public VolumeDataChannelDescriptor[] vdc; + public MetadataReadAccess md; + public MemoryVdsGenerator vds; + public MetadataContainer metadataContainer; + + private String tempVolIndexerFileName; + private String tempVdsCopyFileName; + + @BeforeClass + public void init() { + vds = new MemoryVdsGenerator(200, 200, 200, VolumeDataChannelDescriptor.Format.FORMAT_U8); + url = "inmemory://create_test"; + VolumeDataLayout volumeDataLayout = vds.getLayout(); + + int nbChannel = volumeDataLayout.getChannelCount(); + VolumeDataAccessManager accessManager = vds.getAccessManager(); + + for (VolumeDataLayoutDescriptor.LODLevels l : VolumeDataLayoutDescriptor.LODLevels.values()) { + for (int channel = 0; channel < nbChannel; channel++) { + for (DimensionsND dimGroup : DimensionsND.values()) { + VDSProduceStatus vdsProduceStatus = accessManager.getVDSProduceStatus(dimGroup, l.ordinal(), channel); + } + } + } + + vda = new VolumeDataAxisDescriptor[] { + volumeDataLayout.getAxisDescriptor(0), + volumeDataLayout.getAxisDescriptor(1), + volumeDataLayout.getAxisDescriptor(2)}; + vdc = new VolumeDataChannelDescriptor[] {volumeDataLayout.getChannelDescriptor(0)}; + + md = volumeDataLayout; + ld = volumeDataLayout.getLayoutDescriptor(); + + metadataContainer = new MetadataContainer(); + + long ms = System.currentTimeMillis(); + tempVolIndexerFileName = TEMP_FILE_NAME_VOL_INDEX + "_" + ms + ".vds"; + tempVdsCopyFileName = TEMP_FILE_NAME_COPY + "_" + ms + ".vds"; + } + + @AfterClass + public void cleanFiles() { + String tempDir = System.getProperty("java.io.tmpdir"); + String fileVolIndexPath = tempDir + File.separator + tempVolIndexerFileName; + File fileVolIndex = new File(fileVolIndexPath); + if (fileVolIndex.exists()) { + //fileVolIndex.delete(); + } + + String fileCopyPath = tempDir + File.separator + tempVdsCopyFileName; + File fileCopy = new File(fileCopyPath); + if (fileCopy.exists()) { + //fileCopy.delete(); + } + } + + @Test + public void testVolumeIndexerCreationDeletion() { + try { + // create file in tmp dir + String tmpDir = System.getProperty("java.io.tmpdir"); + String volIndexPath = tmpDir + File.separator + tempVolIndexerFileName; + VDSFileOpenOptions options = new VDSFileOpenOptions(volIndexPath); + VdsHandle vdsTest = OpenVDS.create(options, ld, + vda, + vdc, metadataContainer); + VolumeDataAccessManager accessManager = vdsTest.getAccessManager(); + //ASSERT_TRUE(accessManager); + + int channel = 0; + VolumeDataLayout layout = vdsTest.getLayout(); + VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( + DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND + 0, // lod + channel, // channel + 100, // max pages + VolumeDataPageAccessor.AccessMode.Create.getCode()); // access mode + + VolumeDataPage page = pageAccessor.createPage(0); + VolumeIndexer3D outputIndexer = new VolumeIndexer3D(page, 0, 0, DimensionsND.DIMENSIONS_012.ordinal(), layout); + outputIndexer.finalize(); + + vdsTest.close(); + } + catch (java.io.IOException e) { + System.out.println(e.getMessage()); + fail(); + } + } + + @Test + public void testCopyPageAccessor() { + try { + String tmpDir = System.getProperty("java.io.tmpdir"); + String vdsPath = tmpDir + File.separator + tempVdsCopyFileName; + VDSFileOpenOptions options = new VDSFileOpenOptions(vdsPath); + + // copy information from input VDS + VolumeDataLayoutDescriptor cpLd = new VolumeDataLayoutDescriptor( + ld.getBrickSize(), + ld.getNegativeMargin(), + ld.getPositiveMargin(), + ld.getBrickSizeMultiplier2D(), + ld.getLODLevels(), + ld.isCreate2DLODs(), + ld.isForceFullResolutionDimension(), + ld.getFullResolutionDimension() + ); + + VolumeDataAxisDescriptor[] cpVda = new VolumeDataAxisDescriptor[3]; + for (int i = 0 ; i < 3 ; ++i) { + cpVda[i] = new VolumeDataAxisDescriptor(vda[i].getNumSamples(), + vda[i].getName(), vda[i].getUnit(), + vda[i].getCoordinateMin(), vda[i].getCoordinateMax()); + } + + VolumeDataChannelDescriptor[] cpVdc = new VolumeDataChannelDescriptor[1]; + cpVdc[0] = new VolumeDataChannelDescriptor( + VolumeDataChannelDescriptor.Format.fromCode(vdc[0].getFormat()), + VolumeDataChannelDescriptor.Components.fromCode(vdc[0].getComponents()), + vdc[0].getName(), + vdc[0].getUnit(), + vdc[0].getValueRangeMin(), vdc[0].getValueRangeMax(), + VolumeDataMapping.fromCode(vdc[0].getMapping()), + vdc[0].getMappedValueCount(), + vdc[0].isDiscrete(), + vdc[0].isRenderable(), + vdc[0].isAllowLossyCompression(), + vdc[0].isUseZipForLosslessCompression(), + vdc[0].isUseNoValue(), + vdc[0].getNoValue(), + vdc[0].getIntegerScale(), + vdc[0].getIntegerOffset() + ); + MetadataContainer cpMetadataContainer = new MetadataContainer(); + VdsHandle vdsCopy = OpenVDS.create(options, cpLd, + cpVda, + cpVdc, + cpMetadataContainer); + + VolumeDataAccessManager accessManager = vdsCopy.getAccessManager(); + //ASSERT_TRUE(accessManager); + + int channel = 0; + VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( + DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND + 0, // lod + channel, // channel + 100, // max pages + VolumeDataPageAccessor.AccessMode.Create.getCode()); // access mode + + // get input manager + VolumeDataAccessManager inputAM = vds.getAccessManager(); + VolumeDataPageAccessor pageAccessorInput = inputAM.createVolumeDataPageAccessor( + DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND + 0, // lod + channel, // channel + 100, // max pages + VolumeDataPageAccessor.AccessMode.ReadOnly.getCode()); // access mode + + // copy file + int[] pitch = new int[VolumeDataLayout.Dimensionality_Max]; + long chunkCount = pageAccessorInput.getChunkCount(); + for (long chunk = 0 ; chunk < chunkCount ; ++chunk) { + VolumeDataPage inputPage = pageAccessorInput.readPage(chunk); + VolumeDataPage page = pageAccessor.createPage(chunk); + byte[] data = inputPage.readByteBuffer(pitch); + page.writeByteBuffer(data, pitch); + + inputPage.pageRelease(); + page.pageRelease(); + } + pageAccessor.commit(); + accessManager.flushUploadQueue(); + accessManager.destroyVolumeDataPageAccessor(pageAccessor); + + vdsCopy.close(); + } + catch (java.io.IOException e) { + System.out.println(e.getMessage()); + fail(); + } + } + + /** + * Will test that copied file is the same as the input + * Name of method is the same + Suffix so that it's executed after the copy test + */ + @Test + public void testCopyPageAccessorValidation() { + try { + String tmpDir = System.getProperty("java.io.tmpdir"); + String vdsPath = tmpDir + File.separator + tempVdsCopyFileName; + VDSFileOpenOptions options = new VDSFileOpenOptions(vdsPath); + VdsHandle vdsCopy = OpenVDS.open(options); + VolumeDataAccessManager accessManager = vdsCopy.getAccessManager(); + + int channel = 0; + VolumeDataLayout layout = vdsCopy.getLayout(); + VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( + DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND + 0, // lod + channel, // channel + 20, // max pages + VolumeDataPageAccessor.AccessMode.ReadOnly.getCode()); // access mode + + // get input manager + VolumeDataAccessManager inputAM = vds.getAccessManager(); + VolumeDataPageAccessor pageAccessorInput = inputAM.createVolumeDataPageAccessor( + DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND + 0, // lod + channel, // channel + 20, // max pages + VolumeDataPageAccessor.AccessMode.ReadOnly.getCode()); // access mode + + // compares block data + int[] pitchInput = new int[VolumeDataLayout.Dimensionality_Max]; + int[] pitchOutput = new int[VolumeDataLayout.Dimensionality_Max]; + + long chunkCount = pageAccessorInput.getChunkCount(); + for (long chunk = 0 ; chunk < chunkCount ; ++chunk) { + VolumeDataPage inputPage = pageAccessorInput.readPage(chunk); + VolumeDataPage page = pageAccessor.readPage(chunk); + byte[] dataIn = inputPage.readByteBuffer(pitchInput); + byte[] dataOut = page.readByteBuffer(pitchOutput); + + inputPage.pageRelease(); + page.pageRelease(); + + Assert.assertEquals(pitchInput, pitchOutput); + Assert.assertEquals(dataIn, dataOut); + } + + accessManager.destroyVolumeDataPageAccessor(pageAccessor); + + vdsCopy.close(); + } + catch (java.io.IOException e) { + System.out.println(e.getMessage()); + fail(); + } + } + + /** + * Will test that copied file has the same layout than original (test that 3D positions are in the same chunk index) + * Name of method is the same + Suffix so that it's executed after the copy test + */ + @Test + public void testCopyPageAccessorValidationChunkIndex() { + try { + String tmpDir = System.getProperty("java.io.tmpdir"); + String vdsPath = tmpDir + File.separator + tempVdsCopyFileName; + VDSFileOpenOptions options = new VDSFileOpenOptions(vdsPath); + VdsHandle vdsCopy = OpenVDS.open(options); + VolumeDataAccessManager accessManager = vdsCopy.getAccessManager(); + + int channel = 0; + VolumeDataLayout layout = vdsCopy.getLayout(); + VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( + DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND + 0, // lod + channel, // channel + 20, // max pages + VolumeDataPageAccessor.AccessMode.ReadOnly.getCode()); // access mode + + // compares block data + int[] chunkMin = new int[VolumeDataLayout.Dimensionality_Max]; + int[] chunkMax = new int[VolumeDataLayout.Dimensionality_Max]; + int[] chunkMaxPos = new int[VolumeDataLayout.Dimensionality_Max]; + + long chunkCount = pageAccessor.getChunkCount(); + for (long chunk = 0 ; chunk < chunkCount ; ++chunk) { + VolumeDataPage inputPage = pageAccessor.readPage(chunk); + + // check that chunk index matches current index + inputPage.getMinMaxExcludingMargin(chunkMin, chunkMax); + for (int i = 0 ; i < VolumeDataLayout.Dimensionality_Max ; ++i) { + chunkMaxPos[i] = chunkMax[i] != 0 ? chunkMax[i] - 1 : chunkMax[i]; + } + long idxChMin = pageAccessor.getChunkIndex(chunkMin); + long idxChMax = pageAccessor.getChunkIndex(chunkMaxPos); + + Assert.assertEquals(chunk, idxChMin); + Assert.assertEquals(idxChMin, idxChMax); + + inputPage.pageRelease(); + } + + accessManager.destroyVolumeDataPageAccessor(pageAccessor); + + vdsCopy.close(); + } + catch (java.io.IOException e) { + System.out.println(e.getMessage()); + fail(); + } + } +} -- GitLab From 7345872fe3c6783eb4bed15ee4d9a4e59b0b9fe2 Mon Sep 17 00:00:00 2001 From: Julien Lacoste Date: Thu, 9 Sep 2021 17:18:35 +0200 Subject: [PATCH 21/25] Rename ErrorInfo class --- java/CMakeLists.txt | 1 - .../src/org/opengroup/openvds/ErrorInfo.java | 43 ------------------- .../openvds/VolumeDataAccessManager.java | 8 ++-- 3 files changed, 4 insertions(+), 48 deletions(-) delete mode 100644 java/java/src/org/opengroup/openvds/ErrorInfo.java diff --git a/java/CMakeLists.txt b/java/CMakeLists.txt index 673537f5..7f954fcb 100644 --- a/java/CMakeLists.txt +++ b/java/CMakeLists.txt @@ -21,7 +21,6 @@ set(JAVA_SOURCE_FILES java/src/org/opengroup/openvds/ConnectionErrorInfo.java java/src/org/opengroup/openvds/DimensionsND.java java/src/org/opengroup/openvds/experimental/VariousJavaTests.java - java/src/org/opengroup/openvds/ErrorInfo.java java/src/org/opengroup/openvds/GoogleOpenOptions.java java/src/org/opengroup/openvds/IndexRegionFloat.java java/src/org/opengroup/openvds/IndexRegionLong.java diff --git a/java/java/src/org/opengroup/openvds/ErrorInfo.java b/java/java/src/org/opengroup/openvds/ErrorInfo.java deleted file mode 100644 index fc9b81b7..00000000 --- a/java/java/src/org/opengroup/openvds/ErrorInfo.java +++ /dev/null @@ -1,43 +0,0 @@ -package org.opengroup.openvds; - -/** - * Simple class holding information on Upload/Download error - */ -public class ErrorInfo { - - private int errorCode = -1; - private String errorMessage; - private String objectID; - - public ErrorInfo() { } - - public ErrorInfo(int code, String msg, String id) { - errorCode = code; - errorMessage = msg; - objectID = id; - } - - public int getErrorCode() { - return errorCode; - } - - public void setErrorCode(int errorCode) { - this.errorCode = errorCode; - } - - public String getErrorMessage() { - return errorMessage; - } - - public void setErrorMessage(String errorMessage) { - this.errorMessage = errorMessage; - } - - public String getObjectID() { - return objectID; - } - - public void setObjectID(String objectID) { - this.objectID = objectID; - } -} diff --git a/java/java/src/org/opengroup/openvds/VolumeDataAccessManager.java b/java/java/src/org/opengroup/openvds/VolumeDataAccessManager.java index f0be1ac7..506ac63e 100644 --- a/java/java/src/org/opengroup/openvds/VolumeDataAccessManager.java +++ b/java/java/src/org/opengroup/openvds/VolumeDataAccessManager.java @@ -779,10 +779,10 @@ public class VolumeDataAccessManager extends JniPointerWithoutDeletion { /** * @return current upload error information (code and message in a simple class) */ - public ErrorInfo getCurrentUploadErrorInfo() { - ErrorInfo errorInfo = new ErrorInfo(); - cpGetCurrentUploadErrorInfo(_handle, errorInfo); - return errorInfo; + public ConnectionErrorInfo getCurrentUploadErrorInfo() { + ConnectionErrorInfo connectionErrorInfo = new ConnectionErrorInfo(); + cpGetCurrentUploadErrorInfo(_handle, connectionErrorInfo); + return connectionErrorInfo; } /** -- GitLab From 51de538a0371f00beff7b2ec3a8603b2efc7daae Mon Sep 17 00:00:00 2001 From: Julien Lacoste Date: Thu, 9 Sep 2021 17:22:36 +0200 Subject: [PATCH 22/25] Remove temp test --- .../openvds/PageAccessorByteTest.java | 335 ------------------ 1 file changed, 335 deletions(-) delete mode 100644 java/java/test/org/opengroup/openvds/PageAccessorByteTest.java diff --git a/java/java/test/org/opengroup/openvds/PageAccessorByteTest.java b/java/java/test/org/opengroup/openvds/PageAccessorByteTest.java deleted file mode 100644 index 4e84d0a0..00000000 --- a/java/java/test/org/opengroup/openvds/PageAccessorByteTest.java +++ /dev/null @@ -1,335 +0,0 @@ -/* - * Copyright 2021 The Open Group - * Copyright 2021 INT, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opengroup.openvds; - -import org.testng.Assert; -import org.testng.annotations.AfterClass; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.Test; - -import java.io.File; - -import static org.testng.Assert.fail; - -public class PageAccessorByteTest { - - private static String TEMP_FILE_NAME_VOL_INDEX = "volIndexer"; - private static String TEMP_FILE_NAME_COPY = "vdsCopy"; - - public String url; - public VolumeDataLayoutDescriptor ld; - public VolumeDataAxisDescriptor[] vda; - public VolumeDataChannelDescriptor[] vdc; - public MetadataReadAccess md; - public MemoryVdsGenerator vds; - public MetadataContainer metadataContainer; - - private String tempVolIndexerFileName; - private String tempVdsCopyFileName; - - @BeforeClass - public void init() { - vds = new MemoryVdsGenerator(200, 200, 200, VolumeDataChannelDescriptor.Format.FORMAT_U8); - url = "inmemory://create_test"; - VolumeDataLayout volumeDataLayout = vds.getLayout(); - - int nbChannel = volumeDataLayout.getChannelCount(); - VolumeDataAccessManager accessManager = vds.getAccessManager(); - - for (VolumeDataLayoutDescriptor.LODLevels l : VolumeDataLayoutDescriptor.LODLevels.values()) { - for (int channel = 0; channel < nbChannel; channel++) { - for (DimensionsND dimGroup : DimensionsND.values()) { - VDSProduceStatus vdsProduceStatus = accessManager.getVDSProduceStatus(dimGroup, l.ordinal(), channel); - } - } - } - - vda = new VolumeDataAxisDescriptor[] { - volumeDataLayout.getAxisDescriptor(0), - volumeDataLayout.getAxisDescriptor(1), - volumeDataLayout.getAxisDescriptor(2)}; - vdc = new VolumeDataChannelDescriptor[] {volumeDataLayout.getChannelDescriptor(0)}; - - md = volumeDataLayout; - ld = volumeDataLayout.getLayoutDescriptor(); - - metadataContainer = new MetadataContainer(); - - long ms = System.currentTimeMillis(); - tempVolIndexerFileName = TEMP_FILE_NAME_VOL_INDEX + "_" + ms + ".vds"; - tempVdsCopyFileName = TEMP_FILE_NAME_COPY + "_" + ms + ".vds"; - } - - @AfterClass - public void cleanFiles() { - String tempDir = System.getProperty("java.io.tmpdir"); - String fileVolIndexPath = tempDir + File.separator + tempVolIndexerFileName; - File fileVolIndex = new File(fileVolIndexPath); - if (fileVolIndex.exists()) { - //fileVolIndex.delete(); - } - - String fileCopyPath = tempDir + File.separator + tempVdsCopyFileName; - File fileCopy = new File(fileCopyPath); - if (fileCopy.exists()) { - //fileCopy.delete(); - } - } - - @Test - public void testVolumeIndexerCreationDeletion() { - try { - // create file in tmp dir - String tmpDir = System.getProperty("java.io.tmpdir"); - String volIndexPath = tmpDir + File.separator + tempVolIndexerFileName; - VDSFileOpenOptions options = new VDSFileOpenOptions(volIndexPath); - VdsHandle vdsTest = OpenVDS.create(options, ld, - vda, - vdc, metadataContainer); - VolumeDataAccessManager accessManager = vdsTest.getAccessManager(); - //ASSERT_TRUE(accessManager); - - int channel = 0; - VolumeDataLayout layout = vdsTest.getLayout(); - VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( - DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND - 0, // lod - channel, // channel - 100, // max pages - VolumeDataPageAccessor.AccessMode.Create.getCode()); // access mode - - VolumeDataPage page = pageAccessor.createPage(0); - VolumeIndexer3D outputIndexer = new VolumeIndexer3D(page, 0, 0, DimensionsND.DIMENSIONS_012.ordinal(), layout); - outputIndexer.finalize(); - - vdsTest.close(); - } - catch (java.io.IOException e) { - System.out.println(e.getMessage()); - fail(); - } - } - - @Test - public void testCopyPageAccessor() { - try { - String tmpDir = System.getProperty("java.io.tmpdir"); - String vdsPath = tmpDir + File.separator + tempVdsCopyFileName; - VDSFileOpenOptions options = new VDSFileOpenOptions(vdsPath); - - // copy information from input VDS - VolumeDataLayoutDescriptor cpLd = new VolumeDataLayoutDescriptor( - ld.getBrickSize(), - ld.getNegativeMargin(), - ld.getPositiveMargin(), - ld.getBrickSizeMultiplier2D(), - ld.getLODLevels(), - ld.isCreate2DLODs(), - ld.isForceFullResolutionDimension(), - ld.getFullResolutionDimension() - ); - - VolumeDataAxisDescriptor[] cpVda = new VolumeDataAxisDescriptor[3]; - for (int i = 0 ; i < 3 ; ++i) { - cpVda[i] = new VolumeDataAxisDescriptor(vda[i].getNumSamples(), - vda[i].getName(), vda[i].getUnit(), - vda[i].getCoordinateMin(), vda[i].getCoordinateMax()); - } - - VolumeDataChannelDescriptor[] cpVdc = new VolumeDataChannelDescriptor[1]; - cpVdc[0] = new VolumeDataChannelDescriptor( - VolumeDataChannelDescriptor.Format.fromCode(vdc[0].getFormat()), - VolumeDataChannelDescriptor.Components.fromCode(vdc[0].getComponents()), - vdc[0].getName(), - vdc[0].getUnit(), - vdc[0].getValueRangeMin(), vdc[0].getValueRangeMax(), - VolumeDataMapping.fromCode(vdc[0].getMapping()), - vdc[0].getMappedValueCount(), - vdc[0].isDiscrete(), - vdc[0].isRenderable(), - vdc[0].isAllowLossyCompression(), - vdc[0].isUseZipForLosslessCompression(), - vdc[0].isUseNoValue(), - vdc[0].getNoValue(), - vdc[0].getIntegerScale(), - vdc[0].getIntegerOffset() - ); - MetadataContainer cpMetadataContainer = new MetadataContainer(); - VdsHandle vdsCopy = OpenVDS.create(options, cpLd, - cpVda, - cpVdc, - cpMetadataContainer); - - VolumeDataAccessManager accessManager = vdsCopy.getAccessManager(); - //ASSERT_TRUE(accessManager); - - int channel = 0; - VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( - DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND - 0, // lod - channel, // channel - 100, // max pages - VolumeDataPageAccessor.AccessMode.Create.getCode()); // access mode - - // get input manager - VolumeDataAccessManager inputAM = vds.getAccessManager(); - VolumeDataPageAccessor pageAccessorInput = inputAM.createVolumeDataPageAccessor( - DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND - 0, // lod - channel, // channel - 100, // max pages - VolumeDataPageAccessor.AccessMode.ReadOnly.getCode()); // access mode - - // copy file - int[] pitch = new int[VolumeDataLayout.Dimensionality_Max]; - long chunkCount = pageAccessorInput.getChunkCount(); - for (long chunk = 0 ; chunk < chunkCount ; ++chunk) { - VolumeDataPage inputPage = pageAccessorInput.readPage(chunk); - VolumeDataPage page = pageAccessor.createPage(chunk); - byte[] data = inputPage.readByteBuffer(pitch); - page.writeByteBuffer(data, pitch); - - inputPage.pageRelease(); - page.pageRelease(); - } - pageAccessor.commit(); - accessManager.flushUploadQueue(); - accessManager.destroyVolumeDataPageAccessor(pageAccessor); - - vdsCopy.close(); - } - catch (java.io.IOException e) { - System.out.println(e.getMessage()); - fail(); - } - } - - /** - * Will test that copied file is the same as the input - * Name of method is the same + Suffix so that it's executed after the copy test - */ - @Test - public void testCopyPageAccessorValidation() { - try { - String tmpDir = System.getProperty("java.io.tmpdir"); - String vdsPath = tmpDir + File.separator + tempVdsCopyFileName; - VDSFileOpenOptions options = new VDSFileOpenOptions(vdsPath); - VdsHandle vdsCopy = OpenVDS.open(options); - VolumeDataAccessManager accessManager = vdsCopy.getAccessManager(); - - int channel = 0; - VolumeDataLayout layout = vdsCopy.getLayout(); - VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( - DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND - 0, // lod - channel, // channel - 20, // max pages - VolumeDataPageAccessor.AccessMode.ReadOnly.getCode()); // access mode - - // get input manager - VolumeDataAccessManager inputAM = vds.getAccessManager(); - VolumeDataPageAccessor pageAccessorInput = inputAM.createVolumeDataPageAccessor( - DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND - 0, // lod - channel, // channel - 20, // max pages - VolumeDataPageAccessor.AccessMode.ReadOnly.getCode()); // access mode - - // compares block data - int[] pitchInput = new int[VolumeDataLayout.Dimensionality_Max]; - int[] pitchOutput = new int[VolumeDataLayout.Dimensionality_Max]; - - long chunkCount = pageAccessorInput.getChunkCount(); - for (long chunk = 0 ; chunk < chunkCount ; ++chunk) { - VolumeDataPage inputPage = pageAccessorInput.readPage(chunk); - VolumeDataPage page = pageAccessor.readPage(chunk); - byte[] dataIn = inputPage.readByteBuffer(pitchInput); - byte[] dataOut = page.readByteBuffer(pitchOutput); - - inputPage.pageRelease(); - page.pageRelease(); - - Assert.assertEquals(pitchInput, pitchOutput); - Assert.assertEquals(dataIn, dataOut); - } - - accessManager.destroyVolumeDataPageAccessor(pageAccessor); - - vdsCopy.close(); - } - catch (java.io.IOException e) { - System.out.println(e.getMessage()); - fail(); - } - } - - /** - * Will test that copied file has the same layout than original (test that 3D positions are in the same chunk index) - * Name of method is the same + Suffix so that it's executed after the copy test - */ - @Test - public void testCopyPageAccessorValidationChunkIndex() { - try { - String tmpDir = System.getProperty("java.io.tmpdir"); - String vdsPath = tmpDir + File.separator + tempVdsCopyFileName; - VDSFileOpenOptions options = new VDSFileOpenOptions(vdsPath); - VdsHandle vdsCopy = OpenVDS.open(options); - VolumeDataAccessManager accessManager = vdsCopy.getAccessManager(); - - int channel = 0; - VolumeDataLayout layout = vdsCopy.getLayout(); - VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( - DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND - 0, // lod - channel, // channel - 20, // max pages - VolumeDataPageAccessor.AccessMode.ReadOnly.getCode()); // access mode - - // compares block data - int[] chunkMin = new int[VolumeDataLayout.Dimensionality_Max]; - int[] chunkMax = new int[VolumeDataLayout.Dimensionality_Max]; - int[] chunkMaxPos = new int[VolumeDataLayout.Dimensionality_Max]; - - long chunkCount = pageAccessor.getChunkCount(); - for (long chunk = 0 ; chunk < chunkCount ; ++chunk) { - VolumeDataPage inputPage = pageAccessor.readPage(chunk); - - // check that chunk index matches current index - inputPage.getMinMaxExcludingMargin(chunkMin, chunkMax); - for (int i = 0 ; i < VolumeDataLayout.Dimensionality_Max ; ++i) { - chunkMaxPos[i] = chunkMax[i] != 0 ? chunkMax[i] - 1 : chunkMax[i]; - } - long idxChMin = pageAccessor.getChunkIndex(chunkMin); - long idxChMax = pageAccessor.getChunkIndex(chunkMaxPos); - - Assert.assertEquals(chunk, idxChMin); - Assert.assertEquals(idxChMin, idxChMax); - - inputPage.pageRelease(); - } - - accessManager.destroyVolumeDataPageAccessor(pageAccessor); - - vdsCopy.close(); - } - catch (java.io.IOException e) { - System.out.println(e.getMessage()); - fail(); - } - } -} -- GitLab From 4778ea109b76ccba67d04725f4ad806d9451387a Mon Sep 17 00:00:00 2001 From: Julien Lacoste Date: Tue, 21 Sep 2021 10:12:34 +0200 Subject: [PATCH 23/25] VolumeDataPage : add bindings for short data + adds test for byte/short copy and read --- java/CMakeLists.txt | 4 +- java/cpp/src/VolumeDataPage.cpp | 41 +++ .../org/opengroup/openvds/VolumeDataPage.java | 32 ++ .../openvds/PageAccessorByteTest.java | 335 ++++++++++++++++++ ...orTest.java => PageAccessorFloatTest.java} | 4 +- .../openvds/PageAccessorShortTest.java | 335 ++++++++++++++++++ 6 files changed, 748 insertions(+), 3 deletions(-) create mode 100644 java/java/test/org/opengroup/openvds/PageAccessorByteTest.java rename java/java/test/org/opengroup/openvds/{PageAccessorTest.java => PageAccessorFloatTest.java} (99%) create mode 100644 java/java/test/org/opengroup/openvds/PageAccessorShortTest.java diff --git a/java/CMakeLists.txt b/java/CMakeLists.txt index 7f954fcb..e18a35bc 100644 --- a/java/CMakeLists.txt +++ b/java/CMakeLists.txt @@ -98,7 +98,9 @@ set(JAVA_TEST_SOURCES_FILES java/test/org/opengroup/openvds/CreateVDSTest.java java/test/org/opengroup/openvds/WriteDataTest.java java/test/org/opengroup/openvds/MetaDataContainerTest.java - java/test/org/opengroup/openvds/PageAccessorTest.java + java/test/org/opengroup/openvds/PageAccessorByteTest.java + java/test/org/opengroup/openvds/PageAccessorFloatTest.java + java/test/org/opengroup/openvds/PageAccessorShortTest.java ) add_jar(openvds-java-test diff --git a/java/cpp/src/VolumeDataPage.cpp b/java/cpp/src/VolumeDataPage.cpp index 5309d03d..3b1cc843 100644 --- a/java/cpp/src/VolumeDataPage.cpp +++ b/java/cpp/src/VolumeDataPage.cpp @@ -242,6 +242,47 @@ extern "C" { CATCH_EXCEPTIONS_FOR_JAVA; } + /* + * Class: org_opengroup_openvds_VolumeDataPage + * Method: cpGetShortBuffer + * Signature: (J[II)[S + */ + JNIEXPORT jshortArray JNICALL Java_org_opengroup_openvds_VolumeDataPage_cpGetShortBuffer + (JNIEnv * env, jclass, jlong handle, jintArray pitchParam, jint lod) + { + try { + int pitch[OpenVDS::Dimensionality_Max]; + OpenVDS::VolumeDataPage* page = GetVolumePage(handle); + + const short* readData = (const short*)page->GetBuffer(pitch); + env->SetIntArrayRegion(pitchParam, 0, OpenVDS::Dimensionality_Max, (jint *)pitch); + int nbElem = GetBufferAllocatedSize(page); + return NewJShortArray(env, readData, nbElem); + } + CATCH_EXCEPTIONS_FOR_JAVA; + return NULL; + } + + /* + * Class: org_opengroup_openvds_VolumeDataPage + * Method: cpSetShortBuffer + * Signature: (J[S)V + */ + JNIEXPORT void JNICALL Java_org_opengroup_openvds_VolumeDataPage_cpSetShortBuffer + (JNIEnv * env, jclass, jlong handle, jshortArray values) + { + try { + OpenVDS::VolumeDataPage * page = GetVolumePage(handle); + int pitch[OpenVDS::Dimensionality_Max]; + short* pageBuffer = (short*)page->GetWritableBuffer(pitch); + int valueSize = env->GetArrayLength(values); + jshort *src = env->GetShortArrayElements(values, NULL); + std::memcpy(pageBuffer, src, valueSize * sizeof (short)); + env->ReleaseShortArrayElements(values, src, 0); + } + CATCH_EXCEPTIONS_FOR_JAVA; + } + #ifdef __cplusplus } #endif diff --git a/java/java/src/org/opengroup/openvds/VolumeDataPage.java b/java/java/src/org/opengroup/openvds/VolumeDataPage.java index 262cce4e..c80203c3 100644 --- a/java/java/src/org/opengroup/openvds/VolumeDataPage.java +++ b/java/java/src/org/opengroup/openvds/VolumeDataPage.java @@ -41,6 +41,11 @@ public class VolumeDataPage extends JniPointerWithoutDeletion { private static native void cpSetDoubleBuffer(long handle, double[] buffer); + private static native short[] cpGetShortBuffer(long handle, int[] pitch, int lod); + + private static native void cpSetShortBuffer(long handle, short[] buffer); + + private final int dimensionality; private final int lod; @@ -144,6 +149,26 @@ public class VolumeDataPage extends JniPointerWithoutDeletion { cpSetFloatBuffer(_handle, buffer); } + /** + * Read short array of page + * @param pitch will receive pitch values for this page + * @return the short array of page data + */ + public short[] readShortBuffer(int[] pitch) { + checkDimParamArray(pitch, "Wrong pitch array parameter size, expected "); + return cpGetShortBuffer(_handle, pitch, lod); + } + + /** + * Set short array int page + * @param buffer values to set. Size must match page sample size + * @param pitch chunk pitch (got by a read) + */ + public void writeShortBuffer(short[] buffer, int[] pitch) { + checkBufferSize(buffer, pitch, dimensionality, lod); + cpSetShortBuffer(_handle, buffer); + } + /** * Read double array of page * @param pitch will receive pitch values for this page @@ -192,6 +217,13 @@ public class VolumeDataPage extends JniPointerWithoutDeletion { checkBufferSize(buffer.length, pitch, dimensionality, lod); } + private void checkBufferSize(short[] buffer, int[] pitch, int dim, int lod) { + if (buffer == null) { + throw new IllegalArgumentException("Wrong buffer size, got NULL"); + } + checkBufferSize(buffer.length, pitch, dimensionality, lod); + } + private void checkBufferSize(int sizeInputBuffer, int[] pitch, int dim, int lod) { int pageBufferSize = getAllocatedBufferSize(); if (sizeInputBuffer != pageBufferSize) { diff --git a/java/java/test/org/opengroup/openvds/PageAccessorByteTest.java b/java/java/test/org/opengroup/openvds/PageAccessorByteTest.java new file mode 100644 index 00000000..05a1b344 --- /dev/null +++ b/java/java/test/org/opengroup/openvds/PageAccessorByteTest.java @@ -0,0 +1,335 @@ +/* + * Copyright 2021 The Open Group + * Copyright 2021 INT, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.openvds; + +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.io.File; + +import static org.testng.Assert.fail; + +public class PageAccessorByteTest { + + private static String TEMP_FILE_NAME_VOL_INDEX = "volIndexer"; + private static String TEMP_FILE_NAME_COPY = "vdsCopyByte"; + + public String url; + public VolumeDataLayoutDescriptor ld; + public VolumeDataAxisDescriptor[] vda; + public VolumeDataChannelDescriptor[] vdc; + public MetadataReadAccess md; + public MemoryVdsGenerator vds; + public MetadataContainer metadataContainer; + + private String tempVolIndexerFileName; + private String tempVdsCopyFileName; + + @BeforeClass + public void init() { + vds = new MemoryVdsGenerator(200, 200, 200, VolumeDataChannelDescriptor.Format.FORMAT_U8); + url = "inmemory://create_test"; + VolumeDataLayout volumeDataLayout = vds.getLayout(); + + int nbChannel = volumeDataLayout.getChannelCount(); + VolumeDataAccessManager accessManager = vds.getAccessManager(); + + for (VolumeDataLayoutDescriptor.LODLevels l : VolumeDataLayoutDescriptor.LODLevels.values()) { + for (int channel = 0; channel < nbChannel; channel++) { + for (DimensionsND dimGroup : DimensionsND.values()) { + VDSProduceStatus vdsProduceStatus = accessManager.getVDSProduceStatus(dimGroup, l.ordinal(), channel); + } + } + } + + vda = new VolumeDataAxisDescriptor[] { + volumeDataLayout.getAxisDescriptor(0), + volumeDataLayout.getAxisDescriptor(1), + volumeDataLayout.getAxisDescriptor(2)}; + vdc = new VolumeDataChannelDescriptor[] {volumeDataLayout.getChannelDescriptor(0)}; + + md = volumeDataLayout; + ld = volumeDataLayout.getLayoutDescriptor(); + + metadataContainer = new MetadataContainer(); + + long ms = System.currentTimeMillis(); + tempVolIndexerFileName = TEMP_FILE_NAME_VOL_INDEX + "_" + ms + ".vds"; + tempVdsCopyFileName = TEMP_FILE_NAME_COPY + "_" + ms + ".vds"; + } + + @AfterClass + public void cleanFiles() { + String tempDir = System.getProperty("java.io.tmpdir"); + String fileVolIndexPath = tempDir + File.separator + tempVolIndexerFileName; + File fileVolIndex = new File(fileVolIndexPath); + if (fileVolIndex.exists()) { + fileVolIndex.delete(); + } + + String fileCopyPath = tempDir + File.separator + tempVdsCopyFileName; + File fileCopy = new File(fileCopyPath); + if (fileCopy.exists()) { + fileCopy.delete(); + } + } + + @Test + public void testVolumeIndexerCreationDeletion() { + try { + // create file in tmp dir + String tmpDir = System.getProperty("java.io.tmpdir"); + String volIndexPath = tmpDir + File.separator + tempVolIndexerFileName; + VDSFileOpenOptions options = new VDSFileOpenOptions(volIndexPath); + VdsHandle vdsTest = OpenVDS.create(options, ld, + vda, + vdc, metadataContainer); + VolumeDataAccessManager accessManager = vdsTest.getAccessManager(); + //ASSERT_TRUE(accessManager); + + int channel = 0; + VolumeDataLayout layout = vdsTest.getLayout(); + VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( + DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND + 0, // lod + channel, // channel + 100, // max pages + VolumeDataPageAccessor.AccessMode.Create.getCode()); // access mode + + VolumeDataPage page = pageAccessor.createPage(0); + VolumeIndexer3D outputIndexer = new VolumeIndexer3D(page, 0, 0, DimensionsND.DIMENSIONS_012.ordinal(), layout); + outputIndexer.finalize(); + + vdsTest.close(); + } + catch (java.io.IOException e) { + System.out.println(e.getMessage()); + fail(); + } + } + + @Test + public void testCopyPageAccessor() { + try { + String tmpDir = System.getProperty("java.io.tmpdir"); + String vdsPath = tmpDir + File.separator + tempVdsCopyFileName; + VDSFileOpenOptions options = new VDSFileOpenOptions(vdsPath); + + // copy information from input VDS + VolumeDataLayoutDescriptor cpLd = new VolumeDataLayoutDescriptor( + ld.getBrickSize(), + ld.getNegativeMargin(), + ld.getPositiveMargin(), + ld.getBrickSizeMultiplier2D(), + ld.getLODLevels(), + ld.isCreate2DLODs(), + ld.isForceFullResolutionDimension(), + ld.getFullResolutionDimension() + ); + + VolumeDataAxisDescriptor[] cpVda = new VolumeDataAxisDescriptor[3]; + for (int i = 0 ; i < 3 ; ++i) { + cpVda[i] = new VolumeDataAxisDescriptor(vda[i].getNumSamples(), + vda[i].getName(), vda[i].getUnit(), + vda[i].getCoordinateMin(), vda[i].getCoordinateMax()); + } + + VolumeDataChannelDescriptor[] cpVdc = new VolumeDataChannelDescriptor[1]; + cpVdc[0] = new VolumeDataChannelDescriptor( + VolumeDataChannelDescriptor.Format.fromCode(vdc[0].getFormat()), + VolumeDataChannelDescriptor.Components.fromCode(vdc[0].getComponents()), + vdc[0].getName(), + vdc[0].getUnit(), + vdc[0].getValueRangeMin(), vdc[0].getValueRangeMax(), + VolumeDataMapping.fromCode(vdc[0].getMapping()), + vdc[0].getMappedValueCount(), + vdc[0].isDiscrete(), + vdc[0].isRenderable(), + vdc[0].isAllowLossyCompression(), + vdc[0].isUseZipForLosslessCompression(), + vdc[0].isUseNoValue(), + vdc[0].getNoValue(), + vdc[0].getIntegerScale(), + vdc[0].getIntegerOffset() + ); + MetadataContainer cpMetadataContainer = new MetadataContainer(); + VdsHandle vdsCopy = OpenVDS.create(options, cpLd, + cpVda, + cpVdc, + cpMetadataContainer); + + VolumeDataAccessManager accessManager = vdsCopy.getAccessManager(); + //ASSERT_TRUE(accessManager); + + int channel = 0; + VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( + DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND + 0, // lod + channel, // channel + 100, // max pages + VolumeDataPageAccessor.AccessMode.Create.getCode()); // access mode + + // get input manager + VolumeDataAccessManager inputAM = vds.getAccessManager(); + VolumeDataPageAccessor pageAccessorInput = inputAM.createVolumeDataPageAccessor( + DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND + 0, // lod + channel, // channel + 100, // max pages + VolumeDataPageAccessor.AccessMode.ReadOnly.getCode()); // access mode + + // copy file + int[] pitch = new int[VolumeDataLayout.Dimensionality_Max]; + long chunkCount = pageAccessorInput.getChunkCount(); + for (long chunk = 0 ; chunk < chunkCount ; ++chunk) { + VolumeDataPage inputPage = pageAccessorInput.readPage(chunk); + VolumeDataPage page = pageAccessor.createPage(chunk); + byte[] data = inputPage.readByteBuffer(pitch); + page.writeByteBuffer(data, pitch); + + inputPage.pageRelease(); + page.pageRelease(); + } + pageAccessor.commit(); + accessManager.flushUploadQueue(); + accessManager.destroyVolumeDataPageAccessor(pageAccessor); + + vdsCopy.close(); + } + catch (java.io.IOException e) { + System.out.println(e.getMessage()); + fail(); + } + } + + /** + * Will test that copied file is the same as the input + * Name of method is the same + Suffix so that it's executed after the copy test + */ + @Test + public void testCopyPageAccessorValidation() { + try { + String tmpDir = System.getProperty("java.io.tmpdir"); + String vdsPath = tmpDir + File.separator + tempVdsCopyFileName; + VDSFileOpenOptions options = new VDSFileOpenOptions(vdsPath); + VdsHandle vdsCopy = OpenVDS.open(options); + VolumeDataAccessManager accessManager = vdsCopy.getAccessManager(); + + int channel = 0; + VolumeDataLayout layout = vdsCopy.getLayout(); + VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( + DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND + 0, // lod + channel, // channel + 20, // max pages + VolumeDataPageAccessor.AccessMode.ReadOnly.getCode()); // access mode + + // get input manager + VolumeDataAccessManager inputAM = vds.getAccessManager(); + VolumeDataPageAccessor pageAccessorInput = inputAM.createVolumeDataPageAccessor( + DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND + 0, // lod + channel, // channel + 20, // max pages + VolumeDataPageAccessor.AccessMode.ReadOnly.getCode()); // access mode + + // compares block data + int[] pitchInput = new int[VolumeDataLayout.Dimensionality_Max]; + int[] pitchOutput = new int[VolumeDataLayout.Dimensionality_Max]; + + long chunkCount = pageAccessorInput.getChunkCount(); + for (long chunk = 0 ; chunk < chunkCount ; ++chunk) { + VolumeDataPage inputPage = pageAccessorInput.readPage(chunk); + VolumeDataPage page = pageAccessor.readPage(chunk); + byte[] dataIn = inputPage.readByteBuffer(pitchInput); + byte[] dataOut = page.readByteBuffer(pitchOutput); + + inputPage.pageRelease(); + page.pageRelease(); + + Assert.assertEquals(pitchInput, pitchOutput); + Assert.assertEquals(dataIn, dataOut); + } + + accessManager.destroyVolumeDataPageAccessor(pageAccessor); + + vdsCopy.close(); + } + catch (java.io.IOException e) { + System.out.println(e.getMessage()); + fail(); + } + } + + /** + * Will test that copied file has the same layout than original (test that 3D positions are in the same chunk index) + * Name of method is the same + Suffix so that it's executed after the copy test + */ + @Test + public void testCopyPageAccessorValidationChunkIndex() { + try { + String tmpDir = System.getProperty("java.io.tmpdir"); + String vdsPath = tmpDir + File.separator + tempVdsCopyFileName; + VDSFileOpenOptions options = new VDSFileOpenOptions(vdsPath); + VdsHandle vdsCopy = OpenVDS.open(options); + VolumeDataAccessManager accessManager = vdsCopy.getAccessManager(); + + int channel = 0; + VolumeDataLayout layout = vdsCopy.getLayout(); + VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( + DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND + 0, // lod + channel, // channel + 20, // max pages + VolumeDataPageAccessor.AccessMode.ReadOnly.getCode()); // access mode + + // compares block data + int[] chunkMin = new int[VolumeDataLayout.Dimensionality_Max]; + int[] chunkMax = new int[VolumeDataLayout.Dimensionality_Max]; + int[] chunkMaxPos = new int[VolumeDataLayout.Dimensionality_Max]; + + long chunkCount = pageAccessor.getChunkCount(); + for (long chunk = 0 ; chunk < chunkCount ; ++chunk) { + VolumeDataPage inputPage = pageAccessor.readPage(chunk); + + // check that chunk index matches current index + inputPage.getMinMaxExcludingMargin(chunkMin, chunkMax); + for (int i = 0 ; i < VolumeDataLayout.Dimensionality_Max ; ++i) { + chunkMaxPos[i] = chunkMax[i] != 0 ? chunkMax[i] - 1 : chunkMax[i]; + } + long idxChMin = pageAccessor.getChunkIndex(chunkMin); + long idxChMax = pageAccessor.getChunkIndex(chunkMaxPos); + + Assert.assertEquals(chunk, idxChMin); + Assert.assertEquals(idxChMin, idxChMax); + + inputPage.pageRelease(); + } + + accessManager.destroyVolumeDataPageAccessor(pageAccessor); + + vdsCopy.close(); + } + catch (java.io.IOException e) { + System.out.println(e.getMessage()); + fail(); + } + } +} diff --git a/java/java/test/org/opengroup/openvds/PageAccessorTest.java b/java/java/test/org/opengroup/openvds/PageAccessorFloatTest.java similarity index 99% rename from java/java/test/org/opengroup/openvds/PageAccessorTest.java rename to java/java/test/org/opengroup/openvds/PageAccessorFloatTest.java index f1fa1f1f..00cb0900 100644 --- a/java/java/test/org/opengroup/openvds/PageAccessorTest.java +++ b/java/java/test/org/opengroup/openvds/PageAccessorFloatTest.java @@ -27,10 +27,10 @@ import java.io.File; import static org.testng.Assert.fail; -public class PageAccessorTest { +public class PageAccessorFloatTest { private static String TEMP_FILE_NAME_VOL_INDEX = "volIndexer"; - private static String TEMP_FILE_NAME_COPY = "vdsCopy"; + private static String TEMP_FILE_NAME_COPY = "vdsFloatCopy"; public String url; public VolumeDataLayoutDescriptor ld; diff --git a/java/java/test/org/opengroup/openvds/PageAccessorShortTest.java b/java/java/test/org/opengroup/openvds/PageAccessorShortTest.java new file mode 100644 index 00000000..15990e47 --- /dev/null +++ b/java/java/test/org/opengroup/openvds/PageAccessorShortTest.java @@ -0,0 +1,335 @@ +/* + * Copyright 2021 The Open Group + * Copyright 2021 INT, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.openvds; + +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.io.File; + +import static org.testng.Assert.fail; + +public class PageAccessorShortTest { + + private static String TEMP_FILE_NAME_VOL_INDEX = "volIndexer"; + private static String TEMP_FILE_NAME_COPY = "vdsCopyShort"; + + public String url; + public VolumeDataLayoutDescriptor ld; + public VolumeDataAxisDescriptor[] vda; + public VolumeDataChannelDescriptor[] vdc; + public MetadataReadAccess md; + public MemoryVdsGenerator vds; + public MetadataContainer metadataContainer; + + private String tempVolIndexerFileName; + private String tempVdsCopyFileName; + + @BeforeClass + public void init() { + vds = new MemoryVdsGenerator(200, 200, 200, VolumeDataChannelDescriptor.Format.FORMAT_U16); + url = "inmemory://create_test"; + VolumeDataLayout volumeDataLayout = vds.getLayout(); + + int nbChannel = volumeDataLayout.getChannelCount(); + VolumeDataAccessManager accessManager = vds.getAccessManager(); + + for (VolumeDataLayoutDescriptor.LODLevels l : VolumeDataLayoutDescriptor.LODLevels.values()) { + for (int channel = 0; channel < nbChannel; channel++) { + for (DimensionsND dimGroup : DimensionsND.values()) { + VDSProduceStatus vdsProduceStatus = accessManager.getVDSProduceStatus(dimGroup, l.ordinal(), channel); + } + } + } + + vda = new VolumeDataAxisDescriptor[] { + volumeDataLayout.getAxisDescriptor(0), + volumeDataLayout.getAxisDescriptor(1), + volumeDataLayout.getAxisDescriptor(2)}; + vdc = new VolumeDataChannelDescriptor[] {volumeDataLayout.getChannelDescriptor(0)}; + + md = volumeDataLayout; + ld = volumeDataLayout.getLayoutDescriptor(); + + metadataContainer = new MetadataContainer(); + + long ms = System.currentTimeMillis(); + tempVolIndexerFileName = TEMP_FILE_NAME_VOL_INDEX + "_" + ms + ".vds"; + tempVdsCopyFileName = TEMP_FILE_NAME_COPY + "_" + ms + ".vds"; + } + + @AfterClass + public void cleanFiles() { + String tempDir = System.getProperty("java.io.tmpdir"); + String fileVolIndexPath = tempDir + File.separator + tempVolIndexerFileName; + File fileVolIndex = new File(fileVolIndexPath); + if (fileVolIndex.exists()) { + fileVolIndex.delete(); + } + + String fileCopyPath = tempDir + File.separator + tempVdsCopyFileName; + File fileCopy = new File(fileCopyPath); + if (fileCopy.exists()) { + fileCopy.delete(); + } + } + + @Test + public void testVolumeIndexerCreationDeletion() { + try { + // create file in tmp dir + String tmpDir = System.getProperty("java.io.tmpdir"); + String volIndexPath = tmpDir + File.separator + tempVolIndexerFileName; + VDSFileOpenOptions options = new VDSFileOpenOptions(volIndexPath); + VdsHandle vdsTest = OpenVDS.create(options, ld, + vda, + vdc, metadataContainer); + VolumeDataAccessManager accessManager = vdsTest.getAccessManager(); + //ASSERT_TRUE(accessManager); + + int channel = 0; + VolumeDataLayout layout = vdsTest.getLayout(); + VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( + DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND + 0, // lod + channel, // channel + 100, // max pages + VolumeDataPageAccessor.AccessMode.Create.getCode()); // access mode + + VolumeDataPage page = pageAccessor.createPage(0); + VolumeIndexer3D outputIndexer = new VolumeIndexer3D(page, 0, 0, DimensionsND.DIMENSIONS_012.ordinal(), layout); + outputIndexer.finalize(); + + vdsTest.close(); + } + catch (java.io.IOException e) { + System.out.println(e.getMessage()); + fail(); + } + } + + @Test + public void testCopyPageAccessor() { + try { + String tmpDir = System.getProperty("java.io.tmpdir"); + String vdsPath = tmpDir + File.separator + tempVdsCopyFileName; + VDSFileOpenOptions options = new VDSFileOpenOptions(vdsPath); + + // copy information from input VDS + VolumeDataLayoutDescriptor cpLd = new VolumeDataLayoutDescriptor( + ld.getBrickSize(), + ld.getNegativeMargin(), + ld.getPositiveMargin(), + ld.getBrickSizeMultiplier2D(), + ld.getLODLevels(), + ld.isCreate2DLODs(), + ld.isForceFullResolutionDimension(), + ld.getFullResolutionDimension() + ); + + VolumeDataAxisDescriptor[] cpVda = new VolumeDataAxisDescriptor[3]; + for (int i = 0 ; i < 3 ; ++i) { + cpVda[i] = new VolumeDataAxisDescriptor(vda[i].getNumSamples(), + vda[i].getName(), vda[i].getUnit(), + vda[i].getCoordinateMin(), vda[i].getCoordinateMax()); + } + + VolumeDataChannelDescriptor[] cpVdc = new VolumeDataChannelDescriptor[1]; + cpVdc[0] = new VolumeDataChannelDescriptor( + VolumeDataChannelDescriptor.Format.fromCode(vdc[0].getFormat()), + VolumeDataChannelDescriptor.Components.fromCode(vdc[0].getComponents()), + vdc[0].getName(), + vdc[0].getUnit(), + vdc[0].getValueRangeMin(), vdc[0].getValueRangeMax(), + VolumeDataMapping.fromCode(vdc[0].getMapping()), + vdc[0].getMappedValueCount(), + vdc[0].isDiscrete(), + vdc[0].isRenderable(), + vdc[0].isAllowLossyCompression(), + vdc[0].isUseZipForLosslessCompression(), + vdc[0].isUseNoValue(), + vdc[0].getNoValue(), + vdc[0].getIntegerScale(), + vdc[0].getIntegerOffset() + ); + MetadataContainer cpMetadataContainer = new MetadataContainer(); + VdsHandle vdsCopy = OpenVDS.create(options, cpLd, + cpVda, + cpVdc, + cpMetadataContainer); + + VolumeDataAccessManager accessManager = vdsCopy.getAccessManager(); + //ASSERT_TRUE(accessManager); + + int channel = 0; + VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( + DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND + 0, // lod + channel, // channel + 100, // max pages + VolumeDataPageAccessor.AccessMode.Create.getCode()); // access mode + + // get input manager + VolumeDataAccessManager inputAM = vds.getAccessManager(); + VolumeDataPageAccessor pageAccessorInput = inputAM.createVolumeDataPageAccessor( + DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND + 0, // lod + channel, // channel + 100, // max pages + VolumeDataPageAccessor.AccessMode.ReadOnly.getCode()); // access mode + + // copy file + int[] pitch = new int[VolumeDataLayout.Dimensionality_Max]; + long chunkCount = pageAccessorInput.getChunkCount(); + for (long chunk = 0 ; chunk < chunkCount ; ++chunk) { + VolumeDataPage inputPage = pageAccessorInput.readPage(chunk); + VolumeDataPage page = pageAccessor.createPage(chunk); + short[] data = inputPage.readShortBuffer(pitch); + page.writeShortBuffer(data, pitch); + + inputPage.pageRelease(); + page.pageRelease(); + } + pageAccessor.commit(); + accessManager.flushUploadQueue(); + accessManager.destroyVolumeDataPageAccessor(pageAccessor); + + vdsCopy.close(); + } + catch (java.io.IOException e) { + System.out.println(e.getMessage()); + fail(); + } + } + + /** + * Will test that copied file is the same as the input + * Name of method is the same + Suffix so that it's executed after the copy test + */ + @Test + public void testCopyPageAccessorValidation() { + try { + String tmpDir = System.getProperty("java.io.tmpdir"); + String vdsPath = tmpDir + File.separator + tempVdsCopyFileName; + VDSFileOpenOptions options = new VDSFileOpenOptions(vdsPath); + VdsHandle vdsCopy = OpenVDS.open(options); + VolumeDataAccessManager accessManager = vdsCopy.getAccessManager(); + + int channel = 0; + VolumeDataLayout layout = vdsCopy.getLayout(); + VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( + DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND + 0, // lod + channel, // channel + 20, // max pages + VolumeDataPageAccessor.AccessMode.ReadOnly.getCode()); // access mode + + // get input manager + VolumeDataAccessManager inputAM = vds.getAccessManager(); + VolumeDataPageAccessor pageAccessorInput = inputAM.createVolumeDataPageAccessor( + DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND + 0, // lod + channel, // channel + 20, // max pages + VolumeDataPageAccessor.AccessMode.ReadOnly.getCode()); // access mode + + // compares block data + int[] pitchInput = new int[VolumeDataLayout.Dimensionality_Max]; + int[] pitchOutput = new int[VolumeDataLayout.Dimensionality_Max]; + + long chunkCount = pageAccessorInput.getChunkCount(); + for (long chunk = 0 ; chunk < chunkCount ; ++chunk) { + VolumeDataPage inputPage = pageAccessorInput.readPage(chunk); + VolumeDataPage page = pageAccessor.readPage(chunk); + short[] dataIn = inputPage.readShortBuffer(pitchInput); + short[] dataOut = page.readShortBuffer(pitchOutput); + + inputPage.pageRelease(); + page.pageRelease(); + + Assert.assertEquals(pitchInput, pitchOutput); + Assert.assertEquals(dataIn, dataOut); + } + + accessManager.destroyVolumeDataPageAccessor(pageAccessor); + + vdsCopy.close(); + } + catch (java.io.IOException e) { + System.out.println(e.getMessage()); + fail(); + } + } + + /** + * Will test that copied file has the same layout than original (test that 3D positions are in the same chunk index) + * Name of method is the same + Suffix so that it's executed after the copy test + */ + @Test + public void testCopyPageAccessorValidationChunkIndex() { + try { + String tmpDir = System.getProperty("java.io.tmpdir"); + String vdsPath = tmpDir + File.separator + tempVdsCopyFileName; + VDSFileOpenOptions options = new VDSFileOpenOptions(vdsPath); + VdsHandle vdsCopy = OpenVDS.open(options); + VolumeDataAccessManager accessManager = vdsCopy.getAccessManager(); + + int channel = 0; + VolumeDataLayout layout = vdsCopy.getLayout(); + VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( + DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND + 0, // lod + channel, // channel + 20, // max pages + VolumeDataPageAccessor.AccessMode.ReadOnly.getCode()); // access mode + + // compares block data + int[] chunkMin = new int[VolumeDataLayout.Dimensionality_Max]; + int[] chunkMax = new int[VolumeDataLayout.Dimensionality_Max]; + int[] chunkMaxPos = new int[VolumeDataLayout.Dimensionality_Max]; + + long chunkCount = pageAccessor.getChunkCount(); + for (long chunk = 0 ; chunk < chunkCount ; ++chunk) { + VolumeDataPage inputPage = pageAccessor.readPage(chunk); + + // check that chunk index matches current index + inputPage.getMinMaxExcludingMargin(chunkMin, chunkMax); + for (int i = 0 ; i < VolumeDataLayout.Dimensionality_Max ; ++i) { + chunkMaxPos[i] = chunkMax[i] != 0 ? chunkMax[i] - 1 : chunkMax[i]; + } + long idxChMin = pageAccessor.getChunkIndex(chunkMin); + long idxChMax = pageAccessor.getChunkIndex(chunkMaxPos); + + Assert.assertEquals(chunk, idxChMin); + Assert.assertEquals(idxChMin, idxChMax); + + inputPage.pageRelease(); + } + + accessManager.destroyVolumeDataPageAccessor(pageAccessor); + + vdsCopy.close(); + } + catch (java.io.IOException e) { + System.out.println(e.getMessage()); + fail(); + } + } +} -- GitLab From 531cc488464eace9d15dd1d6ad21364c4a80a532 Mon Sep 17 00:00:00 2001 From: Julien Lacoste Date: Thu, 23 Sep 2021 12:07:21 +0200 Subject: [PATCH 24/25] Add bindings for set/get MetaDatablob --- java/cpp/src/MetadataContainer.cpp | 17 +++++ java/cpp/src/MetadataReadAccess.cpp | 21 ++++++ .../org/opengroup/openvds/BufferUtils.java | 8 +++ .../opengroup/openvds/MetadataContainer.java | 31 ++++++++- .../opengroup/openvds/MetadataReadAccess.java | 25 +++++++ .../openvds/MetaDataContainerTest.java | 66 +++++++++++++++++++ 6 files changed, 166 insertions(+), 2 deletions(-) diff --git a/java/cpp/src/MetadataContainer.cpp b/java/cpp/src/MetadataContainer.cpp index d99ca48b..7c6bf076 100644 --- a/java/cpp/src/MetadataContainer.cpp +++ b/java/cpp/src/MetadataContainer.cpp @@ -285,6 +285,23 @@ JNIEXPORT void JNICALL Java_org_opengroup_openvds_MetadataContainer_cpSetMetadat CATCH_EXCEPTIONS_FOR_JAVA; } +/* + * Class: org_opengroup_openvds_MetadataContainer + * Method: cpSetMetadataBLOB + * Signature: (JLjava/lang/String;Ljava/lang/String;[B)V + */ +JNIEXPORT void JNICALL Java_org_opengroup_openvds_MetadataContainer_cpSetMetadataBLOB + (JNIEnv * env, jclass, jlong handle, jstring category, jstring name, jbyteArray valueArray) +{ + try { + int valueSize = env->GetArrayLength(valueArray); + jbyte *src = env->GetByteArrayElements(valueArray, NULL); + GetAccess( handle )->SetMetadataBLOB(JStringToString( env, category ).c_str(), JStringToString( env, name ).c_str(), src, valueSize); + env->ReleaseByteArrayElements(valueArray, src, 0); + } + CATCH_EXCEPTIONS_FOR_JAVA; +} + #ifdef __cplusplus } #endif diff --git a/java/cpp/src/MetadataReadAccess.cpp b/java/cpp/src/MetadataReadAccess.cpp index dfd120d1..817ea74f 100644 --- a/java/cpp/src/MetadataReadAccess.cpp +++ b/java/cpp/src/MetadataReadAccess.cpp @@ -443,6 +443,27 @@ extern "C" { return NULL; } + /* + * Class: org_opengroup_openvds_MetadataReadAccess + * Method: cpGetMetadataBLOB + * Signature: (JLjava/lang/String;Ljava/lang/String;)[B + */ + JNIEXPORT jbyteArray JNICALL Java_org_opengroup_openvds_MetadataReadAccess_cpGetMetadataBLOB + (JNIEnv *env, jclass, jlong handle, jstring category, jstring name) + { + try { + // read blob + std::vector blob; + GetAccess( handle )->GetMetadataBLOB(JStringToString( env, category ).c_str(), JStringToString( env, name ).c_str(), blob ); + + // convert to array + const char* blobArray = reinterpret_cast(blob.data()); + return NewJByteArray(env, blobArray, blob.size()); + } + CATCH_EXCEPTIONS_FOR_JAVA; + return NULL; + } + /* * Class: org_opengroup_openvds_MetadataReadAccess * Method: cpGetMetadataKeys diff --git a/java/java/src/org/opengroup/openvds/BufferUtils.java b/java/java/src/org/opengroup/openvds/BufferUtils.java index cf98b05e..8cc9e001 100644 --- a/java/java/src/org/opengroup/openvds/BufferUtils.java +++ b/java/java/src/org/opengroup/openvds/BufferUtils.java @@ -27,6 +27,14 @@ import java.nio.ShortBuffer; import java.util.Objects; public abstract class BufferUtils { + /** + * @param array Th array to be copied into the new buffer + * @return A direct buffer copy of the given array + */ + public static ByteBuffer toBuffer(byte[] array) { + return (ByteBuffer) createByteBuffer(array.length).put(array).clear(); + } + /** * @param array Th array to be copied into the new buffer * @return A direct buffer copy of the given array diff --git a/java/java/src/org/opengroup/openvds/MetadataContainer.java b/java/java/src/org/opengroup/openvds/MetadataContainer.java index e156e11c..fdfb8e6a 100644 --- a/java/java/src/org/opengroup/openvds/MetadataContainer.java +++ b/java/java/src/org/opengroup/openvds/MetadataContainer.java @@ -17,8 +17,7 @@ package org.opengroup.openvds; -import java.util.HashMap; -import java.util.Map; +import java.nio.ByteBuffer; public class MetadataContainer extends MetadataReadAccess { @@ -52,6 +51,8 @@ public class MetadataContainer extends MetadataReadAccess { private static native void cpSetMetadataString(long handle, String category, String name, String value); + private static native void cpSetMetadataBLOB(long handle, String category, String name, byte[] blobValues); + /** * Constructor around existing JNI object * @param handle jni pointer to existing container @@ -204,6 +205,32 @@ public class MetadataContainer extends MetadataReadAccess { cpSetMetadataString(_handle, category, name, value); } + /** + * set Meta data blob using a byte array + * @param category + * @param name + * @param blobValues + */ + public void setMetadataBLOB(String category, String name, byte[] blobValues) { + if (blobValues == null) { + throw new IllegalArgumentException("Blob values array is null."); + } + cpSetMetadataBLOB(_handle, category, name, blobValues); + } + + /** + * set Meta data blob using a byte array + * @param category + * @param name + * @param blobValues + */ + public void setMetadataBLOB(String category, String name, ByteBuffer blobValues) { + if (blobValues == null) { + throw new IllegalArgumentException("Blob values buffer is null."); + } + cpSetMetadataBLOB(_handle, category, name, blobValues.array()); + } + private void checkArrayArgument(int[] array, int expectedSize) { if (array == null || array.length != expectedSize) { diff --git a/java/java/src/org/opengroup/openvds/MetadataReadAccess.java b/java/java/src/org/opengroup/openvds/MetadataReadAccess.java index bf36af10..bc9e598e 100644 --- a/java/java/src/org/opengroup/openvds/MetadataReadAccess.java +++ b/java/java/src/org/opengroup/openvds/MetadataReadAccess.java @@ -17,6 +17,8 @@ package org.opengroup.openvds; +import java.nio.ByteBuffer; + public class MetadataReadAccess extends JniPointerWithoutDeletion { private static native boolean cpIsMetadataIntAvailable(long handle, String category, String name); @@ -73,6 +75,8 @@ public class MetadataReadAccess extends JniPointerWithoutDeletion { private static native String cpGetMetadataString(long handle, String category, String name); + private static native byte[] cpGetMetadataBLOB(long handle, String category, String name); + private static native MetadataKey[] cpGetMetadataKeys(long handle); public MetadataReadAccess(long handle) { @@ -337,6 +341,27 @@ public class MetadataReadAccess extends JniPointerWithoutDeletion { return cpGetMetadataString(_handle, category, name); } + /** + * get Metadata blob as byte array + * @param category + * @param name + * @return + */ + public byte[] getMetadataBLOB(String category, String name) { + return cpGetMetadataBLOB(_handle, category, name); + } + + /** + * get Metadata blob as byte buffer + * @param category + * @param name + * @return + */ + public ByteBuffer getMetadataBLOBAsBuffer(String category, String name) { + byte[] blobArray = cpGetMetadataBLOB(_handle, category, name); + return ByteBuffer.wrap(blobArray); + } + /** * @return an array of metadata keys */ diff --git a/java/java/test/org/opengroup/openvds/MetaDataContainerTest.java b/java/java/test/org/opengroup/openvds/MetaDataContainerTest.java index 79b5d4ca..92786bbd 100644 --- a/java/java/test/org/opengroup/openvds/MetaDataContainerTest.java +++ b/java/java/test/org/opengroup/openvds/MetaDataContainerTest.java @@ -17,11 +17,15 @@ package org.opengroup.openvds; +import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; import java.io.File; +import java.nio.ByteBuffer; +import java.nio.FloatBuffer; +import java.util.Random; import static org.testng.Assert.assertEquals; import static org.testng.Assert.fail; @@ -308,4 +312,66 @@ public class MetaDataContainerTest { } } + @Test + public void testMetaDataBlob() { + MetadataContainer metaData = new MetadataContainer(); + + byte[] blobArray = new byte[3200]; + Random rand = new Random(System.currentTimeMillis()); + rand.nextBytes(blobArray); + + metaData.setMetadataBLOB("Blob data", "blob array", blobArray); + + Assert.assertTrue(metaData.isMetadataBLOBAvailable("Blob data", "blob array")); + byte[] metadataBLOBArray = metaData.getMetadataBLOB("Blob data", "blob array"); + Assert.assertEquals(blobArray, metadataBLOBArray); + } + + @Test + public void testMetaDataBlobBuffer() { + MetadataContainer metaData = new MetadataContainer(); + + byte[] blobArray = new byte[3200]; + Random rand = new Random(System.currentTimeMillis()); + rand.nextBytes(blobArray); + + ByteBuffer blobBuffer = ByteBuffer.wrap(blobArray); + metaData.setMetadataBLOB("Blob data", "blob array", blobBuffer); + + Assert.assertTrue(metaData.isMetadataBLOBAvailable("Blob data", "blob array")); + ByteBuffer metadataBLOBBuffer = metaData.getMetadataBLOBAsBuffer("Blob data", "blob array"); + + // compares buffer + Assert.assertTrue(metadataBLOBBuffer.equals(blobBuffer)); + } + + @Test + public void testMetaDataBlobFloatBuffer() { + MetadataContainer metaData = new MetadataContainer(); + + // put floats in a byte buffer + float[] blobFloatArray = new float[2000]; + Random rand = new Random(System.currentTimeMillis()); + for (int i = 0 ; i < blobFloatArray.length ; ++i) { + blobFloatArray[i] = rand.nextFloat(); + } + ByteBuffer byteBuffer = ByteBuffer.allocate(Float.BYTES * blobFloatArray.length); + byteBuffer.asFloatBuffer().put(blobFloatArray); + + // writes them + metaData.setMetadataBLOB("Blob data", "blob array", byteBuffer); + + // reread and compare + Assert.assertTrue(metaData.isMetadataBLOBAvailable("Blob data", "blob array")); + ByteBuffer metadataBLOBBuffer = metaData.getMetadataBLOBAsBuffer("Blob data", "blob array"); + FloatBuffer metaDataFB = metadataBLOBBuffer.asFloatBuffer(); + int size = metaDataFB.remaining(); + float[] readFloats = new float[size]; + metaDataFB.get(readFloats); + + // compares buffer + Assert.assertEquals(blobFloatArray, readFloats); + } } + + -- GitLab From a93cf4ef12a7c83234629d68fca67a9529035e32 Mon Sep 17 00:00:00 2001 From: Julien Lacoste Date: Thu, 23 Sep 2021 14:39:06 +0200 Subject: [PATCH 25/25] Remove CreateVDSCloud.java --- java/java/demo/CreateVDSCloud.java | 578 ----------------------------- 1 file changed, 578 deletions(-) delete mode 100644 java/java/demo/CreateVDSCloud.java diff --git a/java/java/demo/CreateVDSCloud.java b/java/java/demo/CreateVDSCloud.java deleted file mode 100644 index 32a2faac..00000000 --- a/java/java/demo/CreateVDSCloud.java +++ /dev/null @@ -1,578 +0,0 @@ -/* - * Copyright 2021 The Open Group - * Copyright 2021 INT, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.opengroup.openvds.*; - -import java.io.File; -import java.io.IOException; -import java.time.Duration; -import java.time.Instant; -import java.util.ArrayList; -import java.util.List; - -public class CreateVDSCloud { - - // usage : CreateVDSWithLOD n /path/to/file - // where n is the LOD level wanted - public static void main(String[] args) { - try { - if (!checkParams(args)) { - System.out.println("Bad params, usage : n /path/to/file dimX dimY dimZ"); - System.out.println("n is the LOD level wanted (0 <= lod <= 12), path must be a valid non existing file path"); - System.out.println("dims are cube dimension (must be > 0)"); - } - else { - Instant start = Instant.now(); - process(Integer.parseInt(args[0]), args[1], Integer.parseInt(args[2]), Integer.parseInt(args[3]), Integer.parseInt(args[4])); - Instant end = Instant.now(); - - Duration elapsed = Duration.between(start, end); - long hrs = elapsed.toHours(); - long min = elapsed.toMinutes() - (hrs * 60); - long s = (elapsed.toMillis() - (elapsed.toMinutes() * 60 * 1000)) / 1000; - System.out.println("Write VDS TIME : " + hrs + " hrs " + min + " min " + s + "s (" + elapsed.toMillis() + " ms)"); - - testRead(args[1]); - } - } - catch (Throwable t) { - System.out.println(); - t.printStackTrace(); - } - } - - private static boolean checkParams(String[] args) { - if (args == null || args.length != 5) { - return false; - } - try { - Integer lod = Integer.parseInt(args[0]); - if (lod < 0 || lod > 12) { - System.err.println("Invalid LOD value (must be between 0 and 12) !"); - return false; - } - } - catch (NumberFormatException nfe) { - return false; - } - String path = args[1]; - File file = new File(path); - try { - file.getCanonicalPath(); - } - catch (IOException e) { - System.err.println("VDS File path is invalid !"); - return false; - } - if (!path.endsWith(".vds")) { - System.err.println("VDS File path must have vds extension !"); - return false; - } - if (file.exists()) { - System.err.println("VDS File already exists !"); - return false; - } - try { - Integer dimX = Integer.parseInt(args[2]); - if (dimX < 0) { - System.err.println("Invalid x dimension !"); - return false; - } - Integer dimY = Integer.parseInt(args[3]); - if (dimX < 0) { - System.err.println("Invalid y dimension !"); - return false; - } - Integer dimZ = Integer.parseInt(args[4]); - if (dimX < 0) { - System.err.println("Invalid Z dimension !"); - return false; - } - } - catch (NumberFormatException nfe) { - return false; - } - return true; - } - - static void process(int lodParam, String name, int dimX, int dimY, int dimZ) throws Exception { - - int samplesX = dimZ; // time - int samplesY = dimY; // XL - int samplesZ = dimX; // IL - VolumeDataChannelDescriptor.Format format = VolumeDataChannelDescriptor.Format.FORMAT_R32; - - double sizeX = samplesX; - double sizeY = samplesY; - double sizeZ = samplesZ; - - double distMax = distance3D(0, 0, 0, sizeX, sizeY, sizeZ); - double cycles = Math.PI * 2 * 20; - double midX = samplesX / 2f; - double midY = samplesY / 2f; - double midZ = samplesZ / 2f; - - VolumeDataLayoutDescriptor.BrickSize brickSize = VolumeDataLayoutDescriptor.BrickSize.BRICK_SIZE_64; - int negativeMargin = 4; - int positiveMargin = 4; - int brickSize2DMultiplier = 4; - - VolumeDataLayoutDescriptor.LODLevels lodLevels = VolumeDataLayoutDescriptor.LODLevels.values()[lodParam]; - - //VolumeDataLayoutDescriptor.Options layoutOptions = VolumeDataLayoutDescriptor.Options.NONE; - VolumeDataLayoutDescriptor layoutDescriptor = new VolumeDataLayoutDescriptor(brickSize, negativeMargin, positiveMargin, - brickSize2DMultiplier, lodLevels, false,false,0); - - List axisDescriptors = new ArrayList<>(); - axisDescriptors.add(new VolumeDataAxisDescriptor(samplesX, "Sample", "s", 0.0f,6.0f)); - axisDescriptors.add(new VolumeDataAxisDescriptor(samplesY, "Crossline", "",0f, 0f + samplesY - 1f)); - axisDescriptors.add(new VolumeDataAxisDescriptor(samplesZ, "Inline", "", 0f, 0f + samplesZ - 1f)); - - List channelDescriptors = new ArrayList<>(); - - float rangeMin = -1f; - float rangeMax = 1f; - float[] scaleOffset = getScaleOffsetForFormat(rangeMin, rangeMax, true, format); - - VolumeDataChannelDescriptor channelDescriptor = new VolumeDataChannelDescriptor(format, VolumeDataChannelDescriptor.Components.COMPONENTS_1, - "Amplitude", - "", - rangeMin, // range min - rangeMax, // range max - VolumeDataMapping.DIRECT, // mapping - 1, // mapped value count - false, // is discrete - true, // is renderable - false, // allow lossy compression - false, // use zip for lossless compresion - false, // use no value - -999.25f, // no value - scaleOffset[0], // integer scale - scaleOffset[1]); // integer offset - channelDescriptors.add(channelDescriptor); - - // options AWS - String pBucket = ""; // the bucket of the VDS - String pKey = name; // the key prefix of the VDS ? - String pRegion = "eu-west-3"; // the region of the bucket of the VDS - String pEndpointOverride = ""; // This parameter allows to override the endpoint url - AWSOpenOptions optionsAWS = new AWSOpenOptions(pBucket, pKey, pRegion, pEndpointOverride); - optionsAWS.accessKeyId = ""; // ? - optionsAWS.secretKey = ""; - optionsAWS.sessionToken = ""; // ? - - // file options - //VDSFileOpenOptions options = new VDSFileOpenOptions(vdsFilePath); - - MetadataContainer metadataContainer = new MetadataContainer(); - StringBuilder builder = new StringBuilder(); - builder.append("CreateVDS Demo "); - builder.append("dimX: "); - builder.append(dimX); - builder.append(" dimY: "); - builder.append(dimY); - builder.append(" dimZ: "); - builder.append(dimZ); - - metadataContainer.setMetadataString("General Info", "Cube layout", builder.toString()); - metadataContainer.setMetadataInt("General Info", "LOD", lodLevels.ordinal()); - metadataContainer.setMetadataString("General Info", "Brick Size", "BRICK_SIZE_64"); - - VdsHandle vds = OpenVDS.create(optionsAWS, layoutDescriptor, - axisDescriptors.toArray(new VolumeDataAxisDescriptor[0]), - channelDescriptors.toArray(new VolumeDataChannelDescriptor[0]), metadataContainer); - - // for compression use -// float compressionTolerance = 0f; // -// VdsHandle vds = OpenVDS.create(options, layoutDescriptor, -// axisDescriptors.toArray(new VolumeDataAxisDescriptor[0]), -// channelDescriptors.toArray(new VolumeDataChannelDescriptor[0]), metadataContainer, CompressionMethod.ZIP, compressionTolerance); - - VolumeDataLayout layout = vds.getLayout(); - VolumeDataAccessManager accessManager = vds.getAccessManager(); - - int channel = 0; - - int lodCount = lodLevels.ordinal(); - - int[] localIndex = new int[3]; - int[] voxelPos = new int[3]; - - int[] pitch = new int[VolumeDataLayout.Dimensionality_Max]; - int[] chunkMin = new int[VolumeDataLayout.Dimensionality_Max]; - int[] chunkMax = new int[VolumeDataLayout.Dimensionality_Max]; - - for (int lod = 0 ; lod <= lodCount ; ++lod) { - VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( - DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND - lod, // lod - channel, // channel - 50, // max pages - VolumeDataPageAccessor.AccessMode.Create.getCode()); // access mode - - - //ASSERT_TRUE(pageAccessor); - int paLOD = pageAccessor.getLOD(); - int paCI = pageAccessor.getChannelIndex(); - int[] paNumSamples = pageAccessor.getNumSamples(); - - int chunkCount = (int) pageAccessor.getChunkCount(); - System.out.println("LOD : " + lod + " Chunk count : " + chunkCount); - - for (int i = 0; i < chunkCount; i++) { - - long time_1 = System.currentTimeMillis(); - - VolumeDataPage page = pageAccessor.createPage(i); - System.out.print("LOD : " + lod + " / Page " + (i + 1) + " / " + chunkCount); - - VolumeIndexer3D outputIndexer = new VolumeIndexer3D(page, 0, lod, DimensionsND.DIMENSIONS_012.ordinal(), layout); - - int[] numSamplesChunk = new int[3]; - int[] numSamplesDB = new int[3]; - - for (int j = 0; j < 3; j++) { - numSamplesChunk[j] = outputIndexer.getLocalChunkNumSamples(j); - numSamplesDB[j] = outputIndexer.getDataBlockNumSamples(j); - } - System.out.print("\tDimensions Chunk : [" + numSamplesChunk[0] + ", " + numSamplesChunk[1] + ", " + numSamplesChunk[2] + "]"); - System.out.print("\tDimensions DataBlock : [" + numSamplesDB[0] + ", " + numSamplesDB[1] + ", " + numSamplesDB[2] + "]"); - - page.getMinMax(chunkMin, chunkMax); - - System.out.print("\tCoords page : " + chunkMin[0] + "/" + chunkMax[0] + " " + chunkMin[1] + "/" + chunkMax[1] + " " + chunkMin[2] + "/" + chunkMax[2]); - int chunkSizeY = chunkMax[2] - chunkMin[2]; - int chunkSizeX = chunkMax[1] - chunkMin[1]; - int chunkSizeZ = chunkMax[0] - chunkMin[0]; - - // get pitch and allocate buffer size - page.getPitch(pitch); - float[] output = new float[page.getAllocatedBufferSize()]; - - // or read buffer of page - //float[] output = page.readFloatBuffer(pitch); - - System.out.print( " Page Buffer Size : " + output.length + " ");/// [" + stats.getMin() + "," + stats.getMax() + "]"); - // dimension of buffer - System.out.println(" Pitch : [" + pitch[0] + ", " + pitch[1] + ", " + pitch[2] + "]"); - - int[] numSamples = numSamplesChunk; - - for (int iDim2 = 0; iDim2 < numSamples[2]; iDim2++) { - localIndex[2] = iDim2; - for (int iDim1 = 0; iDim1 < numSamples[1]; iDim1++) { - localIndex[1] = iDim1; - for (int iDim0 = 0; iDim0 < numSamples[0]; iDim0++) { - localIndex[0] = iDim0; - - int[] voxelIndex = outputIndexer.localIndexToVoxelIndex(localIndex); - for (int vp = 0; vp < 3; ++vp) { - voxelPos[vp] = voxelIndex[vp]; - } - - float value = 0f; - double dist = 0; - if (voxelPos[0] >= midX) { - dist = distance2D(midY, midZ, voxelPos[1], voxelPos[2]); - } else { - dist = distance3D(midX, midY, midZ, voxelPos[0], voxelPos[1], voxelPos[2]); - } - value = (float) Math.sin((dist * cycles) / distMax); - - int iPos = outputIndexer.localIndexToDataIndex(localIndex); - output[iPos] = value; - } - } - } - - // write buffer then release page - page.writeFloatBuffer(output, pitch); - page.pageRelease(); - } - - int errorCount = accessManager.uploadErrorCount(); - if (errorCount > 0) { - ConnectionErrorInfo upErrInfo = accessManager.getCurrentUploadErrorInfo(); - System.out.println("Upload error code : " + upErrInfo.getErrorCode()); - if (upErrInfo.getErrorMessage() != null) { - System.out.println("Upload error message : " + upErrInfo.getErrorMessage()); - } - else { - System.out.println("No error message"); - } - if (upErrInfo.getObjectID() != null) { - System.out.println("Upload error object ID : " + upErrInfo.getObjectID()); - } - else { - System.out.println("No Object ID"); - } - } - - pageAccessor.commit(); - pageAccessor.setMaxPages(0); - accessManager.flushUploadQueue(); - accessManager.destroyVolumeDataPageAccessor(pageAccessor); - } - - - - // Adds slice data : additional layouts -// System.out.println("\nCreate 01 Layout"); -// create2DLayout(distMax, cycles, midX, midY, midZ, layout, accessManager, channel, lodCount, DimensionsND.DIMENSIONS_01); -// -// System.out.println("\nCreate 02 Layout"); -// create2DLayout(distMax, cycles, midX, midY, midZ, layout, accessManager, channel, lodCount, DimensionsND.DIMENSIONS_02); -// -// System.out.println("\nCreate 12 Layout"); -// create2DLayout(distMax, cycles, midX, midY, midZ, layout, accessManager, channel, lodCount, DimensionsND.DIMENSIONS_12); - - - vds.close(); - System.out.println("VDS closed"); - } - - - private static void create2DLayout(double distMax, double cycles, double midX, double midY, double midZ, VolumeDataLayout layout, VolumeDataAccessManager accessManager, - int channel, int lodCount, DimensionsND dimLYT) { - - int[] localIndex = new int[3]; - - int[] pitch = new int[VolumeDataLayout.Dimensionality_Max]; - int[] chunkMin = new int[VolumeDataLayout.Dimensionality_Max]; - int[] chunkMax = new int[VolumeDataLayout.Dimensionality_Max]; - for (int lod = 0 ; lod <= lodCount ; ++lod) { - VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( - dimLYT.ordinal(), // dimension ND - lod, // lod - channel, // channel - 200, // max pages - VolumeDataPageAccessor.AccessMode.Create.getCode()); // access mode - - int chunkCount = (int) pageAccessor.getChunkCount(); - System.out.println("2D Layout LOD : " + lod + " Chunk count : " + chunkCount); - - for (int i = 0; i < chunkCount; i++) { - VolumeDataPage page = pageAccessor.createPage(i); - System.out.print("2D Layout LOD : " + lod + " / Page " + (i + 1) + " / " + chunkCount); - - VolumeIndexer2D outputIndexer = new VolumeIndexer2D(page, 0, lod, dimLYT.ordinal(), layout); - float valueRangeScale = outputIndexer.getValueRangeMax() - outputIndexer.getValueRangeMin(); - - int[] numSamplesChunk = new int[3]; - int[] numSamplesDB = new int[3]; - - for (int j = 0; j < 3 ; j++) { - numSamplesChunk[j] = outputIndexer.getLocalChunkNumSamples(j); - numSamplesDB[j] = outputIndexer.getDataBlockNumSamples(j); - } - System.out.print("\t2D Layout Dimensions Chunk : [" + numSamplesChunk[0] + ", " + numSamplesChunk[1] + ", " + numSamplesChunk[2] + "]"); - System.out.print("\t2D Layout Dimensions DataBlock : [" + numSamplesDB[0] + ", " + numSamplesDB[1] + ", " + numSamplesDB[2] + "]"); - - page.getMinMax(chunkMin, chunkMax); - - // determine fixed position index and value - int fixedCoordIndex = -1; - int fixedCoordValue = -1; - for (int pos = 0; pos < 3 && fixedCoordIndex == -1 ; ++pos) { - if (Math.abs(chunkMax[pos] - chunkMin[pos]) == 1) { - fixedCoordIndex = pos; - fixedCoordValue = chunkMin[pos]; - } - } - - System.out.print("\tCoords page : " + chunkMin[0] + "/" + chunkMax[0] + " " + chunkMin[1] + "/" + chunkMax[1] + "/" + chunkMin[2] + "/" + chunkMax[2]); - int chunkSizeY = chunkMax[2] - chunkMin[2]; - int chunkSizeX = chunkMax[1] - chunkMin[1]; - int chunkSizeZ = chunkMax[0] - chunkMin[0]; - - // get pitch and allocate buffer size - page.getPitch(pitch); - float[] output = new float[page.getAllocatedBufferSize()]; - - System.out.print( " Page Buffer Size : " + output.length);// + " / [" + stats.getMin() + "," + stats.getMax() + "]"); - // dimension of buffer - System.out.println(" Pitch : [" + pitch[0] + ", " + pitch[1] + "," + pitch[2] + "]"); - - long time_2 = System.currentTimeMillis(); - - int[] numSamples = numSamplesChunk; - - for (int iDim2 = 0; iDim2 < numSamples[2]; iDim2++) { - localIndex[2] = iDim2; - for (int iDim1 = 0; iDim1 < numSamples[1]; iDim1++) { - localIndex[1] = iDim1; - for (int iDim0 = 0; iDim0 < numSamples[0]; iDim0++) { - localIndex[0] = iDim0; - int[] local2DIndex = convertTo2DLocalIndex(localIndex, fixedCoordIndex); - int[] voxelIndex = outputIndexer.localChunkIndexToLocalIndex(local2DIndex); - int[] voxelPos = convertTo3DCoord(voxelIndex, fixedCoordIndex, fixedCoordValue); - - float value = 0f; - double dist = 0; - if (voxelPos[0] >= midX) { - dist = distance2D(midY, midZ, voxelPos[1], voxelPos[2]); - } else { - dist = distance3D(midX, midY, midZ, voxelPos[0], voxelPos[1], voxelPos[2]); - } - value = (float) Math.sin((dist * cycles) / distMax); - - int iPos = outputIndexer.localIndexToDataIndex(local2DIndex); - output[iPos] = value; - } - } - } - - // write buffer then release page - page.writeFloatBuffer(output, pitch); - page.pageRelease(); - } - - pageAccessor.commit(); - accessManager.flushUploadQueue(); - accessManager.destroyVolumeDataPageAccessor(pageAccessor); - } - } - - static void testRead(String vdsName) throws IOException { - // options AWS - String pBucket = "miniovaap"; // the bucket of the VDS - String pKey = vdsName; // the key prefix of the VDS ? - String pRegion = "eu-west-3"; // the region of the bucket of the VDS - String pEndpointOverride = "https://webvr.int.com/"; // This parameter allows to override the endpoint url - AWSOpenOptions optionsAWS = new AWSOpenOptions(pBucket, pKey, pRegion, pEndpointOverride); - optionsAWS.accessKeyId = "minioadmin"; // ? - optionsAWS.secretKey = "minioadmin"; - optionsAWS.sessionToken = ""; // ? - - OpenVDS vdsHandle = OpenVDS.open(optionsAWS); - VolumeDataAccessManager accessManager = vdsHandle.getAccessManager(); - - int channel = 0; - VolumeDataLayout layout = vdsHandle.getLayout(); - VolumeDataPageAccessor pageAccessor = accessManager.createVolumeDataPageAccessor( - DimensionsND.DIMENSIONS_012.ordinal(), // dimension ND - 0, // lod - channel, // channel - 20, // max pages - VolumeDataPageAccessor.AccessMode.ReadOnly.getCode()); // access mode - - - - // compares block data - int[] pitchInput = new int[VolumeDataLayout.Dimensionality_Max]; - - long chunkCount = pageAccessor.getChunkCount(); - for (long chunk = 0 ; chunk < chunkCount ; ++chunk) { - VolumeDataPage inputPage = pageAccessor.readPage(chunk); - VolumeDataPage page = pageAccessor.readPage(chunk); - float[] dataIn = inputPage.readFloatBuffer(pitchInput); - - ConnectionErrorInfo dwnConnectionErrorInfo = accessManager.getCurrentDownloadErrorInfo(); - if (dwnConnectionErrorInfo.getErrorCode() != 0) { - System.out.println("Download error code : " + dwnConnectionErrorInfo.getErrorCode()); - if (dwnConnectionErrorInfo.getErrorMessage() != null) { - System.out.println("Download error message : " + dwnConnectionErrorInfo.getErrorMessage()); - } else { - System.out.println("No error message"); - } - } - } - } - - - /** - * Convert 3D index to 2D index - */ - private static int[] convertTo2DLocalIndex(int[] localIndex, int fixedCoordIndex) { - int[] local2DIndex = new int[2]; - if (fixedCoordIndex == 0) { - local2DIndex[0] = localIndex[1]; - local2DIndex[1] = localIndex[2]; - } - else if (fixedCoordIndex == 1) { - local2DIndex[0] = localIndex[0]; - local2DIndex[1] = localIndex[2]; - } - else if (fixedCoordIndex == 2) { - local2DIndex[0] = localIndex[0]; - local2DIndex[1] = localIndex[1]; - } - return local2DIndex; - } - - /** - * Convert 2D index to 3D index - * @param voxelIndex - * @param fixedCoordIndex - * @param fixedCoordValue - */ - private static int[] convertTo3DCoord(int[] voxelIndex, int fixedCoordIndex, int fixedCoordValue) { - int[] voxelPos = new int[3]; - voxelPos[fixedCoordIndex] = fixedCoordValue; - if (fixedCoordIndex == 0) { - voxelPos[1] = voxelIndex[0]; - voxelPos[2] = voxelIndex[1]; - } - else if (fixedCoordIndex == 1) { - voxelPos[0] = voxelIndex[0]; - voxelPos[2] = voxelIndex[1]; - } - else if (fixedCoordIndex == 2) { - voxelPos[0] = voxelIndex[0]; - voxelPos[1] = voxelIndex[1]; - } - return voxelPos; - } - - private static double distance2D(double x1, double y1, double x2, double y2) { - double diffX = x2 - x1; - double diffY = y2 - y1; - return Math.sqrt((diffX * diffX) + (diffY * diffY)); - } - - private static double distance3D(double x1, double y1, double z1, double x2, double y2, double z2) { - double diffX = x2 - x1; - double diffY = y2 - y1; - double diffZ = z2 - z1; - return Math.sqrt((diffX * diffX) + (diffY * diffY) + (diffZ * diffZ)); - } - - static float[] getScaleOffsetForFormat(float min, float max, boolean useNoValue, VolumeDataChannelDescriptor.Format format) { - float res[] = new float[] {1f, 0f}; - float noValueCmp = useNoValue ? 1f : 0f; - switch (format) { - case FORMAT_U8: - res[0] = 1.f / (255.f - noValueCmp) * (max - min); - res[1] = min; - break; - case FORMAT_U16: - res[0] = 1.f / (65535.f - noValueCmp) * (max - min); - res[1] = min; - break; - case FORMAT_R32: - case FORMAT_U32: - case FORMAT_R64: - case FORMAT_U64: - case FORMAT_1BIT: - case FORMAT_ANY: - res[0] = 1.0f; - res[1] = 0.0f; - } - return res; - } - -} -- GitLab