Skip to content

GitLab

Commit 5b20c9a1 authored by Brian Barran's avatar Brian Barran
Browse files

Merge branch 'master' into feature/brian/compression/bitstream 

Conflicts:
	src/OpenVDS/VDS/VolumeDataStore.cpp
	tests/VDS/DeserializeVolumeDataTest.cpp
parents 5c0246ff 4cba7dde
Hide whitespace changes
Inline Side-by-side
.gitignore
View file @ 5b20c9a1
.vs
.vscode
Builds
Dist
CMakeSettings.json
CMakeLists.txt.user
python/openvds/test/aws_defs.py
CMakeSettings.json
out
*.pyc
build/
python/tools/tmpinclude/
python/tools/generated
python/tools/.vscode
python/tools/WrapperReport.txt
/.pytest_cache
/_skbuild
......
CMake/BuildGoogleClient.cmake
View file @ 5b20c9a1
......@@ -63,8 +63,24 @@ function(addGoogleCloudToTarget target)
${google-cloud-cpp_SOURCE_DIR}/google/cloud/storage/bucket_metadata.cc
${google-cloud-cpp_SOURCE_DIR}/google/cloud/iam_bindings.cc
${google-cloud-cpp_SOURCE_DIR}/google/cloud/storage/lifecycle_rule.cc
#${google-cloud-cpp_SOURCE_DIR}/google/cloud/
#${google-cloud-cpp_SOURCE_DIR}/google/cloud/
${google-cloud-cpp_SOURCE_DIR}/google/cloud/storage/client.cc
${google-cloud-cpp_SOURCE_DIR}/google/cloud/storage/client_options.cc
${google-cloud-cpp_SOURCE_DIR}/google/cloud/storage/internal/hash_validator.cc
${google-cloud-cpp_SOURCE_DIR}/google/cloud/storage/internal/object_streambuf.cc
${google-cloud-cpp_SOURCE_DIR}/google/cloud/storage/internal/logging_client.cc
${google-cloud-cpp_SOURCE_DIR}/google/cloud/storage/internal/policy_document_request.cc
${google-cloud-cpp_SOURCE_DIR}/google/cloud/storage/internal/retry_client.cc
${google-cloud-cpp_SOURCE_DIR}/google/cloud/storage/internal/signed_url_requests.cc
${google-cloud-cpp_SOURCE_DIR}/google/cloud/storage/object_stream.cc
${google-cloud-cpp_SOURCE_DIR}/google/cloud/storage/internal/hash_validator_impl.cc
${google-cloud-cpp_SOURCE_DIR}/google/cloud/iam_policy.cc
${google-cloud-cpp_SOURCE_DIR}/google/cloud/storage/internal/empty_response.cc
${google-cloud-cpp_SOURCE_DIR}/google/cloud/storage/service_account.cc
${google-cloud-cpp_SOURCE_DIR}/google/cloud/storage/internal/logging_resumable_upload_session.cc
${google-cloud-cpp_SOURCE_DIR}/google/cloud/storage/idempotency_policy.cc
${google-cloud-cpp_SOURCE_DIR}/google/cloud/internal/backoff_policy.cc
${google-cloud-cpp_SOURCE_DIR}/google/cloud/storage/internal/retry_object_read_source.cc
${google-cloud-cpp_SOURCE_DIR}/google/cloud/storage/internal/retry_resumable_upload_session.cc
#${google-cloud-cpp_SOURCE_DIR}/google/cloud/
#${google-cloud-cpp_SOURCE_DIR}/google/cloud/
#${google-cloud-cpp_SOURCE_DIR}/google/cloud/
......@@ -73,6 +89,10 @@ function(addGoogleCloudToTarget target)
${CMAKE_BINARY_DIR}/google-cloud-cpp/internal/build_info.cc
)
if (MSVC AND (CMAKE_CXX_COMPILER_ID STREQUAL "Clang"))
target_sources(${target} PRIVATE ${google-cloud-cpp_SOURCE_DIR}/google/cloud/terminate_handler.cc)
endif()
if (NOT (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC"))
set_source_files_properties(${google-cloud-cpp_SOURCE_DIR}/google/cloud/storage/internal/openssl_util.cc PROPERTIES COMPILE_FLAGS -Wno-pedantic)
set_source_files_properties(${google-cloud-cpp_SOURCE_DIR}/google/cloud/storage/internal/bucket_requests.cc PROPERTIES COMPILE_FLAGS -Wno-pedantic)
......
CMake/Fetch3rdPartyInBuild.cmake
View file @ 5b20c9a1
#This file depends on being in open-vds/CMake directory
set(Fetch3rdPartyDir ${CMAKE_CURRENT_LIST_DIR})
set(Fetch3rdPartyDirInternal "${CMAKE_CURRENT_LIST_DIR}")
macro(Fetch3rdParty_File name dir_prefix version ext url url_hash)
get_filename_component(thirdParty "${Fetch3rdPartyDir}/../3rdparty/${dir_prefix}" ABSOLUTE)
if (OPENVDS_3RD_PARTY_DIR)
set(Fetch3rdPartyDir "${OPENVDS_3RD_PARTY_DIR}")
else()
set(Fetch3rdPartyDir "${Fetch3rdPartyDirInternal}/../3rdparty")
endif()
get_filename_component(thirdParty "${Fetch3rdPartyDir}/${dir_prefix}" ABSOLUTE)
file(MAKE_DIRECTORY ${thirdParty})
set(SRC_FILE ${thirdParty}/${name}-${version}.${ext})
set(${name}_SOURCE_FILE ${SRC_FILE} PARENT_SCOPE)
......@@ -17,11 +22,16 @@ macro(Fetch3rdParty_File name dir_prefix version ext url url_hash)
endmacro()
macro(Fetch3rdParty_FileTarget name dir_prefix dest version url url_hash)
get_filename_component(thirdParty "${Fetch3rdPartyDir}/../3rdparty/${name}-${version}/${dir_prefix}" ABSOLUTE)
if (OPENVDS_3RD_PARTY_DIR)
set(Fetch3rdPartyDir "${OPENVDS_3RD_PARTY_DIR}")
else()
set(Fetch3rdPartyDir "${Fetch3rdPartyDirInternal}/../3rdparty")
endif()
get_filename_component(thirdParty "${Fetch3rdPartyDir}/${name}-${version}/${dir_prefix}" ABSOLUTE)
file(MAKE_DIRECTORY ${thirdParty})
set(DEST_FILE ${thirdParty}/${dest})
set(${name}_SOURCE_FILE ${DEST_FILE} PARENT_SCOPE)
set(${name}_SOURCE_DIR ${Fetch3rdPartyDir}/../3rdparty/${name}-${version} PARENT_SCOPE)
set(${name}_SOURCE_DIR ${Fetch3rdPartyDir}/${name}-${version} PARENT_SCOPE)
set(${name}_VERSION ${version} PARENT_SCOPE)
if (NOT (EXISTS ${DEST_FILE}))
file(DOWNLOAD ${url}
......@@ -34,7 +44,12 @@ macro(Fetch3rdParty_FileTarget name dir_prefix dest version url url_hash)
endmacro()
macro(Fetch3rdParty_Package name version url url_hash)
get_filename_component(thirdParty "${Fetch3rdPartyDir}/../3rdparty" ABSOLUTE)
if (OPENVDS_3RD_PARTY_DIR)
set(Fetch3rdPartyDir "${OPENVDS_3RD_PARTY_DIR}")
else()
set(Fetch3rdPartyDir "${Fetch3rdPartyDirInternal}/../3rdparty")
endif()
get_filename_component(thirdParty "${Fetch3rdPartyDir}" ABSOLUTE)
set(SRC_DIR ${thirdParty}/${name}-${version})
set(${name}_SOURCE_DIR ${SRC_DIR} PARENT_SCOPE)
set(${name}_VERSION ${version} PARENT_SCOPE)
......@@ -63,13 +78,13 @@ function(Fetch3rdParty)
Fetch3rdParty_Package(curl 7.68.0 https://github.com/curl/curl/releases/download/curl-7_68_0/curl-7.68.0.tar.gz SHA256=1dd7604e418b0b9a9077f62f763f6684c1b092a7bc17e3f354b8ad5c964d7358)
Fetch3rdParty_Package(libuv 1.34.1 https://github.com/libuv/libuv/archive/v1.34.1.tar.gz SHA256=e3e0105c9b26e181e0547607cb6893462beb0c652674c3795766b2e5555288b3)
Fetch3rdParty_Package(zlib 1.2.11 http://zlib.net/zlib-1.2.11.tar.gz SHA256=c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1)
Fetch3rdParty_Package(libressl 3.1.3 https://ftp.openbsd.org/pub/OpenBSD/LibreSSL/libressl-3.1.3.tar.gz SHA256=c76b0316acf612ecb62f5cb014a20d972a663bd9e40abf952a86f3b998b69fa0)
Fetch3rdParty_Package(libressl 3.1.3 https://ftp.openbsd.org/pub/OpenBSD/LibreSSL/libressl-3.1.3.tar.gz SHA256=c76b0316acf612ecb62f5cb014a20d972a663bd9e40abf952a86f3b998b69fa0)
Fetch3rdParty_Package(absl 20200225.2 https://codeload.github.com/abseil/abseil-cpp/tar.gz/20200225.2 SHA256=f41868f7a938605c92936230081175d1eae87f6ea2c248f41077c8f88316f111)
Fetch3rdParty_Package(crc32c 1.1.1 https://codeload.github.com/google/crc32c/tar.gz/1.1.1 SHA256=a6533f45b1670b5d59b38a514d82b09c6fb70cc1050467220216335e873074e8)
Fetch3rdParty_Package(google-cloud-cpp 1.14.0 https://codeload.github.com/googleapis/google-cloud-cpp/tar.gz/v1.14.0 SHA256=839b2d4dcb36a671734dac6b30ea8c298bbeaafcf7a45ee4a7d7aa5986b16569)
Fetch3rdParty_File(testng java 6.14.3 jar https://repo1.maven.org/maven2/org/testng/testng/6.14.3/testng-6.14.3.jar MD5=9f17a8f9e99165e148c42b21f4b63d7c)
Fetch3rdParty_File(jcommander java 1.72 jar https://repo1.maven.org/maven2/com/beust/jcommander/1.72/jcommander-1.72.jar MD5=9fde6bc0ba1032eceb7267fd1ad1657b)
Fetch3rdParty_File(testng java 6.14.3 jar https://repo1.maven.org/maven2/org/testng/testng/6.14.3/testng-6.14.3.jar MD5=9f17a8f9e99165e148c42b21f4b63d7c)
Fetch3rdParty_File(jcommander java 1.72 jar https://repo1.maven.org/maven2/com/beust/jcommander/1.72/jcommander-1.72.jar MD5=9fde6bc0ba1032eceb7267fd1ad1657b)
Fetch3rdParty_FileTarget(google_nlohmann google/cloud/storage/internal nlohmann_json.hpp 3.4.0 https://raw.githubusercontent.com/nlohmann/json/v3.4.0/single_include/nlohmann/json.hpp MD5=27f3760c1d3a0fff7d8a2407d8db8f9d)
endfunction()
CMakeLists.txt
View file @ 5b20c9a1
......@@ -12,12 +12,16 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH}
include(GNUInstallDirs)
set (TEST_DATA_PATH "" CACHE PATH "Test data path")
set (TEST_SEGY_FILE "" CACHE FILEPATH "SEGY test file")
set (TEST_SEGY_HEADER_FILE "" CACHE FILEPATH "Json file describing the header data of SEGY test file")
set (TEST_URL "" CACHE STRING "Test url for OpenOptions")
set (TEST_CONNECTION "" CACHE STRING "Test connection string for OpenOptions")
set (OPENVDS_LIBDIR_SUFFIX "" CACHE STRING "Suffix to add to lib directories on linux")
set (OPENVDS_PYTHON_VERSION "" CACHE STRING "Exact Python 3 version to discover")
set (OPENVDS_JAVA_VERSION "" CACHE STRING "Exact Java version to discover")
set (OPENVDS_3RD_PARTY_DIR "" CACHE PATH "Path to put 3rd party source code, should be absolute path")
option(FORCE_NO_LIBDIR_SUFFIX "Force there to be no suffix on lib directories linux")
get_property(LIB64 GLOBAL PROPERTY FIND_LIBRARY_USE_LIB64_PATHS)
......@@ -144,9 +148,6 @@ build3rdparty()
include(CMake/CopyDllForTarget.cmake)
include(CMake/AddSystemHeadersToTarget.cmake)
set(CMAKE_C_VISIBILITY_PRESET hidden)
set(CMAKE_CXX_VISIBILITY_PRESET hidden)
add_subdirectory(src)
add_subdirectory(tools)
......
docker/README.md
View file @ 5b20c9a1
......@@ -2,6 +2,8 @@ This folder contains the different docker files used for building OpenVDS for di
There is also a devenv docker file setting up a minimal developement environment. Following are some basic instructions for setting up the development environment on Windows.
It is important to shut down visual studio, since it will lock some files and building the docker image will not work.
1. Build an image.
From the root OpenVDS source folder write the following command:
$ docker build --tag ovds-alpine-devenv -f docker/alpine-devenv.Dockerfile .
......
docker/centos7.Dockerfile
View file @ 5b20c9a1
FROM centos:7
FROM docker.io/centos:7
RUN yum install -y centos-release-scl epel-release
RUN yum install -y rh-python36-scldevel rh-python36-numpy rh-python36-python-pip devtoolset-8
SHELL [ "/usr/bin/scl", "enable", "devtoolset-8", "rh-python36" ]
RUN curl -OL https://github.com/Kitware/CMake/releases/download/v3.17.2/cmake-3.17.2-Linux-x86_64.tar.gz
RUN tar xzvf cmake-3.17.2-Linux-x86_64.tar.gz -C /opt
RUN ln -s /opt/cmake-3.17.2-Linux-x86_64/bin/* /usr/bin/
RUN scl enable rh-python36 devtoolset-8 'pip install ninja scikit-build'
RUN pip install ninja scikit-build
RUN yum install -y git doxygen java-1.8.0-openjdk-devel libxml2-devel zlib-devel boost169-devel openssl-devel libcurl-devel libuv-devel libuuid-devel
# cmake -DCMAKE_BUILD_TYPE=Debug -GNinja -DBOOST_INCLUDEDIR=/usr/include/boost169 -DBOOST_LIBRARYDIR=/usr/lib64/boost169 ..
docker/centos8.Dockerfile 0 → 100644
View file @ 5b20c9a1
FROM docker.io/centos:8
#RUN yum install -y centos-release-scl epel-release
#
#RUN yum install -y rh-python36-scldevel rh-python36-numpy rh-python36-python-pip devtoolset-8
#
#SHELL [ "/usr/bin/scl", "enable", "devtoolset-8", "rh-python36" ]
#
RUN dnf -y install dnf-plugins-core
RUN dnf -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
RUN dnf config-manager --set-enabled PowerTools epel-modular
RUN yum install -y python3 gcc boost-devel git doxygen java-1.8.0-openjdk-devel libxml2-devel zlib-devel boost169-devel openssl-devel libcurl-devel libuv libuuid-devel
RUN dnf -y module enable libuv
RUN yum install -y libuv-devel
RUN yum -y group install "Development Tools"
RUN curl -OL https://github.com/Kitware/CMake/releases/download/v3.17.2/cmake-3.17.2-Linux-x86_64.tar.gz
RUN tar xzvf cmake-3.17.2-Linux-x86_64.tar.gz -C /opt
RUN ln -s /opt/cmake-3.17.2-Linux-x86_64/bin/* /usr/bin/
RUN python3 -m pip install scikit-build ninja
#
## cmake -DCMAKE_BUILD_TYPE=Debug -GNinja ..
examples/JupyterNotebook/OpenVDS_basic_usage.ipynb
View file @ 5b20c9a1
%% Cell type:code id: tags:
``` python
import openvds
import matplotlib.pyplot as plt
import matplotlib
import matplotlib.image as mpimg
import numpy as np
import time
%matplotlib inline
matplotlib.rcParams['figure.figsize'] = (16, 16)
```
%% Cell type:markdown id: tags:
## Open a VDS connection and get the Layout object for the connection
Specific OpenOptions has to be used to open a connection to different cloud vendors. The AWS open options also relies on the AWS standardized credentials file in $HOME/.aws/credentials
Specific OpenOptions can to be used to open a connection to different cloud vendors. The AWS open options also relies on the AWS standardized credentials file in $HOME/.aws/credentials
Alternativly a url and connection string pair can be used. This scheme is descibed here:
http://osdu.pages.community.opengroup.org/platform/domain-data-mgmt-services/seismic/open-vds/connection.html
%% Cell type:code id: tags:
``` python
error = openvds.Error()
openOptions = openvds.AWSOpenOptions("bucket", "folder_in_bucket", "region")
#openOptions = openvds.AWSOpenOptions("bucket", "folder_in_bucket", "region")
#openOptions = openvds.AzureOpenOptions("connection_string", "container", "blob") #blob is ignored for now
#openOptions = openvds.GoogleOpenOptions("bucket", "folder_in_bucket")
vds = openvds.open(openOptions, error)
#vds = openvds.open(openOptions)
vds = openvds.open("some url", "optional connection string")
layout = openvds.getLayout(vds)
```
%% Cell type:markdown id: tags:
## Layout
The layout can be used to retrieve properties about the VDS.
%% Cell type:code id: tags:
``` python
print("ChannelCount: {}".format(layout.getChannelCount()))
for i in range(layout.getChannelCount()):
print(" Channel Name: {}".format(layout.getChannelName(i)))
print(" Value range: {} - {}".format(layout.getChannelValueRangeMin(i), layout.getChannelValueRangeMax(i)))
print("Dimensionality: {}".format(layout.getDimensionality()))
for i in range(layout.getDimensionality()):
print(" Dimension name: {}".format(layout.getDimensionName(i)))
print(" Number of samples: {}".format(layout.getDimensionNumSamples(i)))
print(" Coordinate min max {} - {}".format(layout.getDimensionMin(i), layout.getDimensionMax(i)))
```
%% Cell type:markdown id: tags:
## Metadata accessor
Metadata can be accessed through the layout objects getMetadataXX function. The type is part of the key and no implicit conversion will occur.
- getMetadataInt
- getMetadataIntVector2
- getMetadataIntVector3
- getMetadataIntVector4
- getMetadataFloat
- getMetadataFloatVector2
- getMetadataFloatVector3
- getMetadataFloatVector4
- getMetadataDouble
- getMetadataDoubleVector2
- getMetadataDoubleVector3
- getMetadataDoubleVector4
- getMetadataString
- getMetadataBLOB
%% Cell type:code id: tags:
``` python
MetaType = openvds.MetadataType
if layout.isMetadataDoubleVector2Available("SurveyCoordinateSystem", "Origin"):
print("SurveyCoordinateSystem::Origin: {}".format(layout.getMetadataDoubleVector2("SurveyCoordinateSystem", "Origin")))
if layout.isMetadataDoubleVector2Available("SurveyCoordinateSystem", "InlineSpacing"):
print("SurveyCoordinateSystem::InlineSpacing: {}".format(layout.getMetadataDoubleVector2("SurveyCoordinateSystem", "InlineSpacing")))
if layout.isMetadataDoubleVector2Available("SurveyCoordinateSystem", "CrosslineSpacing"):
print("SurveyCoordinateSystem::CrosslineSpacing: {}".format(layout.getMetadataDoubleVector2("SurveyCoordinateSystem", "CrosslineSpacing")))
#layout.getMetadata("SEGY", "TextHeader", openvds.core.MetadataType.BLOB)
#layout.getMetadata("SEGY", "BinaryHeader", openvds.core.MetadataType.BLOB)
```
%% Cell type:markdown id: tags:
## The AccessManager api
The AccessManager api can be used to retieve data from the vds
%% Cell type:code id: tags:
``` python
accessManager = openvds.VolumeDataAccessManager(vds)
axisDescriptors = [layout.getAxisDescriptor(dim) for dim in range(layout.getDimensionality())]
sliceType = 'inline'
sliceIndex = 55
sliceDimension = 2 if sliceType == 'inline' else 1 if sliceType == 'crossline' else 0 if sliceType == 'timeslice' else 0 if sliceType == 'depthslice' else -1
min = tuple(sliceIndex + 0 if dim == sliceDimension else 0 for dim in range(6))
max = tuple(sliceIndex + 1 if dim == sliceDimension else layout.getDimensionNumSamples(dim) for dim in range(6))
req = accessManager.requestVolumeSubset(min, max, format = openvds.VolumeDataChannelDescriptor.Format.Format_R32)
height = max[0] if sliceDimension != 0 else max[1]
width = max[2] if sliceDimension != 2 else max[1]
data = req.data.reshape(width, height).transpose()
cmap=plt.get_cmap("seismic")
plt.set_cmap(cmap)
plt.imshow(data)
```
......
examples/SliceDump/GenerateVDS.h
View file @ 5b20c9a1
......@@ -4,6 +4,7 @@
#include <OpenVDS/VolumeDataChannelDescriptor.h>
#include <OpenVDS/KnownMetadata.h>
#include <OpenVDS/GlobalMetadataCommon.h>
#include <OpenVDS/MetadataContainer.h>
#include <OpenVDS/VolumeDataLayout.h>
#include <OpenVDS/VolumeDataAccess.h>
#include <OpenVDS/SimplexNoiseKernel.h>
......
examples/SliceDump/main.cpp
View file @ 5b20c9a1
......@@ -72,28 +72,16 @@ int main(int argc, char **argv)
options.positional_help("<output file>");
std::string file_name;
std::string bucket;
std::string region;
std::string url;
std::string connectionString;
std::string container;
int azureParallelismFactor = 0;
std::string object;
std::string azurePresignBase;
std::string azurePresignSuffix;
std::string axis = "0,1,2";
int axis_position = std::numeric_limits<int>::min();
int32_t output_width = 500;
int32_t output_height = 500;
bool generate_noise = false;
options.add_option("", "", "bucket", "AWS S3 bucket to download from.", cxxopts::value(bucket), "<string>");
options.add_option("", "", "region", "AWS region of bucket to download from.", cxxopts::value<std::string>(region), "<string>");
options.add_option("", "", "url", "Url for the VDS", cxxopts::value<std::string>(url), "<string>");
options.add_option("", "", "connection-string", "Azure Blob Storage connection string.", cxxopts::value<std::string>(connectionString), "<string>");
options.add_option("", "", "container", "Azure Blob Storage container to download from .", cxxopts::value<std::string>(container), "<string>");
options.add_option("", "", "parallelism-factor", "Azure parallelism factor.", cxxopts::value<int>(azureParallelismFactor), "<value>");
options.add_option("", "", "object", "ObjectId of the VDS", cxxopts::value(object), "<string>");
options.add_option("", "", "azure-presign-base", "Base URL for presigned Azure requests", cxxopts::value(azurePresignBase), "<value>");
options.add_option("", "", "azure-presign-suffix", "Suffix of the presigned URL for Azure requests", cxxopts::value(azurePresignSuffix), "<value>");
options.add_option("", "", "axis", "Axis mapping. Comma seperated list. First digite is the axis for the slice. "
"Second is the x axis and third is the y axis", cxxopts::value(axis), "<axis id>");
options.add_option("", "", "noise", "Generate a noise VDS in memory, and grab a slice from this (default off).", cxxopts::value(generate_noise), "<enable>");
......@@ -107,23 +95,31 @@ int main(int argc, char **argv)
{
options.parse(argc, argv);
}
catch(cxxopts::OptionParseException *e)
catch(cxxopts::OptionParseException &e)
{
fmt::print(stderr, "{}\n", e->what());
fmt::print(stderr, "{}\n", e.what());
return EXIT_FAILURE;
}
if (bucket.empty() && container.empty() && !generate_noise && azurePresignBase.empty())
OpenVDS::Error error;
OpenVDS::VDSHandle handle = nullptr;
if (generate_noise)
{
handle = generateSimpleInMemory3DVDS(60,60,60, OpenVDS::VolumeDataChannelDescriptor::Format_U8);
if (handle)
fill3DVDSWithNoise(handle);
}
else if(url.empty())
{
fmt::print(stderr, "Either an Azure Blob Storage container name or an AWS S3 bucket name must be specified");
fmt::print(stderr, "Either specify noise generation or provide a url.\n");
return EXIT_FAILURE;
}
if(azureParallelismFactor && container.empty())
else
{
fmt::print(stderr, "Cannot specify parallelism-factor with other backends than Azure");
handle = OpenVDS::Open(url, connectionString, error);
}
if (file_name.empty())
{
fmt::print(stderr, "No output filename specified");
......@@ -139,46 +135,6 @@ int main(int argc, char **argv)
}
fmt::print(stdout, "Using axis mapping [{}, {}, {}]\n", axis_mapper[0], axis_mapper[1], axis_mapper[2]);
OpenVDS::Error error;
OpenVDS::VDSHandle handle = nullptr;
if (generate_noise)
{
handle = generateSimpleInMemory3DVDS(60,60,60, OpenVDS::VolumeDataChannelDescriptor::Format_U8);
if (handle)
fill3DVDSWithNoise(handle);
}
else if(!bucket.empty())
{
OpenVDS::AWSOpenOptions openOptions;
openOptions.key = object;
openOptions.bucket = bucket;
openOptions.region = region;
handle = OpenVDS::Open(openOptions, error);
}
else if(!container.empty())
{
OpenVDS::AzureOpenOptions openOptions;
openOptions.blob = object;
openOptions.container = container;
openOptions.connectionString = connectionString;
if(azureParallelismFactor)
{
openOptions.parallelism_factor = azureParallelismFactor;
}
handle = OpenVDS::Open(openOptions, error);
}
else if (!azurePresignBase.empty())
{
OpenVDS::AzurePresignedOpenOptions openOptions;
openOptions.baseUrl = azurePresignBase;
openOptions.urlSuffix = azurePresignSuffix;
handle = OpenVDS::Open(openOptions, error);
}
if (!handle)
{
......
examples/SliceServer/sliceserver.py
View file @ 5b20c9a1
......@@ -5,6 +5,7 @@ import re
import io
import numpy
import openvds
import sys
from http import HTTPStatus
from flask import Flask, make_response, request, abort
......@@ -13,49 +14,55 @@ from PIL import Image
VDSs = {}
parser = argparse.ArgumentParser(description="Starts a service that creates png images of the slices of a VDS.")
parser.add_argument("--bucket", help="The AWS bucket of the VDSs to start serving")
parser.add_argument("--region", help="The AWS region of the VDSs to start serving")
parser.add_argument("--connection-string", help="The Azure Blob Storage connection string.");
parser.add_argument("--container", help="Azure Blob Storage container of the VDSs to start serving");
parser.add_argument("--parallelism-factor", help="Azure parallelism factor.", type=int);
parser.add_argument("--url", help="The AWS bucket of the VDSs to start serving")
parser.add_argument("--connection", help="The Azure Blob Storage connection string.");
parser.add_argument("--key", help="The key of the VDSs to start serving")
parser.add_argument("--allow-remote", action='store_true', help="Allow remote connections to server.")
parser.add_argument("--allow-remote", action='store_true', help="Allow remote connections to server.")
if len(sys.argv) == 1:
parser.print_usage()
sys.exit(0)
args = parser.parse_args()
persistentID = args.key if args.key else ""
if args.bucket:
opt = openvds.AWSOpenOptions(bucket=args.bucket, region=args.region if args.region else "", key=args.key if args.key else "")
elif args.container:
opt = openvds.AzureOpenOptions(container=args.container, connectionString=args.connection_string if args.connection_string else "", blob=args.key if args.key else "")
if args.parallelism_factor is not None:
opt.parallelismFactor = args.parallelism_factor
persistentID = args.key if args.key else ""
VDSs[persistentID] = openvds.open(opt)
VDSs[persistentID] = openvds.open(args.url)
app = Flask(__name__)
def handle_request(vds, sliceType, sliceIndex):
accessManager = openvds.getAccessManager(vds)
layout = openvds.getLayout(vds)
axisDescriptors = [layout.getAxisDescriptor(dim) for dim in range(layout.getDimensionality())]
sliceDimension = 2 if sliceType == 'inline' else 1 if sliceType == 'crossline' else 0 if sliceType == 'timeslice' else 0 if sliceType == 'depthslice' else -1
min = tuple(sliceIndex + 0 if dim == sliceDimension else 0 for dim in range(6))
max = tuple(sliceIndex + 1 if dim == sliceDimension else layout.getDimensionNumSamples(dim) for dim in range(6))
req = accessManager.requestVolumeSubset(min, max, format = openvds.VolumeDataChannelDescriptor.Format.Format_U8)
height = max[0] if sliceDimension != 0 else max[1]
width = max[2] if sliceDimension != 2 else max[1]
data = req.data.reshape(width, height).transpose()
image = Image.fromarray(data, mode='L')
with io.BytesIO() as output:
image.save(output, format='PNG')
responseData = output.getvalue()
response = make_response((responseData, HTTPStatus.OK))
response.headers['Content-Type'] = 'image/png'
response.headers['Content-Disposition'] = 'inline; filename={0}{1}.png'.format(sliceType, sliceIndex)
return response
@app.route('/<sliceType>/<int:sliceIndex>')
def vds_data(sliceType, sliceIndex):
if len(VDSs) == 0:
abort(HTTPStatus.NOT_FOUND)
return handle_request(next(iter(VDSs.values())), sliceType, sliceIndex)
@app.route('/<persistentID>/<sliceType>/<int:sliceIndex>')
def vds_data(persistentID, sliceType, sliceIndex):
def vds_data_persistentID(persistentID, sliceType, sliceIndex):
if persistentID in VDSs:
accessManager = openvds.getAccessManager(VDSs[persistentID])
layout = openvds.getLayout(VDSs[persistentID])
axisDescriptors = [layout.getAxisDescriptor(dim) for dim in range(layout.getDimensionality())]
sliceDimension = 2 if sliceType == 'inline' else 1 if sliceType == 'crossline' else 0 if sliceType == 'timeslice' else 0 if sliceType == 'depthslice' else -1
min = tuple(sliceIndex + 0 if dim == sliceDimension else 0 for dim in range(6))
max = tuple(sliceIndex + 1 if dim == sliceDimension else layout.getDimensionNumSamples(dim) for dim in range(6))
req = accessManager.requestVolumeSubset(min, max, format = openvds.VolumeDataChannelDescriptor.Format.Format_U8)
height = max[0] if sliceDimension != 0 else max[1]
width = max[2] if sliceDimension != 2 else max[1]
data = req.data.reshape(width, height).transpose()
image = Image.fromarray(data, mode='L')
with io.BytesIO() as output:
image.save(output, format='PNG')
responseData = output.getvalue()
response = make_response((responseData, HTTPStatus.OK))
response.headers['Content-Type'] = 'image/png'
response.headers['Content-Disposition'] = 'inline; filename={0}{1}.png'.format(sliceType, sliceIndex)
return response
return handle_request(VDSs[persistentID], slizeType, sliceIndex)
abort(HTTPStatus.NOT_FOUND)
if __name__ == '__main__':
app.run(debug=True, host='0.0.0.0' if args.allow_remote else '127.0.0.1')
app.run(debug=True, host='0.0.0.0' if args.allow_remote else '127.0.0.1')
java/CMakeLists.txt
View file @ 5b20c9a1
include(UseJava)
set(OPENVDS_JAVA_VERSION 0.0.6)
set(CMAKE_JAVA_COMPILE_FLAGS "-source" "1.8" "-target" "1.8")
set(GENERATED_JNI_HEADERS_DIR ${CMAKE_CURRENT_SOURCE_DIR}/cpp/generated)
#file(REMOVE_RECURSE ${GENERATED_JNI_HEADERS_DIR})
file(MAKE_DIRECTORY ${GENERATED_JNI_HEADERS_DIR})
......@@ -79,6 +83,7 @@ set(JAVA_TEST_SOURCES_FILES
java/test/org/opengroup/openvds/AmazonS3Test.java
java/test/org/opengroup/openvds/MemoryVdsGeneratorTest.java
java/test/org/opengroup/openvds/CreateVDSTest.java
java/test/org/opengroup/openvds/WriteDataTest.java
)
add_jar(openvds-java-test
......@@ -105,6 +110,7 @@ add_library(openvds-javacpp
cpp/src/VolumeDataLayout.cpp
cpp/src/VolumeDataPage.cpp
cpp/src/VolumeDataPageAccessor.cpp
cpp/src/WriteVolumeData.cpp
)
set_target_properties(openvds-javacpp PROPERTIES
......
java/cpp/src/AzureVdsGenerator.cpp
View file @ 5b20c9a1
/*
* Copyright 2019 The Open Group
* Copyright 2019 INT, Inc.
* Copyright 2020 The Open Group
* Copyright 2020 Bluware, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -18,25 +18,34 @@
#include <org_opengroup_openvds_AzureVdsGenerator.h>
#include <CommonJni.h>
#include <OpenVDS/VolumeDataChannelDescriptor.h>
// Begin of stuff from open-vds-master/examples/SliceDump/GenerateVDS.h
#include <OpenVDS/OpenVDS.h>
#include <OpenVDS/VolumeDataLayoutDescriptor.h>
#include <OpenVDS/VolumeDataAxisDescriptor.h>
#include <OpenVDS/VolumeDataChannelDescriptor.h>
#include <OpenVDS/KnownMetadata.h>
#include <OpenVDS/GlobalMetadataCommon.h>
#include <OpenVDS/VolumeDataLayout.h>
#include <OpenVDS/VolumeDataAccess.h>
#include <OpenVDS/MetadataContainer.h>