From 7c258ccab80d7d9f638ff648e2ab14df503f53c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Lind?= Date: Mon, 28 Jun 2021 10:05:08 +0200 Subject: [PATCH 1/5] Make it possible to disable the init/deinit of the AWS sdk --- docs/connection.rst | 1 + src/OpenVDS/IO/IOManagerAWS.cpp | 9 +++++++-- src/OpenVDS/IO/IOManagerAWS.h | 1 + src/OpenVDS/OpenVDS.cpp | 15 +++++++++++++++ src/OpenVDS/OpenVDS/OpenVDS.h | 5 +++-- 5 files changed, 27 insertions(+), 4 deletions(-) diff --git a/docs/connection.rst b/docs/connection.rst index 71ace16f..45383283 100644 --- a/docs/connection.rst +++ b/docs/connection.rst @@ -40,6 +40,7 @@ The recognised keys for ``s3`` connection string are - LogLevel (Note: this is to control the AWS specific logs. Possible values: Off, Fatal, Error, Warn, Info, Debug, Trace) - ConnectionTimoutMs - RequestTimeoutMs + - DisableInitApi (This will disable init/deinit of AWS sdk. Possible values: 1, On, True, Yes (case insensitive), all other values are false.) ``azure`` has two modes. The connection string mode or the bearer token mode. If the connection string contains a key with name BearerToken it will parse the diff --git a/src/OpenVDS/IO/IOManagerAWS.cpp b/src/OpenVDS/IO/IOManagerAWS.cpp index 4139ba63..e6cbf8d3 100644 --- a/src/OpenVDS/IO/IOManagerAWS.cpp +++ b/src/OpenVDS/IO/IOManagerAWS.cpp @@ -326,6 +326,7 @@ namespace OpenVDS , m_region(openOptions.region) , m_bucket(openOptions.bucket) , m_objectId(openOptions.key) + , m_disableInitializeSdk(openOptions.disableInitApi) { if (m_bucket.empty()) { @@ -336,7 +337,9 @@ namespace OpenVDS if (m_objectId.size() && m_objectId[m_objectId.size() -1] == '/') m_objectId.resize(m_objectId.size() - 1); - initializeAWSSDK(openOptions.logFilenamePrefix, openOptions.loglevel); + + if (!m_disableInitializeSdk) + initializeAWSSDK(openOptions.logFilenamePrefix, openOptions.loglevel); Aws::String profileName = ""; @@ -455,7 +458,9 @@ namespace OpenVDS IOManagerAWS::~IOManagerAWS() { m_s3Client.reset(); - deinitializeAWSSDK(); + + if (!m_disableInitializeSdk) + deinitializeAWSSDK(); } std::shared_ptr IOManagerAWS::ReadObjectInfo(const std::string &objectName, std::shared_ptr handler) diff --git a/src/OpenVDS/IO/IOManagerAWS.h b/src/OpenVDS/IO/IOManagerAWS.h index a071d16c..aab71689 100644 --- a/src/OpenVDS/IO/IOManagerAWS.h +++ b/src/OpenVDS/IO/IOManagerAWS.h @@ -100,6 +100,7 @@ namespace OpenVDS std::string m_bucket; std::string m_objectId; std::unique_ptr m_s3Client; + bool m_disableInitializeSdk; }; } #endif //IOMANAGERAWS_H diff --git a/src/OpenVDS/OpenVDS.cpp b/src/OpenVDS/OpenVDS.cpp index 79cc0be6..ee211b79 100644 --- a/src/OpenVDS/OpenVDS.cpp +++ b/src/OpenVDS/OpenVDS.cpp @@ -172,6 +172,21 @@ static std::unique_ptr createS3OpenOptions(const StringWrapper &url return nullptr; } } + else if (connectionPair.first == "disableinitapi") + { + auto value = connectionPair.second; + std::transform(value.begin(), value.end(), value.begin(), asciitolower); + static const std::string trueValues[] = { "1", "on", "true", "yes" }; + openOptions->disableInitApi = false; + for (auto& trueValue : trueValues) + { + if (value == trueValue) + { + openOptions->disableInitApi = true; + break; + } + } + } else { error.code = -1; diff --git a/src/OpenVDS/OpenVDS/OpenVDS.h b/src/OpenVDS/OpenVDS/OpenVDS.h index 2fc543de..f3ad63ce 100644 --- a/src/OpenVDS/OpenVDS/OpenVDS.h +++ b/src/OpenVDS/OpenVDS/OpenVDS.h @@ -92,8 +92,9 @@ struct AWSOpenOptions : OpenOptions std::string loglevel; int connectionTimeoutMs; int requestTimeoutMs; + bool disableInitApi; - AWSOpenOptions() : OpenOptions(AWS), connectionTimeoutMs(3000), requestTimeoutMs(6000) {} + AWSOpenOptions() : OpenOptions(AWS), connectionTimeoutMs(3000), requestTimeoutMs(6000), disableInitApi(false) {} /// /// AWSOpenOptions constructor /// @@ -115,7 +116,7 @@ struct AWSOpenOptions : OpenOptions /// /// This paramter allows to override the time a request can take /// - AWSOpenOptions(std::string const & bucket, std::string const & key, std::string const & region = std::string(), std::string const & endpointOverride = std::string(), int connectionTimeoutMs = 3000, int requestTimeoutMs = 6000) : OpenOptions(AWS), bucket(bucket), key(key), region(region), endpointOverride(endpointOverride), connectionTimeoutMs(connectionTimeoutMs), requestTimeoutMs(requestTimeoutMs) {} + AWSOpenOptions(std::string const & bucket, std::string const & key, std::string const & region = std::string(), std::string const & endpointOverride = std::string(), int connectionTimeoutMs = 3000, int requestTimeoutMs = 6000, bool disableInitApi = false) : OpenOptions(AWS), bucket(bucket), key(key), region(region), endpointOverride(endpointOverride), connectionTimeoutMs(connectionTimeoutMs), requestTimeoutMs(requestTimeoutMs), disableInitApi(disableInitApi) {} }; /// -- GitLab From 54df7c47dfe7c71f5ee8cf405af10de41b2440e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Lind?= Date: Thu, 17 Jun 2021 08:35:39 +0200 Subject: [PATCH 2/5] DMS Refresh Token --- docs/connection.rst | 26 ++++++++++++++--- src/OpenVDS/CMakeLists.txt | 4 +-- src/OpenVDS/IO/IOManagerDms.cpp | 25 +++++++++++++++- src/OpenVDS/IO/IOManagerDms.h | 5 ++++ src/OpenVDS/IO/IORefreshToken.cpp | 48 +++++++++++++++++++++++++++++++ src/OpenVDS/IO/IORefreshToken.h | 27 +++++++++++++++++ src/OpenVDS/OpenVDS.cpp | 16 ++++++++--- src/OpenVDS/OpenVDS/OpenVDS.h | 13 +++++++-- 8 files changed, 151 insertions(+), 13 deletions(-) create mode 100644 src/OpenVDS/IO/IORefreshToken.cpp create mode 100644 src/OpenVDS/IO/IORefreshToken.h diff --git a/docs/connection.rst b/docs/connection.rst index 45383283..1b397bea 100644 --- a/docs/connection.rst +++ b/docs/connection.rst @@ -62,10 +62,28 @@ The Token is the OAuth2 token. ``sd`` will accept the following keys - - sdauthorityurl - - sdapikey - - sdtoken - - loglevel + - SdAuthorityUrl + - SdApiKey + - SdToken + - LogLevel + +The token `SdToken` is typically a short lived token. The sd IO backend also +supports using a `RefreshToken` flow. Then its not needed to specify the sdtoken +but following key/value pairs have to specified: + + - AuthTokenUrl + - ClientId + - ClientSecret + - RefreshToken + +When these paramaters are provided OpenVDS will send a "x-www-form-urlencoded" +http request to `AuthTokenUrl` with supplied values and also added: +grant_type=refresh_token&scope=openid email. The used refresh token will be +updated If the response contains a refresh_token member. It is the access_token +member of the return that will be used as the sdtoken until it expires. + +NOTE: Key values for `sd` can use `snake_case` instead of `CammelCase` to match the +look and feel of the response from the token provider. ``azureSAS`` will accept the following keys diff --git a/src/OpenVDS/CMakeLists.txt b/src/OpenVDS/CMakeLists.txt index 9eada19f..47ec4058 100644 --- a/src/OpenVDS/CMakeLists.txt +++ b/src/OpenVDS/CMakeLists.txt @@ -42,7 +42,7 @@ set(SOURCE_FILES VDS/VolumeIndexer.cpp VDS/Env.cpp ${COMMON_DIR}/Base64/Base64.cpp - ) + "IO/IORefreshToken.cpp") set (PRIVATE_HEADER_FILES IO/File.h @@ -94,7 +94,7 @@ set (PRIVATE_HEADER_FILES VDS/GlobalStateImpl.h ${COMMON_DIR}/Base64/Base64.h ${COMMON_DIR}/ThreadPool/ThreadPool.h - ) + "IO/IORefreshToken.h") file(GLOB EXPORTED_HEADER_FILES OpenVDS/*.h) diff --git a/src/OpenVDS/IO/IOManagerDms.cpp b/src/OpenVDS/IO/IOManagerDms.cpp index 2917867a..b11ab808 100644 --- a/src/OpenVDS/IO/IOManagerDms.cpp +++ b/src/OpenVDS/IO/IOManagerDms.cpp @@ -10,6 +10,9 @@ #endif #include +#include "IOManagerCurl.h" +#include "IORefreshToken.h" + namespace OpenVDS { GetHeadRequestDms::GetHeadRequestDms(seismicdrive::SDGenericDataset &dataset, const std::string& id, const std::shared_ptr& handler) @@ -178,6 +181,12 @@ namespace OpenVDS }); } + std::string IOManagerDms::AuthProviderCallback(const void* data) + { + IOManagerDms* iomanager = static_cast(const_cast(data)); + return iomanager->m_tokenRefresher->newToken(); + } + IOManagerDms::IOManagerDms(const DMSOpenOptions& openOptions, IOManager::AccessPattern accessPatttern, Error& error) : IOManager(openOptions.connectionType) , m_opened(false) @@ -195,9 +204,23 @@ namespace OpenVDS m_filename = openOptions.datasetPath.substr(it+1); } } + + try { m_sdManager.reset(new seismicdrive::SDManager(openOptions.sdAuthorityUrl, openOptions.sdApiKey, openOptions.logLevel)); - m_sdManager->setAuthProviderFromString(openOptions.sdToken); + if (openOptions.authTokenUrl.size() && openOptions.clientId.size() && openOptions.clientSecret.size() && openOptions.refreshToken.size()) + { + m_curlHandler.reset(new CurlHandler(error)); + if (error.code != 0) + return; + m_tokenRefresher.reset(new TokenRefresher(openOptions.authTokenUrl, openOptions.clientId, openOptions.clientSecret, openOptions.refreshToken, *m_curlHandler, std::function())); + m_sdManager->setAuthProviderCallback(&AuthProviderCallback, this); + } + else + { + m_sdManager->setAuthProviderFromString(openOptions.sdToken); + } + m_dataset.reset(new seismicdrive::SDGenericDataset(m_sdManager.get(), openOptions.datasetPath, openOptions.logLevel != 0)); seismicdrive::SDDatasetDisposition disposition = seismicdrive::SDDatasetDisposition::READ_ONLY; diff --git a/src/OpenVDS/IO/IOManagerDms.h b/src/OpenVDS/IO/IOManagerDms.h index 4af77bc4..3d490f45 100644 --- a/src/OpenVDS/IO/IOManagerDms.h +++ b/src/OpenVDS/IO/IOManagerDms.h @@ -24,6 +24,8 @@ namespace OpenVDS { + class CurlHandler; + class TokenRefresher; class GetHeadRequestDms : public RequestImpl { public: @@ -75,11 +77,14 @@ namespace OpenVDS std::shared_ptr WriteObject(const std::string& requestName, const std::string& contentDispostionFilename, const std::string& contentType, const std::vector>& metadataHeader, std::shared_ptr> data, std::function completedCallback = nullptr) override; private: + static std::string AuthProviderCallback(const void* data); std::unique_ptr m_sdManager; std::unique_ptr m_dataset; std::string m_filename; bool m_opened; ThreadPool m_threadPool; + std::unique_ptr m_curlHandler; + std::unique_ptr m_tokenRefresher; }; } diff --git a/src/OpenVDS/IO/IORefreshToken.cpp b/src/OpenVDS/IO/IORefreshToken.cpp new file mode 100644 index 00000000..37c3df3b --- /dev/null +++ b/src/OpenVDS/IO/IORefreshToken.cpp @@ -0,0 +1,48 @@ +#include "IORefreshToken.h" + +#include +#include + +namespace OpenVDS +{ + TokenRefresher::TokenRefresher(const std::string& authTokenUrl, const std::string &clientId, const std::string &clientSecret, const std::string& refreshToken, CurlHandler& curlHandler, const std::function &newTokenCallback) + : m_authTokenUrl(authTokenUrl) + , m_clientId(clientId) + , m_clientSecret(clientSecret) + , m_refreshToken(refreshToken) + , m_curlHandler(curlHandler) + , m_newTokenCallback(newTokenCallback) + { + } + + std::string TokenRefresher::newToken() + { + std::vector>> data; + data.emplace_back(std::make_shared>()); + + std::string form = fmt::format("grant_type={}&client_id={}&client_secret={}&refresh_token={}&scope={}", "refresh_token", m_clientId, m_clientSecret, m_refreshToken, "openid email"); + data.back()->insert(data.back()->end(), form.begin(), form.end()); + + std::shared_ptr request = std::make_shared("refresh_token", std::function()); + + std::vector headers; + headers.emplace_back("Content-Type: application/x-www-form-urlencoded"); + m_curlHandler.addUploadRequest(request, m_authTokenUrl, headers, true, std::move(data), form.size()); + Error error; + request->WaitForFinish(error); + if (error.code || !request->m_uploadHandler) + return ""; + std::string respons_data; + respons_data.insert(respons_data.end(), request->m_uploadHandler->responsData.begin(), request->m_uploadHandler->responsData.end()); + Json::Value value; + if (!Json::Reader().parse(respons_data, value)) + { + return ""; + } + if (value.isMember("refresh_token")) + { + m_refreshToken = value["refresh_token"].asString(); + } + return value["access_token"].asString(); + } +} \ No newline at end of file diff --git a/src/OpenVDS/IO/IORefreshToken.h b/src/OpenVDS/IO/IORefreshToken.h new file mode 100644 index 00000000..8f3fe76c --- /dev/null +++ b/src/OpenVDS/IO/IORefreshToken.h @@ -0,0 +1,27 @@ +#ifndef IOREFRESHTOKEN +#define IOREFRESHTOKEN + +#include "IOManagerCurl.h" + +#include +#include + +namespace OpenVDS +{ +class TokenRefresher +{ +public: + TokenRefresher(const std::string& authTokenUrl, const std::string& clientId, const std::string& clientSecret, const std::string& refreshToken, CurlHandler& curlHandler, const std::function& newTokenCallback); + + std::string newToken(); + + std::string m_authTokenUrl; + std::string m_clientId; + std::string m_clientSecret; + std::string m_refreshToken; + CurlHandler& m_curlHandler; + std::function m_newTokenCallback; +}; +} + +#endif \ No newline at end of file diff --git a/src/OpenVDS/OpenVDS.cpp b/src/OpenVDS/OpenVDS.cpp index ee211b79..d69407e8 100644 --- a/src/OpenVDS/OpenVDS.cpp +++ b/src/OpenVDS/OpenVDS.cpp @@ -370,14 +370,22 @@ static std::unique_ptr createDMSOpenOptions(const StringWrapper& ur for (auto& connectionPair : connectionStringMap) { - if (connectionPair.first == "sdauthorityurl") + if (connectionPair.first == "sdauthorityurl" || connectionPair.first == "sd_authority_url") openOptions->sdAuthorityUrl = connectionPair.second; - if (connectionPair.first == "sdapikey") + if (connectionPair.first == "sdapikey" || connectionPair.first == "sd_api_key") openOptions->sdApiKey = connectionPair.second; - if (connectionPair.first == "sdtoken") + if (connectionPair.first == "sdtoken" || connectionPair.first == "sd_token") openOptions->sdToken = connectionPair.second; - if (connectionPair.first == "loglevel") + if (connectionPair.first == "loglevel" || connectionPair.first == "log_level") openOptions->logLevel = atoi(connectionPair.second.c_str()); + if (connectionPair.first == "authtokenurl" || connectionPair.first == "auth_token_url") + openOptions->authTokenUrl = connectionPair.second; + if (connectionPair.first == "refreshtoken" || connectionPair.first == "refresh_token") + openOptions->refreshToken = connectionPair.second; + if (connectionPair.first == "clientid" || connectionPair.first == "client_id") + openOptions->clientId = connectionPair.second; + if (connectionPair.first == "clientsecret" || connectionPair.first == "client_secret") + openOptions->clientSecret = connectionPair.second; } return openOptions; diff --git a/src/OpenVDS/OpenVDS/OpenVDS.h b/src/OpenVDS/OpenVDS/OpenVDS.h index f3ad63ce..b46d70be 100644 --- a/src/OpenVDS/OpenVDS/OpenVDS.h +++ b/src/OpenVDS/OpenVDS/OpenVDS.h @@ -454,14 +454,19 @@ struct GoogleOpenOptions : OpenOptions struct DMSOpenOptions : OpenOptions { - DMSOpenOptions() : OpenOptions(DMS) {} + DMSOpenOptions() : OpenOptions(DMS), logLevel(0) {} - DMSOpenOptions(std::string const& sdAuthorityUrl, std::string const& sdApiKey, std::string const &sdToken, std::string const &datasetPath, int logLevel) + DMSOpenOptions(std::string const& sdAuthorityUrl, std::string const& sdApiKey, std::string const &sdToken, std::string const &datasetPath, int logLevel, std::string const &authTokenUrl = std::string(), std::string const &refreshToken = std::string(), std::string const &clientId = std::string(), std::string const &clientSecret = std::string()) : OpenOptions(DMS) , sdAuthorityUrl(sdAuthorityUrl) , sdApiKey(sdApiKey) , sdToken(sdToken) + , datasetPath(datasetPath) , logLevel(logLevel) + , authTokenUrl(authTokenUrl) + , refreshToken(refreshToken) + , clientId(clientId) + , clientSecret(clientSecret) {} std::string sdAuthorityUrl; @@ -469,6 +474,10 @@ struct DMSOpenOptions : OpenOptions std::string sdToken; std::string datasetPath; int logLevel; + std::string authTokenUrl; + std::string refreshToken; + std::string clientId; + std::string clientSecret; }; /// -- GitLab From a577ba43cf1346e4e4ded60752e048852e290cc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Lind?= Date: Wed, 30 Jun 2021 10:56:08 +0200 Subject: [PATCH 3/5] Update s3 connection string key values --- docs/connection.rst | 5 ++++- src/OpenVDS/OpenVDS.cpp | 19 ++++++++++--------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/docs/connection.rst b/docs/connection.rst index 1b397bea..1abb8257 100644 --- a/docs/connection.rst +++ b/docs/connection.rst @@ -33,7 +33,7 @@ The recognised keys for ``s3`` connection string are - Region - EndpointOverride - AccessKeyId - - SecretKey + - SecretKey or SecretAccessKey - SessionToken - Expiration (Note: AWS SDK only accepts 'Z' timezone for ISO 8601 time) - LogFilenamePrefix @@ -42,6 +42,9 @@ The recognised keys for ``s3`` connection string are - RequestTimeoutMs - DisableInitApi (This will disable init/deinit of AWS sdk. Possible values: 1, On, True, Yes (case insensitive), all other values are false.) +Note: Key values for S3 can be specified in `CamelCase` or `snake_case` + + ``azure`` has two modes. The connection string mode or the bearer token mode. If the connection string contains a key with name BearerToken it will parse the string and extract following parameters: diff --git a/src/OpenVDS/OpenVDS.cpp b/src/OpenVDS/OpenVDS.cpp index d69407e8..809f53b3 100644 --- a/src/OpenVDS/OpenVDS.cpp +++ b/src/OpenVDS/OpenVDS.cpp @@ -124,19 +124,20 @@ static std::unique_ptr createS3OpenOptions(const StringWrapper &url { openOptions->region = connectionPair.second; } - else if (connectionPair.first == "endpointoverride") + else if (connectionPair.first == "endpointoverride" || connectionPair.first == "endpoint_override") { openOptions->endpointOverride = connectionPair.second; } - else if (connectionPair.first == "accesskeyid") + else if (connectionPair.first == "accesskeyid" || connectionPair.first == "access_key_id") { openOptions->accessKeyId = connectionPair.second; } - else if (connectionPair.first == "secretkey") + else if (connectionPair.first == "secretkey" || connectionPair.first == "secretaccesskey" + || connectionPair.first == "secret_key" || connectionPair.first == "secret_access_key") { openOptions->secretKey = connectionPair.second; } - else if (connectionPair.first == "sessiontoken") + else if (connectionPair.first == "sessiontoken" || connectionPair.first == "session_token") { openOptions->sessionToken = connectionPair.second; } @@ -144,15 +145,15 @@ static std::unique_ptr createS3OpenOptions(const StringWrapper &url { openOptions->expiration = connectionPair.second; } - else if (connectionPair.first == "logfilenameprefix") + else if (connectionPair.first == "logfilenameprefix" || connectionPair.first == "log_filename_prefix") { openOptions->logFilenamePrefix = connectionPair.second; } - else if (connectionPair.first == "loglevel") + else if (connectionPair.first == "loglevel" || connectionPair.first == "log_level") { openOptions->loglevel = connectionPair.second; } - else if (connectionPair.first == "connectiontimeoutms") + else if (connectionPair.first == "connectiontimeoutms" || connectionPair.first == "connection_timeout_ms") { openOptions->connectionTimeoutMs = strtol(&connectionPair.second[0], nullptr, 10); if (openOptions->connectionTimeoutMs == 0) @@ -162,7 +163,7 @@ static std::unique_ptr createS3OpenOptions(const StringWrapper &url return nullptr; } } - else if (connectionPair.first == "requesttimeoutms") + else if (connectionPair.first == "requesttimeoutms" || connectionPair.first == "request_timeout_ms") { openOptions->requestTimeoutMs = strtol(&connectionPair.second[0], nullptr, 10); if (openOptions->requestTimeoutMs == 0) @@ -172,7 +173,7 @@ static std::unique_ptr createS3OpenOptions(const StringWrapper &url return nullptr; } } - else if (connectionPair.first == "disableinitapi") + else if (connectionPair.first == "disableinitapi" || connectionPair.first == "disable_init_api") { auto value = connectionPair.second; std::transform(value.begin(), value.end(), value.begin(), asciitolower); -- GitLab From 25cc932af8cb6a90da970ea43f700168529ac06f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Lind?= Date: Wed, 30 Jun 2021 11:29:16 +0200 Subject: [PATCH 4/5] Update python APIs --- python/openvds/PyExceptions.cpp | 3 +- python/openvds/PyGlobal.cpp | 11 ++- python/openvds/PyVolumeDataAccessManager.cpp | 2 + python/openvds/generated_docstrings.h | 84 ++++++++++++++++++++ 4 files changed, 97 insertions(+), 3 deletions(-) diff --git a/python/openvds/PyExceptions.cpp b/python/openvds/PyExceptions.cpp index 33bf095d..8c3cbd71 100644 --- a/python/openvds/PyExceptions.cpp +++ b/python/openvds/PyExceptions.cpp @@ -68,7 +68,8 @@ py::register_exception(m, "InvalidArgument"); // IMPLEMENTED : // IndexOutOfRangeException // IMPLEMENTED : py::class_ // IMPLEMENTED : IndexOutOfRangeException_(m,"IndexOutOfRangeException", OPENVDS_DOCSTRING(IndexOutOfRangeException)); - +// IMPLEMENTED : py::class_ +// IMPLEMENTED : IndexOutOfRangeException_.def(py::init(), py::arg("errorMessage").none(false), OPENVDS_DOCSTRING(IndexOutOfRangeException_IndexOutOfRangeException)); // IMPLEMENTED : IndexOutOfRangeException_.def("getErrorMessage" , static_cast(&IndexOutOfRangeException::GetErrorMessage), py::call_guard(), OPENVDS_DOCSTRING(IndexOutOfRangeException_GetErrorMessage)); // IMPLEMENTED : IndexOutOfRangeException_.def_property_readonly("errorMessage", &IndexOutOfRangeException::GetErrorMessage, OPENVDS_DOCSTRING(IndexOutOfRangeException_GetErrorMessage)); py::register_exception(m, "IndexOutOfRangeException"); diff --git a/python/openvds/PyGlobal.cpp b/python/openvds/PyGlobal.cpp index d86de5c4..e1f964c3 100644 --- a/python/openvds/PyGlobal.cpp +++ b/python/openvds/PyGlobal.cpp @@ -67,7 +67,7 @@ PyGlobal::initModule(py::module& m) AWSOpenOptions_(m,"AWSOpenOptions", OPENVDS_DOCSTRING(AWSOpenOptions)); AWSOpenOptions_.def(py::init< >(), OPENVDS_DOCSTRING(AWSOpenOptions_AWSOpenOptions)); - AWSOpenOptions_.def(py::init(), py::arg("bucket").none(false), py::arg("key").none(false), py::arg("region").none(false), py::arg("endpointOverride").none(false), py::arg("connectionTimeoutMs") = 3000, py::arg("requestTimeoutMs") = 6000, OPENVDS_DOCSTRING(AWSOpenOptions_AWSOpenOptions_2)); + AWSOpenOptions_.def(py::init(), py::arg("bucket").none(false), py::arg("key").none(false), py::arg("region").none(false), py::arg("endpointOverride").none(false), py::arg("connectionTimeoutMs") = 3000, py::arg("requestTimeoutMs") = 6000, py::arg("disableInitApi") = false, OPENVDS_DOCSTRING(AWSOpenOptions_AWSOpenOptions_2)); AWSOpenOptions_.def_readwrite("bucket" , &AWSOpenOptions::bucket , OPENVDS_DOCSTRING(AWSOpenOptions_bucket)); AWSOpenOptions_.def_readwrite("key" , &AWSOpenOptions::key , OPENVDS_DOCSTRING(AWSOpenOptions_key)); AWSOpenOptions_.def_readwrite("region" , &AWSOpenOptions::region , OPENVDS_DOCSTRING(AWSOpenOptions_region)); @@ -80,6 +80,7 @@ PyGlobal::initModule(py::module& m) AWSOpenOptions_.def_readwrite("loglevel" , &AWSOpenOptions::loglevel , OPENVDS_DOCSTRING(AWSOpenOptions_loglevel)); AWSOpenOptions_.def_readwrite("connectionTimeoutMs" , &AWSOpenOptions::connectionTimeoutMs, OPENVDS_DOCSTRING(AWSOpenOptions_connectionTimeoutMs)); AWSOpenOptions_.def_readwrite("requestTimeoutMs" , &AWSOpenOptions::requestTimeoutMs, OPENVDS_DOCSTRING(AWSOpenOptions_requestTimeoutMs)); + AWSOpenOptions_.def_readwrite("disableInitApi" , &AWSOpenOptions::disableInitApi, OPENVDS_DOCSTRING(AWSOpenOptions_disableInitApi)); // AzureOpenOptions py::class_ @@ -188,12 +189,16 @@ PyGlobal::initModule(py::module& m) DMSOpenOptions_(m,"DMSOpenOptions", OPENVDS_DOCSTRING(DMSOpenOptions)); DMSOpenOptions_.def(py::init< >(), OPENVDS_DOCSTRING(DMSOpenOptions_DMSOpenOptions)); - DMSOpenOptions_.def(py::init(), py::arg("sdAuthorityUrl").none(false), py::arg("sdApiKey").none(false), py::arg("sdToken").none(false), py::arg("datasetPath").none(false), py::arg("logLevel").none(false), OPENVDS_DOCSTRING(DMSOpenOptions_DMSOpenOptions_2)); + DMSOpenOptions_.def(py::init(), py::arg("sdAuthorityUrl").none(false), py::arg("sdApiKey").none(false), py::arg("sdToken").none(false), py::arg("datasetPath").none(false), py::arg("logLevel").none(false), py::arg("authTokenUrl").none(false), py::arg("refreshToken").none(false), py::arg("clientId").none(false), py::arg("clientSecret").none(false), OPENVDS_DOCSTRING(DMSOpenOptions_DMSOpenOptions_2)); DMSOpenOptions_.def_readwrite("sdAuthorityUrl" , &DMSOpenOptions::sdAuthorityUrl, OPENVDS_DOCSTRING(DMSOpenOptions_sdAuthorityUrl)); DMSOpenOptions_.def_readwrite("sdApiKey" , &DMSOpenOptions::sdApiKey , OPENVDS_DOCSTRING(DMSOpenOptions_sdApiKey)); DMSOpenOptions_.def_readwrite("sdToken" , &DMSOpenOptions::sdToken , OPENVDS_DOCSTRING(DMSOpenOptions_sdToken)); DMSOpenOptions_.def_readwrite("datasetPath" , &DMSOpenOptions::datasetPath , OPENVDS_DOCSTRING(DMSOpenOptions_datasetPath)); DMSOpenOptions_.def_readwrite("logLevel" , &DMSOpenOptions::logLevel , OPENVDS_DOCSTRING(DMSOpenOptions_logLevel)); + DMSOpenOptions_.def_readwrite("authTokenUrl" , &DMSOpenOptions::authTokenUrl , OPENVDS_DOCSTRING(DMSOpenOptions_authTokenUrl)); + DMSOpenOptions_.def_readwrite("refreshToken" , &DMSOpenOptions::refreshToken , OPENVDS_DOCSTRING(DMSOpenOptions_refreshToken)); + DMSOpenOptions_.def_readwrite("clientId" , &DMSOpenOptions::clientId , OPENVDS_DOCSTRING(DMSOpenOptions_clientId)); + DMSOpenOptions_.def_readwrite("clientSecret" , &DMSOpenOptions::clientSecret , OPENVDS_DOCSTRING(DMSOpenOptions_clientSecret)); // HttpOpenOptions py::class_ @@ -271,6 +276,8 @@ PyGlobal::initModule(py::module& m) m.def("getLayout" , static_cast(&GetLayout), py::arg("handle").none(false), py::call_guard(), OPENVDS_DOCSTRING(GetLayout)); m.def("getAccessManagerInterface" , static_cast(&GetAccessManagerInterface), py::arg("handle").none(false), py::call_guard(), OPENVDS_DOCSTRING(GetAccessManagerInterface)); m.def("getAccessManager" , static_cast(&GetAccessManager), py::arg("handle").none(false), py::call_guard(), OPENVDS_DOCSTRING(GetAccessManager)); + m.def("getCompressionMethod" , static_cast(&GetCompressionMethod), py::arg("handle").none(false), py::call_guard(), OPENVDS_DOCSTRING(GetCompressionMethod)); + m.def("getCompressionTolerance" , static_cast(&GetCompressionTolerance), py::arg("handle").none(false), py::call_guard(), OPENVDS_DOCSTRING(GetCompressionTolerance)); m.def("close" , static_cast(&Close), py::arg("handle").none(false), py::call_guard(), OPENVDS_DOCSTRING(Close)); m.def("getGlobalState" , static_cast(&GetGlobalState), py::call_guard(), OPENVDS_DOCSTRING(GetGlobalState)); //AUTOGEN-END diff --git a/python/openvds/PyVolumeDataAccessManager.cpp b/python/openvds/PyVolumeDataAccessManager.cpp index 60e1fb6a..4400963e 100644 --- a/python/openvds/PyVolumeDataAccessManager.cpp +++ b/python/openvds/PyVolumeDataAccessManager.cpp @@ -33,6 +33,7 @@ PyVolumeDataAccessManager::initModule(py::module& m) IVolumeDataAccessManager_.def("getVolumeDataLayout" , static_cast(&IVolumeDataAccessManager::GetVolumeDataLayout), py::call_guard(), OPENVDS_DOCSTRING(IVolumeDataAccessManager_GetVolumeDataLayout)); IVolumeDataAccessManager_.def_property_readonly("volumeDataLayout", &IVolumeDataAccessManager::GetVolumeDataLayout, OPENVDS_DOCSTRING(IVolumeDataAccessManager_GetVolumeDataLayout)); IVolumeDataAccessManager_.def("getVDSProduceStatus" , static_cast(&IVolumeDataAccessManager::GetVDSProduceStatus), py::arg("dimensionsND").none(false), py::arg("LOD").none(false), py::arg("channel").none(false), py::call_guard(), OPENVDS_DOCSTRING(IVolumeDataAccessManager_GetVDSProduceStatus)); + IVolumeDataAccessManager_.def("getVDSChunkCount" , static_cast(&IVolumeDataAccessManager::GetVDSChunkCount), py::arg("dimensionsND").none(false), py::arg("LOD").none(false), py::arg("channel").none(false), py::call_guard(), OPENVDS_DOCSTRING(IVolumeDataAccessManager_GetVDSChunkCount)); IVolumeDataAccessManager_.def("createVolumeDataPageAccessor", static_cast(&IVolumeDataAccessManager::CreateVolumeDataPageAccessor), py::arg("dimensionsND").none(false), py::arg("LOD").none(false), py::arg("channel").none(false), py::arg("maxPages").none(false), py::arg("accessMode").none(false), py::arg("chunkMetadataPageSize") = 1024, py::call_guard(), OPENVDS_DOCSTRING(IVolumeDataAccessManager_CreateVolumeDataPageAccessor)); IVolumeDataAccessManager_.def("destroyVolumeDataPageAccessor", static_cast(&IVolumeDataAccessManager::DestroyVolumeDataPageAccessor), py::arg("volumeDataPageAccessor").none(false), py::call_guard(), OPENVDS_DOCSTRING(IVolumeDataAccessManager_DestroyVolumeDataPageAccessor)); IVolumeDataAccessManager_.def("destroyVolumeDataAccessor" , static_cast(&IVolumeDataAccessManager::DestroyVolumeDataAccessor), py::arg("accessor").none(false), py::call_guard(), OPENVDS_DOCSTRING(IVolumeDataAccessManager_DestroyVolumeDataAccessor)); @@ -153,6 +154,7 @@ PyVolumeDataAccessManager::initModule(py::module& m) VolumeDataAccessManager_.def("getVolumeDataLayout" , static_cast(&VolumeDataAccessManager::GetVolumeDataLayout), py::call_guard(), OPENVDS_DOCSTRING(VolumeDataAccessManager_GetVolumeDataLayout)); VolumeDataAccessManager_.def_property_readonly("volumeDataLayout", &VolumeDataAccessManager::GetVolumeDataLayout, OPENVDS_DOCSTRING(VolumeDataAccessManager_GetVolumeDataLayout)); VolumeDataAccessManager_.def("getVDSProduceStatus" , static_cast(&VolumeDataAccessManager::GetVDSProduceStatus), py::arg("dimensionsND").none(false), py::arg("LOD").none(false), py::arg("channel").none(false), py::call_guard(), OPENVDS_DOCSTRING(VolumeDataAccessManager_GetVDSProduceStatus)); + VolumeDataAccessManager_.def("getVDSChunkCount" , static_cast(&VolumeDataAccessManager::GetVDSChunkCount), py::arg("dimensionsND").none(false), py::arg("LOD").none(false), py::arg("channel").none(false), py::call_guard(), OPENVDS_DOCSTRING(VolumeDataAccessManager_GetVDSChunkCount)); VolumeDataAccessManager_.def("createVolumeDataPageAccessor", static_cast(&VolumeDataAccessManager::CreateVolumeDataPageAccessor), py::arg("dimensionsND").none(false), py::arg("LOD").none(false), py::arg("channel").none(false), py::arg("maxPages").none(false), py::arg("accessMode").none(false), py::arg("chunkMetadataPageSize") = 1024, py::call_guard(), OPENVDS_DOCSTRING(VolumeDataAccessManager_CreateVolumeDataPageAccessor)); VolumeDataAccessManager_.def("destroyVolumeDataPageAccessor", static_cast(&VolumeDataAccessManager::DestroyVolumeDataPageAccessor), py::arg("volumeDataPageAccessor").none(false), py::call_guard(), OPENVDS_DOCSTRING(VolumeDataAccessManager_DestroyVolumeDataPageAccessor)); VolumeDataAccessManager_.def("createVolumeData2DInterpolatingAccessorR64", static_cast)>(&VolumeDataAccessManager::CreateVolumeData2DInterpolatingAccessorR64), py::arg("dimensionsND").none(false), py::arg("LOD").none(false), py::arg("channel").none(false), py::arg("interpolationMethod").none(false), py::arg("maxPages").none(false), py::arg("replacementNoValue") = nullptr, py::call_guard(), OPENVDS_DOCSTRING(VolumeDataAccessManager_CreateVolumeData2DInterpolatingAccessorR64)); diff --git a/python/openvds/generated_docstrings.h b/python/openvds/generated_docstrings.h index f52ae635..b686d40e 100644 --- a/python/openvds/generated_docstrings.h +++ b/python/openvds/generated_docstrings.h @@ -58,6 +58,8 @@ static const char *__doc_OpenVDS_AWSOpenOptions_bucket = R"doc()doc"; static const char *__doc_OpenVDS_AWSOpenOptions_connectionTimeoutMs = R"doc()doc"; +static const char *__doc_OpenVDS_AWSOpenOptions_disableInitApi = R"doc()doc"; + static const char *__doc_OpenVDS_AWSOpenOptions_endpointOverride = R"doc()doc"; static const char *__doc_OpenVDS_AWSOpenOptions_expiration = R"doc()doc"; @@ -674,10 +676,18 @@ static const char *__doc_OpenVDS_DMSOpenOptions_DMSOpenOptions = R"doc()doc"; static const char *__doc_OpenVDS_DMSOpenOptions_DMSOpenOptions_2 = R"doc()doc"; +static const char *__doc_OpenVDS_DMSOpenOptions_authTokenUrl = R"doc()doc"; + +static const char *__doc_OpenVDS_DMSOpenOptions_clientId = R"doc()doc"; + +static const char *__doc_OpenVDS_DMSOpenOptions_clientSecret = R"doc()doc"; + static const char *__doc_OpenVDS_DMSOpenOptions_datasetPath = R"doc()doc"; static const char *__doc_OpenVDS_DMSOpenOptions_logLevel = R"doc()doc"; +static const char *__doc_OpenVDS_DMSOpenOptions_refreshToken = R"doc()doc"; + static const char *__doc_OpenVDS_DMSOpenOptions_sdApiKey = R"doc()doc"; static const char *__doc_OpenVDS_DMSOpenOptions_sdAuthorityUrl = R"doc()doc"; @@ -806,6 +816,32 @@ Returns: -------- The VolumeDataAccessManagerInterface of the VDS)doc"; +static const char *__doc_OpenVDS_GetCompressionMethod = +R"doc(Get the primary CompressionMethod for a VDS + +Parameters: +----------- + +handle : + The handle of the VDS + +Returns: +-------- + The CompressionMethod used for the VDS)doc"; + +static const char *__doc_OpenVDS_GetCompressionTolerance = +R"doc(Get the primary compression tolerance used for a VDS + +Parameters: +----------- + +handle : + The handle of the VDS + +Returns: +-------- + The compression tolerance used for the VDS)doc"; + static const char *__doc_OpenVDS_GetGlobalState = R"doc(Get the GlobalState interface @@ -1330,6 +1366,27 @@ Returns: -------- The buffer size needed.)doc"; +static const char *__doc_OpenVDS_IVolumeDataAccessManager_GetVDSChunkCount = +R"doc(Get the chunk count for the specific DimensionsND/LOD/Channel +combination. + +Parameters: +----------- + +dimensionsND : + The dimensions group we're getting the chunk count for. + +LOD : + The LOD level we're getting the chunk count status for. + +channel : + The channel index we're getting the chunk count status for. + +Returns: +-------- + The chunk count for the specific DimensionsND/LOD/Channel + combination.)doc"; + static const char *__doc_OpenVDS_IVolumeDataAccessManager_GetVDSProduceStatus = R"doc(Get the produce status for the specific DimensionsND/LOD/Channel combination. @@ -1761,6 +1818,10 @@ static const char *__doc_OpenVDS_IndexOutOfRangeException = R"doc()doc"; static const char *__doc_OpenVDS_IndexOutOfRangeException_GetErrorMessage = R"doc()doc"; +static const char *__doc_OpenVDS_IndexOutOfRangeException_IndexOutOfRangeException = R"doc()doc"; + +static const char *__doc_OpenVDS_IndexOutOfRangeException_m_errorMessage = R"doc()doc"; + static const char *__doc_OpenVDS_IndexRegion = R"doc()doc"; static const char *__doc_OpenVDS_IndexRegion_IndexRegion = R"doc()doc"; @@ -2523,6 +2584,8 @@ static const char *__doc_OpenVDS_OpenOptions_ConnectionType_Azure = R"doc()doc"; static const char *__doc_OpenVDS_OpenOptions_ConnectionType_AzurePresigned = R"doc()doc"; +static const char *__doc_OpenVDS_OpenOptions_ConnectionType_AzureSdkForCpp = R"doc()doc"; + static const char *__doc_OpenVDS_OpenOptions_ConnectionType_ConnectionTypeCount = R"doc()doc"; static const char *__doc_OpenVDS_OpenOptions_ConnectionType_DMS = R"doc()doc"; @@ -3159,6 +3222,27 @@ Returns: static const char *__doc_OpenVDS_VolumeDataAccessManager_GetProjectedVolumeSubsetBufferSize_2 = R"doc()doc"; +static const char *__doc_OpenVDS_VolumeDataAccessManager_GetVDSChunkCount = +R"doc(Get the chunk count for the specific DimensionsND/LOD/Channel +combination. + +Parameters: +----------- + +dimensionsND : + The dimensions group we're getting the chunk count for. + +LOD : + The LOD level we're getting the chunk count status for. + +channel : + The channel index we're getting the chunk count status for. + +Returns: +-------- + The chunk count for the specific DimensionsND/LOD/Channel + combination.)doc"; + static const char *__doc_OpenVDS_VolumeDataAccessManager_GetVDSProduceStatus = R"doc(Get the produce status for the specific DimensionsND/LOD/Channel combination. -- GitLab From 7f8e64220f5a83fe045334f55cafd3db753c7b3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Lind?= Date: Wed, 30 Jun 2021 14:06:22 +0200 Subject: [PATCH 5/5] Mark 2.1.3 release --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c50c5f18..6834c3a7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,7 +5,7 @@ if(CCACHE_PROGRAM) set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE "${CCACHE_PROGRAM}") endif() -set(OpenVDSVersion 2.1.2) +set(OpenVDSVersion 2.1.3) project(OpenVDS VERSION ${OpenVDSVersion}) -- GitLab