From 89d1bfa4578f75a66a8aa2d9fc358023cf9a2ee0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Lind?= Date: Tue, 1 Jun 2021 10:33:46 +0200 Subject: [PATCH] AzureSdkForCpp --- 3rdparty/BuildDms/CMakeLists.txt | 7 +- CMake/Build3rdParty.cmake | 34 ++- CMake/BuildAzureSdkForCpp.cmake | 38 +++ CMake/BuildLibXML2.cmake | 21 ++ CMake/Fetch3rdPartyInBuild.cmake | 2 + CMakeLists.txt | 17 +- python/openvds/PyGlobal.cpp | 1 + src/OpenVDS/CMakeLists.txt | 12 + src/OpenVDS/IO/IOManager.cpp | 18 +- src/OpenVDS/IO/IOManagerAzureSdkForCpp.cpp | 321 +++++++++++++++++++++ src/OpenVDS/IO/IOManagerAzureSdkForCpp.h | 55 ++++ src/OpenVDS/OpenVDS/OpenVDS.h | 1 + 12 files changed, 516 insertions(+), 11 deletions(-) create mode 100644 CMake/BuildAzureSdkForCpp.cmake create mode 100644 CMake/BuildLibXML2.cmake create mode 100644 src/OpenVDS/IO/IOManagerAzureSdkForCpp.cpp create mode 100644 src/OpenVDS/IO/IOManagerAzureSdkForCpp.h diff --git a/3rdparty/BuildDms/CMakeLists.txt b/3rdparty/BuildDms/CMakeLists.txt index d74efe1d..947d6db1 100644 --- a/3rdparty/BuildDms/CMakeLists.txt +++ b/3rdparty/BuildDms/CMakeLists.txt @@ -32,6 +32,12 @@ if (NOT DISABLE_AZURE_IOMANAGER) set(SOURCES ${SOURCES} ${SRC_LIB_PROVIDERS_AZURE}) endif() +if (NOT DISABLE_AZURESDKFORCPP_IOMANAGER) + add_definitions( -DHAS_AZURE_BLOB_STORAGE_PROVIDER_CURL ) + file(GLOB SRC_LIB_PROVIDERS_AZURE_NEW ${dmsCMakeDir}/src/lib/cloud/providers/azure/curl/*.cc) + set(SOURCES ${SOURCES} ${SRC_LIB_PROVIDERS_AZURE_NEW}) +endif() + if (NOT DISABLE_AWS_IOMANAGER) add_definitions( -DHAS_AWS_S3_STORAGE_PROVIDER ) file(GLOB SRC_LIB_PROVIDERS_AWS ${dmsCMakeDir}/src/lib/cloud/providers/aws/*.cc) @@ -83,7 +89,6 @@ get_property(runtime_3rdparty_debug GLOBAL PROPERTY OPENVDS_RUNTIME_LIBS_DEBUG get_property(include_3rdparty GLOBAL PROPERTY OPENVDS_INCLUDE_LIBRARIES) get_property(dependency_targets_3rdparty GLOBAL PROPERTY OPENVDS_DEPENDENCY_TARGETS) - addSystemHeadersToTarget(sdapi_objects "${include_3rdparty}") if (UNIX) diff --git a/CMake/Build3rdParty.cmake b/CMake/Build3rdParty.cmake index 31c6b129..3fee8a83 100644 --- a/CMake/Build3rdParty.cmake +++ b/CMake/Build3rdParty.cmake @@ -13,33 +13,51 @@ include(CMake/BuildDms.cmake) if (BUILD_ZLIB) include(CMake/BuildZlib.cmake) endif() +if (BUILD_LIBXML) +include(CMake/BuildLibXML2.cmake) +endif() +include(CMake/BuildAzureSdkForCpp.cmake) macro(build3rdparty) - if (NOT DISABLE_AWS_IOMANAGER) - BuildAwsSdk() - endif() BuildJsonCPP() + BuildFmt() + if (BUILD_ZLIB) BuildZlib() endif() - BuildFmt() - if (NOT DISABLE_AZURE_IOMANAGER) - BuildCppRestSdk() - BuildAzure() - endif() + if (Python3_FOUND) include(CMake/FindPythonLibsNew.cmake) add_subdirectory(${pybind11_SOURCE_DIR} ${PROJECT_BINARY_DIR}/pybind11_${pybind11_VERSION} EXCLUDE_FROM_ALL) endif() + + if (NOT DISABLE_AWS_IOMANAGER) + BuildAwsSdk() + endif() + + if (NOT DISABLE_AZURE_IOMANAGER) + BuildCppRestSdk() + BuildAzure() + endif() + if (NOT DISABLE_CURL_IOMANAGER OR NOT DISABLE_GCP_IOMANAGER OR NOT DISABLE_DMS_IOMANAGER) BuildCurl() BuildLibUV() endif() + if (NOT DISABLE_GCP_IOMANAGER OR NOT DISABLE_DMS_IOMANAGER) BuildLibreSSL() BuildCrc32c() BuildGoogleCloud() endif() + + if (NOT DISABLE_AZURESDKFORCPP_IOMANAGER) + if (BUILD_LIBXML) + BuildLibXML2() + endif() + BuildAzureSdkForCpp() + endif() + if (NOT DISABLE_DMS_IOMANAGER) BuildDms() endif() diff --git a/CMake/BuildAzureSdkForCpp.cmake b/CMake/BuildAzureSdkForCpp.cmake new file mode 100644 index 00000000..e68c6048 --- /dev/null +++ b/CMake/BuildAzureSdkForCpp.cmake @@ -0,0 +1,38 @@ +function(BuildAzureSdkForCpp) + if (WIN32) + LIST(APPEND AZURE_SDK_FOR_CPP_LIBS_LIST "lib/azure-core.lib") + LIST(APPEND AZURE_SDK_FOR_CPP_LIBS_LIST "lib/azure-identity.lib") + LIST(APPEND AZURE_SDK_FOR_CPP_LIBS_LIST "lib/azure-security-keyvault-common.lib") + LIST(APPEND AZURE_SDK_FOR_CPP_LIBS_LIST "lib/azure-security-keyvault-keys.lib") + LIST(APPEND AZURE_SDK_FOR_CPP_LIBS_LIST "lib/azure-storage-blobs.lib") + LIST(APPEND AZURE_SDK_FOR_CPP_LIBS_LIST "lib/azure-storage-common.lib") + LIST(APPEND AZURE_SDK_FOR_CPP_LIBS_LIST "lib/azure-storage-files-datalake.lib") + LIST(APPEND AZURE_SDK_FOR_CPP_LIBS_LIST "lib/azure-storage-files-shares.lib") + LIST(APPEND AZURE_SDK_FOR_CPP_LIBS_LIST "lib/azure-template.lib") + + LIST(APPEND AZURE_SDK_FOR_CPP_DLLS_LIST "bin/azure-core.dll") + LIST(APPEND AZURE_SDK_FOR_CPP_DLLS_LIST "bin/azure-identity.dll") + LIST(APPEND AZURE_SDK_FOR_CPP_DLLS_LIST "bin/azure-security-keyvault-common.dll") + LIST(APPEND AZURE_SDK_FOR_CPP_DLLS_LIST "bin/azure-security-keyvault-keys.dll") + LIST(APPEND AZURE_SDK_FOR_CPP_DLLS_LIST "bin/azure-storage-blobs.dll") + LIST(APPEND AZURE_SDK_FOR_CPP_DLLS_LIST "bin/azure-storage-common.dll") + LIST(APPEND AZURE_SDK_FOR_CPP_DLLS_LIST "bin/azure-storage-files-datalake.dll") + LIST(APPEND AZURE_SDK_FOR_CPP_DLLS_LIST "bin/azure-storage-files-shares.dll") + LIST(APPEND AZURE_SDK_FOR_CPP_DLLS_LIST "bin/azure-template.dll") + elseif (APPLE) + else() + LIST(APPEND AZURE_SDK_FOR_CPP_LIBS_LIST "lib${LIBSUFFIX}/libazure-core.so") + LIST(APPEND AZURE_SDK_FOR_CPP_LIBS_LIST "lib${LIBSUFFIX}/libazure-storage-common.so") + LIST(APPEND AZURE_SDK_FOR_CPP_LIBS_LIST "lib${LIBSUFFIX}/libazure-template.so") + LIST(APPEND AZURE_SDK_FOR_CPP_LIBS_LIST "lib${LIBSUFFIX}/libazure-identity.so") + LIST(APPEND AZURE_SDK_FOR_CPP_LIBS_LIST "lib${LIBSUFFIX}/libazure-security-keyvault-common.so") + LIST(APPEND AZURE_SDK_FOR_CPP_LIBS_LIST "lib${LIBSUFFIX}/libazure-security-keyvault-keys.so") + LIST(APPEND AZURE_SDK_FOR_CPP_LIBS_LIST "lib${LIBSUFFIX}/libazure-storage-blobs.so") + LIST(APPEND AZURE_SDK_FOR_CPP_LIBS_LIST "lib${LIBSUFFIX}/libazure-storage-files-datalake.so") + LIST(APPEND AZURE_SDK_FOR_CPP_LIBS_LIST "lib${LIBSUFFIX}/libazure-storage-files-shares.so") + endif() + if (BUILD_LIBXML) + set(DEPENDENCIES libxml2) + endif() + BuildExternal(azure-sdk-for-cpp ${azure-sdk-for-cpp_VERSION} "${DEPENDENCIES}" ${azure-sdk-for-cpp_SOURCE_DIR} "${AZURE_SDK_FOR_CPP_LIBS_LIST}" "${AZURE_SDK_FOR_CPP_DLLS_LIST}" "" "" "${cmake_arg};-DLibXml2_DIR=${libxml2_root};-DBUILD_SHARED_LIBS=ON;-DCMAKE_FIND_PACKAGE_PREFER_CONFIG=ON") +endfunction() diff --git a/CMake/BuildLibXML2.cmake b/CMake/BuildLibXML2.cmake new file mode 100644 index 00000000..33491f9b --- /dev/null +++ b/CMake/BuildLibXML2.cmake @@ -0,0 +1,21 @@ +function(BuildLibXML2) + if (WIN32) + list(APPEND LIBXML2_LIBS_LIST_DEBUG "lib/libxml2d.lib") + list(APPEND LIBXML2_LIBS_LIST_RELEASE "lib/libxml2.lib") + list(APPEND LIBXML2_DLLS_LIST_DEBUG "bin/libxml2d.dll") + list(APPEND LIBXML2_DLLS_LIST_RELEASE "bin/libxml2.dll") + elseif (APPLE) + else() + list(APPEND LIBXML2_DLLS_LIST_DEBUG "lib/libxml2d.so") + list(APPEND LIBXML2_DLLS_LIST_RELEASE "lib/libxml2.so") + endif() + + list(APPEND LIBXML2_CMAKE_ARGS "-DLIBXML2_WITH_ICONV=OFF") + list(APPEND LIBXML2_CMAKE_ARGS "-DLIBXML2_WITH_LZMA=OFF") + list(APPEND LIBXML2_CMAKE_ARGS "-DLIBXML2_WITH_PYTHON=OFF") + list(APPEND LIBXML2_CMAKE_ARGS "-DLIBXML2_WITH_TESTS=OFF") + list(APPEND LIBXML2_CMAKE_ARGS "-DLIBXML2_WITH_ZLIB=OFF") + BuildExternal(libxml2 ${libxml2_VERSION} "" ${libxml2_SOURCE_DIR} "${LIBXML2_LIBS_LIST_RELEASE}" "${LIBXML2_DLLS_LIST_RELEASE}" "${LIBXML2_LIBS_LIST_DEBUG}" "${LIBXML2_DLLS_LIST_DEBUG}" "${LIBXML2_CMAKE_ARGS}") + set(libxml2_root ${libxml2_INSTALL_INT_CONFIG}/lib/cmake/libxml2-2.9.10 PARENT_SCOPE) + set(libxml2_INSTALL_INT_CONFIG ${libxml2_INSTALL_INT_CONFIG} PARENT_SCOPE) +endfunction() diff --git a/CMake/Fetch3rdPartyInBuild.cmake b/CMake/Fetch3rdPartyInBuild.cmake index 4715772d..0f31f665 100644 --- a/CMake/Fetch3rdPartyInBuild.cmake +++ b/CMake/Fetch3rdPartyInBuild.cmake @@ -82,6 +82,8 @@ function(Fetch3rdParty) Fetch3rdParty_Package(absl 20200225.2 https://codeload.github.com/abseil/abseil-cpp/tar.gz/20200225.2 SHA256=f41868f7a938605c92936230081175d1eae87f6ea2c248f41077c8f88316f111) Fetch3rdParty_Package(crc32c 1.1.1 https://codeload.github.com/google/crc32c/tar.gz/1.1.1 SHA256=a6533f45b1670b5d59b38a514d82b09c6fb70cc1050467220216335e873074e8) Fetch3rdParty_Package(google-cloud-cpp 1.14.0 https://codeload.github.com/googleapis/google-cloud-cpp/tar.gz/v1.14.0 SHA256=839b2d4dcb36a671734dac6b30ea8c298bbeaafcf7a45ee4a7d7aa5986b16569) + Fetch3rdParty_Package(libxml2 2.9.12a https://codeload.github.com/GNOME/libxml2/tar.gz/v2.9.12 SHA256=8a4ddd706419c210b30b8978a51388937fd9362c34fc9a3d69e4fcc6f8055ee0) + Fetch3rdParty_Package(azure-sdk-for-cpp 12.0.0b11 https://codeload.github.com/Azure/azure-sdk-for-cpp/tar.gz/azure-storage-blobs_12.0.0-beta.11 SHA256=b111636335340e3e7a5675351216dde606b8345b9906ed2f42ff8a794f5f2375) Fetch3rdParty_Package(dms 18a85b75c7 https://community.opengroup.org/osdu/platform/domain-data-mgmt-services/seismic/seismic-dms-suite/seismic-store-cpp-lib/-/archive/18a85b75c7c10ce1bb8788d23c79964e060cee64/seismic-store-cpp-lib-master.tar.gz SHA256=146ed18ee82277355273588b6c4c834605a86a1ec40d48c5c9c11cbd47ec0165) Fetch3rdParty_File(testng java 6.14.3 jar https://repo1.maven.org/maven2/org/testng/testng/6.14.3/testng-6.14.3.jar MD5=9f17a8f9e99165e148c42b21f4b63d7c) Fetch3rdParty_File(jcommander java 1.72 jar https://repo1.maven.org/maven2/com/beust/jcommander/1.72/jcommander-1.72.jar MD5=9fde6bc0ba1032eceb7267fd1ad1657b) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8e4dd5ea..86d92dce 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -74,11 +74,20 @@ if (WIN32) set(DEFAULT_BUILD_CURL ON) set(DEFAULT_BUILD_UV ON) set(DEFAULT_USE_LIBRESSL ON) + set(DEFAULT_BUILD_LIBXML ON) + set(DEFAULT_DISABLE_AZURESDKFORCPP OFF) else() set(DEFAULT_BUILD_ZLIB OFF) set(DEFAULT_BUILD_CURL OFF) set(DEFAULT_BUILD_UV OFF) set(DEFAULT_USE_LIBRESSL OFF) + set(DEFAULT_BUILD_LIBXML OFF) + find_package(CURL REQUIRED) + if (CURL_VERSION_STRING VERSION_LESS_EQUAL "7.44") + set(DEFAULT_DISABLE_AZURESDKFORCPP ON) + else() + set(DEFAULT_DISABLE_AZURESDKFORCPP OFF) + endif() endif() option(BUILD_PYTHON "Build python bindings" ${BUILD_PYTHON_DEFAULT}) @@ -92,12 +101,14 @@ option(ENABLE_RUNPATH_ORIGIN "On Linux make the install have ORIGIN as RUNPATH" option(DISABLE_STRICT_WARNINGS "Disable strict warnings settings. This includes making warnings to errors." OFF) option(DISABLE_AWS_IOMANAGER "Disable compiling the Aws-cpp-sdk IOManager backend." OFF) option(DISABLE_AZURE_IOMANAGER "Disable compiling Azure-storage-cpp and cpprestapi. This also disables Azure support for DMS." OFF) +option(DISABLE_AZURESDKFORCPP_IOMANAGER "Disable compiling AzureSdkForCpp. This also disables Azure support for DMS." ${DEFAULT_DISABLE_AZURESDKFORCPP}) option(DISABLE_AZURE_PRESIGNED_IOMANAGER "Disable compiling Azure presigned IOManager backend." OFF) option(DISABLE_GCP_IOMANAGER "Disable compiling GCP IOManager backend." OFF) option(DISABLE_DMS_IOMANAGER "Disable compiling DMS IOManager backend." OFF) option(BUILD_ZLIB "Build zlib" ${DEFAULT_BUILD_ZLIB}) option(BUILD_CURL "Build libcurl as part of the openVDS build" ${DEFAULT_BUILD_CURL}) option(BUILD_UV "Build libuv as part of the openVDS build" ${DEFAULT_BUILD_UV}) +option(BUILD_LIBXML "Build libxml2 as part of the openVDS build" ${DEFAULT_BUILD_LIBXML}) option(USE_LIBRESSL "Build libressl as part of the openVDS build and use it instead of OpenSSL" ${DEFAULT_USE_LIBRESSL}) include(CMakeDependentOption) @@ -111,8 +122,12 @@ if ((${CMAKE_MAJOR_VERSION} EQUAL 3) AND (${CMAKE_MINOR_VERSION} EQUAL 19) AND ( message(FATAL_ERROR "A bug in CMake 3.19 series makes it is makes it not possible to build the AWS backend code.\n Please upgrade og downgrade. Its possible to use custom version of cmake in Visual Studio under CMake Settings.\n Disable AWS IO Manager by defining -DDISABLE_AWS_IOMANAGER=ON as a cmake parameter") endif() +if (DISABLE_AZURESDKFORCPP_IOMANAGER) + set(CMAKE_CXX_STANDARD 11) +else() + set(CMAKE_CXX_STANDARD 14) +endif() -set(CMAKE_CXX_STANDARD 11) set(CMAKE_CXX_EXTENSIONS OFF) set(CMAKE_CXX_VISIBILITY_PRESET hidden) set(CMAKE_POSITION_INDEPENDENT_CODE ON) diff --git a/python/openvds/PyGlobal.cpp b/python/openvds/PyGlobal.cpp index d86de5c4..fee2d3cd 100644 --- a/python/openvds/PyGlobal.cpp +++ b/python/openvds/PyGlobal.cpp @@ -284,6 +284,7 @@ PyGlobal::initModule(py::module& m) { case OpenOptions::ConnectionType::AWS : conn = std::string("AWS" ); break; case OpenOptions::ConnectionType::Azure : conn = std::string("Azure" ); break; + case OpenOptions::ConnectionType::AzureSdkForCpp : conn = std::string("AzureSdkForCpp" ); break; case OpenOptions::ConnectionType::AzurePresigned : conn = std::string("AzurePresigned" ); break; case OpenOptions::ConnectionType::GoogleStorage : conn = std::string("GoogleStorage" ); break; case OpenOptions::ConnectionType::Http : conn = std::string("Http" ); break; diff --git a/src/OpenVDS/CMakeLists.txt b/src/OpenVDS/CMakeLists.txt index 9eada19f..05695ce4 100644 --- a/src/OpenVDS/CMakeLists.txt +++ b/src/OpenVDS/CMakeLists.txt @@ -6,6 +6,7 @@ set(SOURCE_FILES IO/IOManager.cpp IO/IOManagerAWS.cpp IO/IOManagerAzure.cpp + IO/IOManagerAzureSdkForCpp.cpp IO/IOManagerInMemory.cpp IO/IOManagerCurl.cpp IO/IOManagerAzurePresigned.cpp @@ -49,6 +50,7 @@ set (PRIVATE_HEADER_FILES IO/IOManager.h IO/IOManagerAWS.h IO/IOManagerAzure.h + IO/IOManagerAzureSdkForCpp.h IO/IOManagerInMemory.h IO/IOManagerCurl.h IO/IOManagerAzurePresigned.h @@ -184,6 +186,16 @@ else() endif() endif() +if (DISABLE_AZURESDKFORCPP_IOMANAGER) + target_compile_definitions(openvds_objects PRIVATE OPENVDS_NO_AZURE_SDK_FOR_CPP_IOMANAGER) + set_source_files_properties(IO/IOManagerAzureSdkForCpp.cpp PROPERTIES HEADER_FILE_ONLY TRUE) +else() + if (UNIX) + find_package(CURL REQUIRED) + find_package(LibXml2 REQUIRED) + target_link_libraries(openvds_objects PRIVATE CURL::libcurl LibXml2::LibXml2) + endif() +endif() if (DISABLE_CURL_IOMANAGER) set_source_files_properties(IO/IOManagerCurl.cpp PROPERTIES HEADER_FILE_ONLY TRUE) set_source_files_properties(IO/IOManagerHttp.cpp PROPERTIES HEADER_FILE_ONLY TRUE) diff --git a/src/OpenVDS/IO/IOManager.cpp b/src/OpenVDS/IO/IOManager.cpp index 659f2374..4e412465 100644 --- a/src/OpenVDS/IO/IOManager.cpp +++ b/src/OpenVDS/IO/IOManager.cpp @@ -27,6 +27,9 @@ #ifndef OPENVDS_NO_AZURE_IOMANAGER #include "IOManagerAzure.h" #endif +#ifndef OPENVDS_NO_AZURE_SDK_FOR_CPP_IOMANAGER +#include "IOManagerAzureSdkForCpp.h" +#endif #ifndef OPENVDS_NO_GCP_IOMANAGER #include "IOManagerGoogle.h" #endif @@ -39,6 +42,8 @@ #include "IOManagerDms.h" +#include "VDS/Env.h" + namespace OpenVDS { @@ -50,10 +55,21 @@ IOManager* IOManager::CreateIOManager(const OpenOptions& options, IOManager::Acc case OpenOptions::AWS: return new IOManagerAWS(static_cast(options), error); #endif -#ifndef OPENVDS_NO_AZURE_IOMANAGER case OpenOptions::Azure: + { +#ifndef OPENVDS_NO_AZURE_SDK_FOR_CPP_IOMANAGER + bool useAzureSdkForCpp = getBooleanEnvironmentVariable("OPENVDS_AZURESDKFORCPP"); + if (useAzureSdkForCpp) + return new IOManagerAzureSdkForCpp(static_cast(options), error); +#endif +#ifndef OPENVDS_NO_AZURE_IOMANAGER + if (options.connectionType == OpenOptions::Azure) return new IOManagerAzure(static_cast(options), error); #endif + error.code = -1; + error.string = "Unknown type for OpenOptions"; + return nullptr; + } #ifndef OPENVDS_NO_AZURE_PRESIGNED_IOMANAGER case OpenOptions::AzurePresigned: return new IOManagerAzurePresigned(static_cast(options).baseUrl, static_cast(options).urlSuffix, error); diff --git a/src/OpenVDS/IO/IOManagerAzureSdkForCpp.cpp b/src/OpenVDS/IO/IOManagerAzureSdkForCpp.cpp new file mode 100644 index 00000000..12c4d804 --- /dev/null +++ b/src/OpenVDS/IO/IOManagerAzureSdkForCpp.cpp @@ -0,0 +1,321 @@ +/**************************************************************************** +** Copyright 2020 The Open Group +** Copyright 2020 Bluware, Inc. +** Copyright 2020 Microsoft Corp. +** +** Licensed under the Apache License, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** http://www.apache.org/licenses/LICENSE-2.0 +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +****************************************************************************/ + +#include "IOManagerAzureSdkForCpp.h" + +#include +#include +#include +#include +#include + +#include + +namespace OpenVDS +{ + class GetHeadRequestAzureSdkForCpp : public RequestImpl + { + public: + GetHeadRequestAzureSdkForCpp(const std::string& id, const std::shared_ptr& handler); + ~GetHeadRequestAzureSdkForCpp() override; + + void Cancel() override; + + std::shared_ptr m_handler; + }; + + class ReadObjectInfoRequestAzureSdkForCpp : public GetHeadRequestAzureSdkForCpp + { + public: + ReadObjectInfoRequestAzureSdkForCpp(const std::string& id, const std::shared_ptr& handler); + + void run(ThreadPool &threadPool, Azure::Storage::Blobs::BlobServiceClient &serviceClient, const std::string &containerName, const std::string& requestName, std::weak_ptr request); + }; + + class DownloadRequestAzureSdkForCpp : public GetHeadRequestAzureSdkForCpp + { + public: + DownloadRequestAzureSdkForCpp(const std::string& id, const std::shared_ptr& handler); + + void run(ThreadPool &threadPool, Azure::Storage::Blobs::BlobServiceClient &serviceClient, const std::string &containerName, const std::string& requestName, const IORange& range, std::weak_ptr request); + + IORange m_requestedRange; + }; + + class UploadRequestAzureSdkForCpp : public RequestImpl + { + public: + UploadRequestAzureSdkForCpp(const std::string& id, std::function completedCallback); + void run(ThreadPool &threadPool, Azure::Storage::Blobs::BlobServiceClient &serviceClient, const std::string &containerName, const std::string& requestName, const std::string& contentDispositionFilename, const std::string& contentType, const std::vector>& metadataHeader, std::shared_ptr> data, std::weak_ptr uploadRequest); + void Cancel() override; + + std::function m_completedCallback; + std::shared_ptr> m_data; + }; + + GetHeadRequestAzureSdkForCpp::GetHeadRequestAzureSdkForCpp(const std::string& id, const std::shared_ptr& handler) + : RequestImpl(id) + , m_handler(handler) + { + } + + GetHeadRequestAzureSdkForCpp::~GetHeadRequestAzureSdkForCpp() + { + } + + void GetHeadRequestAzureSdkForCpp::Cancel() + { + RequestImpl::Cancel(); + } + + ReadObjectInfoRequestAzureSdkForCpp::ReadObjectInfoRequestAzureSdkForCpp(const std::string& id, const std::shared_ptr& handler) + : GetHeadRequestAzureSdkForCpp(id, handler) + { + } + + void ReadObjectInfoRequestAzureSdkForCpp::run(ThreadPool &threadPool, Azure::Storage::Blobs::BlobServiceClient &serviceClient, const std::string &containerName, const std::string& requestName, std::weak_ptr weak_request) + { + threadPool.Enqueue([serviceClient, containerName, requestName, weak_request] + { + auto request = weak_request.lock(); + if (!request) + return; + RequestStateHandler requestStateHandler(*request); + if (requestStateHandler.isCancelledRequested()) + return; + try + { + auto blobContainerClient = serviceClient.GetBlobContainerClient(containerName); + auto blockBlobClient = blobContainerClient.GetBlockBlobClient(requestName); + auto props = blockBlobClient.GetProperties().Value; + for (auto metaPair : props.Metadata) + { + request->m_handler->HandleMetadata(metaPair.first, metaPair.second); + } + request->m_handler->HandleObjectSize(props.BlobSize); + request->m_handler->HandleObjectLastWriteTime(props.LastModified.ToString()); + } + catch (const std::exception& ex) + { + request->m_error.string = ex.what(); + request->m_error.code = -1; + } + request->m_handler->Completed(*request, request->m_error); + }); + } + + DownloadRequestAzureSdkForCpp::DownloadRequestAzureSdkForCpp(const std::string& id, const std::shared_ptr& handler) + : GetHeadRequestAzureSdkForCpp(id, handler) + { + } + + void DownloadRequestAzureSdkForCpp::run(ThreadPool &threadPool, Azure::Storage::Blobs::BlobServiceClient &serviceClient, const std::string &containerName, const std::string& requestName, const IORange& range, std::weak_ptr weak_request) + { + // set options, we should probably get these through AzureOpenOptions instead of haddong here - default set in the IOMangerAzure + m_requestedRange = range; + threadPool.Enqueue([serviceClient, containerName, requestName, weak_request] + { + auto request = weak_request.lock(); + if (!request) + return; + RequestStateHandler requestStateHandler(*request); + if (requestStateHandler.isCancelledRequested()) + return; + try + { + auto blobContainerClient = serviceClient.GetBlobContainerClient(containerName); + auto blockBlobClient = blobContainerClient.GetBlockBlobClient(requestName); + Azure::Storage::Blobs::DownloadBlobOptions options; + if (request->m_requestedRange.end) + { + options.Range = Azure::Core::Http::HttpRange(); + options.Range.Value().Offset = request->m_requestedRange.start; + options.Range.Value().Length = request->m_requestedRange.end - request->m_requestedRange.start; + } + auto download = blockBlobClient.Download(options); + for (auto metaPair : download.Value.Details.Metadata) + { + request->m_handler->HandleMetadata(metaPair.first, metaPair.second); + } + request->m_handler->HandleObjectSize(download.Value.BlobSize); + request->m_handler->HandleObjectLastWriteTime(download.Value.Details.LastModified.ToString()); + std::vector data; + if (request->m_requestedRange.end) + data.resize(request->m_requestedRange.end - request->m_requestedRange.start); + else + data.resize(download.Value.BlobSize); + download.Value.BodyStream->ReadToCount(data.data(), data.size()); + request->m_handler->HandleData(std::move(data)); + } + catch (const std::exception& ex) + { + request->m_error.string = ex.what(); + request->m_error.code = -1; + } + request->m_handler->Completed(*request, request->m_error); + }); + } + + UploadRequestAzureSdkForCpp::UploadRequestAzureSdkForCpp(const std::string& id, std::function completedCallback) + : RequestImpl(id) + , m_completedCallback(completedCallback) + { + } + + void UploadRequestAzureSdkForCpp::run(ThreadPool &threadPool, Azure::Storage::Blobs::BlobServiceClient &serviceClient, const std::string &containerName, const std::string& requestName, const std::string& contentDispositionFilename, const std::string& contentType, const std::vector>& metadataHeader, std::shared_ptr> data, std::weak_ptr weak_request) + { + m_data = data; + threadPool.Enqueue([serviceClient, containerName, requestName, metadataHeader, contentDispositionFilename, weak_request] + { + auto request = weak_request.lock(); + if (!request) + return; + RequestStateHandler requestStateHandler(*request); + if (requestStateHandler.isCancelledRequested()) + return; + + try + { + auto blobContainerClient = serviceClient.GetBlobContainerClient(containerName); + auto blockBlobClient = blobContainerClient.GetBlockBlobClient(requestName); + Azure::Storage::Blobs::UploadBlockBlobFromOptions options; + for (auto& pair : metadataHeader) + { + options.Metadata.emplace(pair.first, pair.second); + } + options.HttpHeaders.ContentDisposition = contentDispositionFilename; + blockBlobClient.UploadFrom(request->m_data->data(), request->m_data->size(), options); + } + catch (const std::exception& ex) + { + request->m_error.string = ex.what(); + request->m_error.code = -1; + } + + if (request->m_completedCallback) + request->m_completedCallback(*request, request->m_error); + }); + } + + void UploadRequestAzureSdkForCpp::Cancel() + { + RequestImpl::Cancel(); + } + + class BearerCredentials : public Azure::Core::Credentials::TokenCredential + { + + }; + + IOManagerAzureSdkForCpp::IOManagerAzureSdkForCpp(const AzureOpenOptions& openOptions, Error& error) + : IOManager(OpenOptions::AzureSdkForCpp) + , m_threadPool(16) + , m_containerStr(openOptions.container) + , m_prefix(openOptions.blob) + { + if (openOptions.connectionString.empty() && openOptions.bearerToken.empty()) + { + error.code = -1; + error.string = "Azure Config error. Must provide a connection string or a bearer token"; + return; + } + if (m_containerStr.empty()) + { + error.code = -1; + error.string = "Azure Config error. Empty container or blob name"; + return; + } + + auto this_is_to_work_around_linking_inn_azure_storage_common = Azure::Storage::_internal::UrlEncodeQueryParameter(openOptions.connectionString); + (void)this_is_to_work_around_linking_inn_azure_storage_common; + + if (openOptions.connectionString.size()) + { + try + { + Azure::Storage::Blobs::BlobServiceClient serviceClient = Azure::Storage::Blobs::BlobServiceClient::CreateFromConnectionString(openOptions.connectionString); + m_serviceClient.reset(new Azure::Storage::Blobs::BlobServiceClient(serviceClient)); + } + catch (const std::exception& exception) + { + error.string = exception.what(); + error.code = -1; + return; + } + } + else + { + if (openOptions.bearerToken.empty()) + { + error.code = -1; + error.string = "Azure Config error. Neither Bearer token or connection string specified"; + return; + } + if (openOptions.accountName.empty()) + { + error.code = -1; + error.string = "Azure Config error. Account Name is mandatory when specifying bearer token"; + return; + } + error.code = -1; + error.string = "Azure Config error. Bearer token is not supported by AzureSdkForCpp"; + return; + } + } + + IOManagerAzureSdkForCpp::~IOManagerAzureSdkForCpp() + { + } + + static std::string create_id(const std::string& prefix, const std::string& objectName) + { + if (objectName.empty()) + { + return prefix; + } + if (prefix.empty()) + { + return objectName; + } + return prefix + "/" + objectName; + } + + std::shared_ptr IOManagerAzureSdkForCpp::ReadObjectInfo(const std::string& objectName, std::shared_ptr handler) + { + std::string id = create_id(m_prefix, objectName); + std::shared_ptr azureRequest = std::make_shared(id, handler); + azureRequest->run(m_threadPool, *m_serviceClient, m_containerStr, id, azureRequest); + return azureRequest; + } + + std::shared_ptr IOManagerAzureSdkForCpp::ReadObject(const std::string& objectName, std::shared_ptr handler, const IORange& range) + { + std::string id = create_id(m_prefix, objectName); + std::shared_ptr azureRequest = std::make_shared(id, handler); + azureRequest->run(m_threadPool, *m_serviceClient, m_containerStr, id, range, azureRequest); + return azureRequest; + } + + std::shared_ptr IOManagerAzureSdkForCpp::WriteObject(const std::string& objectName, const std::string& contentDispositionFilename, const std::string& contentType, const std::vector>& metadataHeader, std::shared_ptr> data, std::function completedCallback) + { + std::string id = create_id(m_prefix, objectName); + std::shared_ptr azureRequest = std::make_shared(id, completedCallback); + azureRequest->run(m_threadPool, *m_serviceClient, m_containerStr, id, contentDispositionFilename, contentType, metadataHeader, data, azureRequest); + return azureRequest; + } +} diff --git a/src/OpenVDS/IO/IOManagerAzureSdkForCpp.h b/src/OpenVDS/IO/IOManagerAzureSdkForCpp.h new file mode 100644 index 00000000..3483f46e --- /dev/null +++ b/src/OpenVDS/IO/IOManagerAzureSdkForCpp.h @@ -0,0 +1,55 @@ +/**************************************************************************** +** Copyright 2019 The Open Group +** Copyright 2019 Bluware, Inc. +** Copyright 2020 Microsoft Corp. +** +** Licensed under the Apache License, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** http://www.apache.org/licenses/LICENSE-2.0 +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +****************************************************************************/ + +#ifndef IOMANAGERAZURESDKFORCPP_H +#define IOMANAGERAZURESDKFORCPP_H + +#include "IOManager.h" +#include "IOManagerRequestImpl.h" + +#include +#include + +#include + +#include +#include + +#include + +namespace OpenVDS +{ + class IOManagerAzureSdkForCpp : public IOManager + { + public: + IOManagerAzureSdkForCpp(const AzureOpenOptions& openOptions, Error& error); + ~IOManagerAzureSdkForCpp() override; + + std::shared_ptr ReadObjectInfo(const std::string& objectName, std::shared_ptr handler) override; + std::shared_ptr ReadObject(const std::string& requestName, std::shared_ptr handler, const IORange& range = IORange()) override; + std::shared_ptr WriteObject(const std::string& requestName, const std::string& contentDispostionFilename, const std::string& contentType, const std::vector>& metadataHeader, std::shared_ptr> data, std::function completedCallback = nullptr) override; + private: + ThreadPool m_threadPool; + std::string m_containerStr; + std::string m_prefix; + std::unique_ptr m_serviceClient; + }; +} + + +#endif //IOMANAGERAZURESDKFORCPP_H diff --git a/src/OpenVDS/OpenVDS/OpenVDS.h b/src/OpenVDS/OpenVDS/OpenVDS.h index 2fc543de..5935dd63 100644 --- a/src/OpenVDS/OpenVDS/OpenVDS.h +++ b/src/OpenVDS/OpenVDS/OpenVDS.h @@ -51,6 +51,7 @@ struct OpenOptions { AWS, Azure, + AzureSdkForCpp, AzurePresigned, GoogleStorage, DMS, -- GitLab