From 58e2289f32f9163321dbe99f8414f6d1c8a76cda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Lind?= Date: Mon, 7 Jun 2021 21:17:16 +0200 Subject: [PATCH 1/5] DMS upgrade and general fixup We can not enable the performance fix because of another bug in the AWS backend, but we now detect zero size blobs by a sideeffect of reading head. --- 3rdparty/.gitignore | 1 + CMake/Fetch3rdPartyInBuild.cmake | 23 +++- src/OpenVDS/IO/IOManagerDms.cpp | 144 ++++++++++++++++---------- src/OpenVDS/IO/IOManagerRequestImpl.h | 2 +- src/OpenVDS/IO/IOManagerTransformer.h | 2 +- 5 files changed, 113 insertions(+), 59 deletions(-) diff --git a/3rdparty/.gitignore b/3rdparty/.gitignore index 29ebde67..e8d3639a 100644 --- a/3rdparty/.gitignore +++ b/3rdparty/.gitignore @@ -17,3 +17,4 @@ google_nlohmann* libressl* dms* cmakerc* +!dms-git diff --git a/CMake/Fetch3rdPartyInBuild.cmake b/CMake/Fetch3rdPartyInBuild.cmake index 0f31f665..17c70dfe 100644 --- a/CMake/Fetch3rdPartyInBuild.cmake +++ b/CMake/Fetch3rdPartyInBuild.cmake @@ -63,6 +63,26 @@ macro(Fetch3rdParty_Package name version url url_hash) endif() endmacro() +macro(Fetch3rdParty_Git name version url tag) + if (OPENVDS_3RD_PARTY_DIR) + set(Fetch3rdPartyDir "${OPENVDS_3RD_PARTY_DIR}") + else() + set(Fetch3rdPartyDir "${Fetch3rdPartyDirInternal}/../3rdparty") + endif() + get_filename_component(thirdParty "${Fetch3rdPartyDir}" ABSOLUTE) + set(SRC_DIR ${thirdParty}/${name}-${version}) + set(${name}_SOURCE_DIR ${SRC_DIR} PARENT_SCOPE) + set(${name}_VERSION ${version} PARENT_SCOPE) + if (NOT (EXISTS ${SRC_DIR})) + FetchContent_Populate(${name} + GIT_REPOSITORY ${url} + GIT_TAG ${url_hash} + SOURCE_DIR ${SRC_DIR} + SUBBUILD_DIR ${thirdParty}/CMakeArtifacts/${name}-sub-${version} + BINARY_DIR ${thirdParty}/CMakeArtifacts/${name}-${version}) + endif() +endmacro() + function(Fetch3rdParty) set(FETCHCONTENT_QUIET OFF) @@ -84,7 +104,8 @@ function(Fetch3rdParty) Fetch3rdParty_Package(google-cloud-cpp 1.14.0 https://codeload.github.com/googleapis/google-cloud-cpp/tar.gz/v1.14.0 SHA256=839b2d4dcb36a671734dac6b30ea8c298bbeaafcf7a45ee4a7d7aa5986b16569) Fetch3rdParty_Package(libxml2 2.9.12a https://codeload.github.com/GNOME/libxml2/tar.gz/v2.9.12 SHA256=8a4ddd706419c210b30b8978a51388937fd9362c34fc9a3d69e4fcc6f8055ee0) Fetch3rdParty_Package(azure-sdk-for-cpp 12.0.0b11 https://codeload.github.com/Azure/azure-sdk-for-cpp/tar.gz/azure-storage-blobs_12.0.0-beta.11 SHA256=b111636335340e3e7a5675351216dde606b8345b9906ed2f42ff8a794f5f2375) - Fetch3rdParty_Package(dms 18a85b75c7 https://community.opengroup.org/osdu/platform/domain-data-mgmt-services/seismic/seismic-dms-suite/seismic-store-cpp-lib/-/archive/18a85b75c7c10ce1bb8788d23c79964e060cee64/seismic-store-cpp-lib-master.tar.gz SHA256=146ed18ee82277355273588b6c4c834605a86a1ec40d48c5c9c11cbd47ec0165) + Fetch3rdParty_Package(dms 172dcf6a3 https://community.opengroup.org/osdu/platform/domain-data-mgmt-services/seismic/seismic-dms-suite/seismic-store-cpp-lib/-/archive/172dcf6a3b64a4f288a11c1d0bffb0dce31b29e8/seismic-store-cpp-lib-master.tar.gz SHA256=510481a8e7ed1403290d987d949db0c392d60e9776a6d9f2b8a7e225d071f2e3) + #Fetch3rdParty_Git(dms git git@community.opengroup.org:osdu/platform/domain-data-mgmt-services/seismic/seismic-dms-suite/seismic-store-cpp-lib.git master) Fetch3rdParty_File(testng java 6.14.3 jar https://repo1.maven.org/maven2/org/testng/testng/6.14.3/testng-6.14.3.jar MD5=9f17a8f9e99165e148c42b21f4b63d7c) Fetch3rdParty_File(jcommander java 1.72 jar https://repo1.maven.org/maven2/com/beust/jcommander/1.72/jcommander-1.72.jar MD5=9fde6bc0ba1032eceb7267fd1ad1657b) Fetch3rdParty_FileTarget(google_nlohmann google/cloud/storage/internal nlohmann_json.hpp 3.4.0 https://raw.githubusercontent.com/nlohmann/json/v3.4.0/single_include/nlohmann/json.hpp MD5=27f3760c1d3a0fff7d8a2407d8db8f9d) diff --git a/src/OpenVDS/IO/IOManagerDms.cpp b/src/OpenVDS/IO/IOManagerDms.cpp index ba397218..440ca885 100644 --- a/src/OpenVDS/IO/IOManagerDms.cpp +++ b/src/OpenVDS/IO/IOManagerDms.cpp @@ -35,74 +35,105 @@ namespace OpenVDS } template - static void run_request(const std::string& requestName, std::weak_ptr request, const IORange &range, std::vector* data) + static void run_request(const std::string& requestName, std::weak_ptr weak_request, const IORange &range, std::vector* data) { - auto request_ptr = request.lock(); - if (!request_ptr) - return; + } - RequestStateHandler requestHandler(*request_ptr); - - if (requestHandler.isCancelledRequested()) - { - return; - } + void ReadObjectInfoRequestDms::run(const std::string& requestName, std::weak_ptr weak_request, ThreadPool &threadPool) + { + m_job = threadPool.Enqueue([requestName, weak_request]() { + auto request = weak_request.lock(); + if (!request) + return; - uint64_t size; - std::string created_date; - try - { - size = request_ptr->m_dataset.getBlockSize(requestName); - created_date = request_ptr->m_dataset.getCreatedDate(); + RequestStateHandler requestHandler(*request); + + if (requestHandler.isCancelledRequested()) + { + return; + } + + uint64_t size; + std::string created_date; + try + { + size = request->m_dataset.getBlockSize(requestName); + created_date = request->m_dataset.getCreatedDate(); + } + catch (const seismicdrive::SDException& ex) + { + request->m_error.code = -1; + request->m_error.string = ex.what(); + } + catch (...) + { + request->m_error.code = -1; + request->m_error.string = "Unknown exception in DMS upload"; + } - if (data && size) + if (request->m_error.code == 0) + { + request->m_handler->HandleObjectSize(size); + request->m_handler->HandleObjectLastWriteTime(created_date); + } + request->m_handler->Completed(*request, request->m_error); + }); + } + void DownloadRequestDms::run(const std::string& requestName, const IORange& range, std::weak_ptr weak_request, ThreadPool &threadPool) + { + m_job = threadPool.Enqueue([requestName, weak_request, range]() { + auto request = weak_request.lock(); + if (!request) + return; + + RequestStateHandler requestHandler(*request); + + if (requestHandler.isCancelledRequested()) + { + return; + } + + std::vector data; + try { - uint64_t offset; if (range.end) { - data->resize(range.end - range.start); - offset = range.start; + data.resize(range.end - range.start); + request->m_dataset.readBlock(requestName, (char*)data.data(), range.start, data.size()); } else { - offset = 0; - data->resize(size); + //char* read_data = nullptr; + //std::size_t read_size = 0; + //request->m_dataset.readBlock(requestName, &read_data, read_size); + //data.resize(read_size); + //memcpy(data.data(), read_data, data.size()); + //delete[] read_data; + size_t read_size = size_t(request->m_dataset.getBlockSize(requestName)); + if (read_size) + { + data.resize(read_size); + request->m_dataset.readBlock(requestName, (char*)data.data(), read_size); + } } - request_ptr->m_dataset.readBlock(requestName, (char*)data->data(), offset, data->size()); } - } - catch (const seismicdrive::SDException& ex) - { - request_ptr->m_error.code = -1; - request_ptr->m_error.string = ex.what(); - } - catch (...) - { - request_ptr->m_error.code = -1; - request_ptr->m_error.string = "Unknown exception in DMS upload"; - } - - if (request_ptr->m_error.code == 0) - { - request_ptr->m_handler->HandleObjectSize(size); - request_ptr->m_handler->HandleObjectLastWriteTime(created_date); - if (data) - request_ptr->m_handler->HandleData(std::move(*data)); - } - request_ptr->m_handler->Completed(*request_ptr, request_ptr->m_error); - } + catch (const seismicdrive::SDException& ex) + { + request->m_error.code = -1; + request->m_error.string = ex.what(); + } + catch (...) + { + request->m_error.code = -1; + request->m_error.string = "Unknown exception in DMS upload"; + } - void ReadObjectInfoRequestDms::run(const std::string& requestName, std::weak_ptr request, ThreadPool &threadPool) - { - m_job = threadPool.Enqueue([requestName, request]() { - run_request(requestName, request, IORange(), nullptr); - }); - } - void DownloadRequestDms::run(const std::string& requestName, const IORange& range, std::weak_ptr request, ThreadPool &threadPool) - { - m_job = threadPool.Enqueue([requestName, request, range]() { - std::vector data; - run_request(requestName, request, range, &data); + if (request->m_error.code == 0) + { + request->m_handler->HandleObjectSize(data.size()); + request->m_handler->HandleData(std::move(data)); + } + request->m_handler->Completed(*request, request->m_error); }); } @@ -128,7 +159,8 @@ namespace OpenVDS try { - request_ptr->m_dataset.writeBlock(requestName, (const char*)data->data(), data->size(), false); + if (data->size()) + request_ptr->m_dataset.writeBlock(requestName, (const char*)data->data(), data->size(), false); } catch (const seismicdrive::SDException& ex) { diff --git a/src/OpenVDS/IO/IOManagerRequestImpl.h b/src/OpenVDS/IO/IOManagerRequestImpl.h index 53a4128d..686cf0e7 100644 --- a/src/OpenVDS/IO/IOManagerRequestImpl.h +++ b/src/OpenVDS/IO/IOManagerRequestImpl.h @@ -46,7 +46,7 @@ public: return error.code == 0; } - void Cancel() + void Cancel() override { m_cancelledRequested = true; } diff --git a/src/OpenVDS/IO/IOManagerTransformer.h b/src/OpenVDS/IO/IOManagerTransformer.h index e30f87f4..a75d0540 100644 --- a/src/OpenVDS/IO/IOManagerTransformer.h +++ b/src/OpenVDS/IO/IOManagerTransformer.h @@ -16,7 +16,7 @@ ****************************************************************************/ #ifndef IOMANAGERTRANSFORMER_H -#define IOMANAGERTRANSFORMAT_H +#define IOMANAGERTRANSFORMER_H #include -- GitLab From 0f117f142926a2c27587e49383ca24eb2f1f0706 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Lind?= Date: Thu, 17 Jun 2021 10:58:13 +0200 Subject: [PATCH 2/5] Install OpenMP libs --- CMakeLists.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5c6a965f..99b8cdaf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -161,7 +161,10 @@ set(LIB_TOOLSET_DIR "${LIB_TOOLSET_DIR}$<$>:/$>") set_property(GLOBAL PROPERTY USE_FOLDERS ON) find_package(Threads) if (ENABLE_OPENMP) - find_package(OpenMP) + find_package(OpenMP) +endif() +if (OpenMP_CXX_FOUND) + set(CMAKE_INSTALL_OPENMP_LIBRARIES TRUE) endif() include(CMake/SetWarnings.cmake) -- GitLab From 5866b8cc109909b64facb692055d24e45e3ff6da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Lind?= Date: Thu, 17 Jun 2021 13:45:21 +0200 Subject: [PATCH 3/5] LinuxLinkerChainHack --- src/OpenVDS/IO/IOManagerAWS.cpp | 8 ++++++++ tests/CMakeLists.txt | 7 +++++++ tests/LinuxLinkerChainHack.cpp | 25 +++++++++++++++++++++++++ 3 files changed, 40 insertions(+) create mode 100644 tests/LinuxLinkerChainHack.cpp diff --git a/src/OpenVDS/IO/IOManagerAWS.cpp b/src/OpenVDS/IO/IOManagerAWS.cpp index 5af16bce..4139ba63 100644 --- a/src/OpenVDS/IO/IOManagerAWS.cpp +++ b/src/OpenVDS/IO/IOManagerAWS.cpp @@ -36,6 +36,7 @@ #include #include #include +#include #include #include @@ -442,6 +443,13 @@ namespace OpenVDS { m_s3Client.reset(new Aws::S3::S3Client(credentials, clientConfig, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, useVirtualAddressing)); } + + //We do this to use a symbol from the transfermanager so we get the linker chain working on linux + Aws::Utils::Threading::DefaultExecutor threadExecutor; + Aws::Transfer::TransferManagerConfiguration transferConfig(&threadExecutor); + std::shared_ptr s3ClientSharedPtr(m_s3Client.get(), [](Aws::S3::S3Client*) {}); + transferConfig.s3Client = s3ClientSharedPtr; + auto manager = Aws::Transfer::TransferManager::Create(transferConfig); } IOManagerAWS::~IOManagerAWS() diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index e57033ae..298c4a74 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -51,6 +51,13 @@ function(add_test_executable name) if (OpenMP_CXX_FOUND) target_link_libraries(${name} PRIVATE OpenMP::OpenMP_CXX) endif() + if (DISABLE_AWS_IOMANAGER) + target_compile_definitions(${name} PRIVATE OPENVDS_NO_AWS_IOMANAGER) + endif() + if (DISABLE_AZURESDKFORCPP_IOMANAGER) + target_compile_definitions(${name} PRIVATE OPENVDS_NO_AZURE_SDK_FOR_CPP_IOMANAGER) + endif() + target_sources(${name} PRIVATE "LinuxLinkerChainHack.cpp") endfunction() add_test_executable(io_tests diff --git a/tests/LinuxLinkerChainHack.cpp b/tests/LinuxLinkerChainHack.cpp new file mode 100644 index 00000000..e98a0756 --- /dev/null +++ b/tests/LinuxLinkerChainHack.cpp @@ -0,0 +1,25 @@ +#ifndef OPENVDS_NO_AWS_IOMANAGER +#include +#include + +void addSomeSymbolsToTheExecutableSoTheLinkerChainWorkAws() +{ + //We do this to use a symbol from the transfermanager so we get the linker chain working on linux + Aws::Auth::AWSCredentials credentials; + Aws::S3::S3Client client(credentials); + Aws::Utils::Threading::DefaultExecutor threadExecutor; + Aws::Transfer::TransferManagerConfiguration transferConfig(&threadExecutor); + std::shared_ptr s3ClientSharedPtr(&client, [](Aws::S3::S3Client*) {}); + transferConfig.s3Client = s3ClientSharedPtr; + auto manager = Aws::Transfer::TransferManager::Create(transferConfig); +} +#endif + +#ifndef OPENVDS_NO_AZURE_SDK_FOR_CPP_IOMANAGER +#include +void addSomeSymbolsToTheExecutableSoTheLinkerChainWorkAzure() +{ + auto this_is_to_work_around_linking_inn_azure_storage_common = Azure::Storage::_internal::UrlEncodeQueryParameter("hello world"); + (void)this_is_to_work_around_linking_inn_azure_storage_common; +} +#endif -- GitLab From cb4c6529e6cad9e0f0b016b884b6894c6345cad3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Lind?= Date: Fri, 18 Jun 2021 00:12:43 +0200 Subject: [PATCH 4/5] Disable the "missing metadata tags" warning with OPENVDS_DISABLE_WARNINGS --- src/OpenVDS/VDS/Env.cpp | 23 ++++++++++++++------ src/OpenVDS/VDS/VolumeDataStoreIOManager.cpp | 13 ++++++++--- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/src/OpenVDS/VDS/Env.cpp b/src/OpenVDS/VDS/Env.cpp index 0618b0ce..b3d29c9c 100644 --- a/src/OpenVDS/VDS/Env.cpp +++ b/src/OpenVDS/VDS/Env.cpp @@ -3,21 +3,30 @@ #include #include +#include +#include namespace OpenVDS { + + inline char asciitolower(char in) { + if (in <= 'Z' && in >= 'A') + return in - ('Z' - 'z'); + return in; +} + bool getBooleanEnvironmentVariable(const char *name) { - const char *var = getenv(name); - if (!var) - return false; - if (strcmp(var, "") == 0) + const char *c_var = getenv(name); + if (c_var == nullptr) return false; - if (strcmp(var, "false") == 0) + std::string var(c_var); + std::transform(var.begin(), var.end(), var.begin(), asciitolower); + if (var == "false") return false; - if (strcmp(var, "FALSE") == 0) + if (var == "0") return false; - if (strcmp(var, "0") == 0) + if (var == "off") return false; return true; } diff --git a/src/OpenVDS/VDS/VolumeDataStoreIOManager.cpp b/src/OpenVDS/VDS/VolumeDataStoreIOManager.cpp index 2ba4ff6c..0c89de02 100644 --- a/src/OpenVDS/VDS/VolumeDataStoreIOManager.cpp +++ b/src/OpenVDS/VDS/VolumeDataStoreIOManager.cpp @@ -27,6 +27,8 @@ #include "WaveletTypes.h" +#include "Env.h" + #include #include @@ -179,7 +181,7 @@ VolumeDataStoreIOManager:: VolumeDataStoreIOManager(VDS &vds, IOManager *ioManag : VolumeDataStore(ioManager->connectionType()) , m_vds(vds) , m_ioManager(ioManager) - , m_warnedAboutMissingMetadataTag(false) + , m_warnedAboutMissingMetadataTag(getBooleanEnvironmentVariable("OPENVDS_DISABLE_WARNINGS")) { } @@ -506,8 +508,13 @@ bool VolumeDataStoreIOManager::ReadChunk(const VolumeDataChunk &chunk, int adapt { if (!m_warnedAboutMissingMetadataTag) // Log once and move along. { - fmt::print(stderr, "Dataset has missing metadata tags, degraded data verification, reverting to metadata pages"); - m_warnedAboutMissingMetadataTag = true; + lock.lock(); + if (!m_warnedAboutMissingMetadataTag) + { + fmt::print(stderr, "Dataset has missing metadata tags, degraded data verification, reverting to metadata pages"); + m_warnedAboutMissingMetadataTag = true; + } + lock.unlock(); } if (moveData) -- GitLab From e57de1d32f6e88718e7dcb7703beb209de1b475a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Lind?= Date: Fri, 18 Jun 2021 08:08:12 +0000 Subject: [PATCH 5/5] Fix Alpine build --- CMakeLists.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 99b8cdaf..5950bb2f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -47,6 +47,12 @@ else() set(LIBSUFFIX "") endif() +if (EXISTS /etc/alpine-release) #the Alpine folks desided to patch cmake but forgot to patch the FIND_LIBRARY_USE_LIB64_PATHS + if (NOT ${CMAKE_INSTALL_LIBDIR} STREQUAL "lib64") + set(LIBSUFFIX "") + endif() +endif() + set(BUILD_PYTHON_DEFAULT ON) if (Python_ROOT_DIR) set(PYTHON_ROOT_DIR_HINT "HINTS ${Python_ROOT_DIR}") -- GitLab