From ec87733e8c3decf77d017c2410fbbd7426b1c27f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Lind?= Date: Mon, 7 Jun 2021 21:17:16 +0200 Subject: [PATCH 1/8] DMS upgrade and general fixup We can not enable the performance fix because of another bug in the AWS backend, but we now detect zero size blobs by a sideeffect of reading head. --- 3rdparty/.gitignore | 2 + CMake/Fetch3rdPartyInBuild.cmake | 23 +++- src/OpenVDS/IO/IOManagerDms.cpp | 144 ++++++++++++++++---------- src/OpenVDS/IO/IOManagerRequestImpl.h | 2 +- src/OpenVDS/IO/IOManagerTransformer.h | 2 +- 5 files changed, 114 insertions(+), 59 deletions(-) diff --git a/3rdparty/.gitignore b/3rdparty/.gitignore index 6a930069..e8d3639a 100644 --- a/3rdparty/.gitignore +++ b/3rdparty/.gitignore @@ -16,3 +16,5 @@ java* google_nlohmann* libressl* dms* +cmakerc* +!dms-git diff --git a/CMake/Fetch3rdPartyInBuild.cmake b/CMake/Fetch3rdPartyInBuild.cmake index 2d8b0ec7..3a79dc24 100644 --- a/CMake/Fetch3rdPartyInBuild.cmake +++ b/CMake/Fetch3rdPartyInBuild.cmake @@ -63,6 +63,26 @@ macro(Fetch3rdParty_Package name version url url_hash) endif() endmacro() +macro(Fetch3rdParty_Git name version url tag) + if (OPENVDS_3RD_PARTY_DIR) + set(Fetch3rdPartyDir "${OPENVDS_3RD_PARTY_DIR}") + else() + set(Fetch3rdPartyDir "${Fetch3rdPartyDirInternal}/../3rdparty") + endif() + get_filename_component(thirdParty "${Fetch3rdPartyDir}" ABSOLUTE) + set(SRC_DIR ${thirdParty}/${name}-${version}) + set(${name}_SOURCE_DIR ${SRC_DIR} PARENT_SCOPE) + set(${name}_VERSION ${version} PARENT_SCOPE) + if (NOT (EXISTS ${SRC_DIR})) + FetchContent_Populate(${name} + GIT_REPOSITORY ${url} + GIT_TAG ${url_hash} + SOURCE_DIR ${SRC_DIR} + SUBBUILD_DIR ${thirdParty}/CMakeArtifacts/${name}-sub-${version} + BINARY_DIR ${thirdParty}/CMakeArtifacts/${name}-${version}) + endif() +endmacro() + function(Fetch3rdParty) include(CMake/FetchContentLocal.cmake) @@ -82,7 +102,8 @@ function(Fetch3rdParty) Fetch3rdParty_Package(absl 20200225.2 https://codeload.github.com/abseil/abseil-cpp/tar.gz/20200225.2 SHA256=f41868f7a938605c92936230081175d1eae87f6ea2c248f41077c8f88316f111) Fetch3rdParty_Package(crc32c 1.1.1 https://codeload.github.com/google/crc32c/tar.gz/1.1.1 SHA256=a6533f45b1670b5d59b38a514d82b09c6fb70cc1050467220216335e873074e8) Fetch3rdParty_Package(google-cloud-cpp 1.14.0 https://codeload.github.com/googleapis/google-cloud-cpp/tar.gz/v1.14.0 SHA256=839b2d4dcb36a671734dac6b30ea8c298bbeaafcf7a45ee4a7d7aa5986b16569) - Fetch3rdParty_Package(dms 18a85b75c7 https://community.opengroup.org/osdu/platform/domain-data-mgmt-services/seismic/seismic-dms-suite/seismic-store-cpp-lib/-/archive/18a85b75c7c10ce1bb8788d23c79964e060cee64/seismic-store-cpp-lib-master.tar.gz SHA256=146ed18ee82277355273588b6c4c834605a86a1ec40d48c5c9c11cbd47ec0165) + Fetch3rdParty_Package(dms 172dcf6a3 https://community.opengroup.org/osdu/platform/domain-data-mgmt-services/seismic/seismic-dms-suite/seismic-store-cpp-lib/-/archive/172dcf6a3b64a4f288a11c1d0bffb0dce31b29e8/seismic-store-cpp-lib-master.tar.gz SHA256=510481a8e7ed1403290d987d949db0c392d60e9776a6d9f2b8a7e225d071f2e3) + #Fetch3rdParty_Git(dms git git@community.opengroup.org:osdu/platform/domain-data-mgmt-services/seismic/seismic-dms-suite/seismic-store-cpp-lib.git master) Fetch3rdParty_File(testng java 6.14.3 jar https://repo1.maven.org/maven2/org/testng/testng/6.14.3/testng-6.14.3.jar MD5=9f17a8f9e99165e148c42b21f4b63d7c) Fetch3rdParty_File(jcommander java 1.72 jar https://repo1.maven.org/maven2/com/beust/jcommander/1.72/jcommander-1.72.jar MD5=9fde6bc0ba1032eceb7267fd1ad1657b) Fetch3rdParty_FileTarget(google_nlohmann google/cloud/storage/internal nlohmann_json.hpp 3.4.0 https://raw.githubusercontent.com/nlohmann/json/v3.4.0/single_include/nlohmann/json.hpp MD5=27f3760c1d3a0fff7d8a2407d8db8f9d) diff --git a/src/OpenVDS/IO/IOManagerDms.cpp b/src/OpenVDS/IO/IOManagerDms.cpp index ba397218..440ca885 100644 --- a/src/OpenVDS/IO/IOManagerDms.cpp +++ b/src/OpenVDS/IO/IOManagerDms.cpp @@ -35,74 +35,105 @@ namespace OpenVDS } template - static void run_request(const std::string& requestName, std::weak_ptr request, const IORange &range, std::vector* data) + static void run_request(const std::string& requestName, std::weak_ptr weak_request, const IORange &range, std::vector* data) { - auto request_ptr = request.lock(); - if (!request_ptr) - return; + } - RequestStateHandler requestHandler(*request_ptr); - - if (requestHandler.isCancelledRequested()) - { - return; - } + void ReadObjectInfoRequestDms::run(const std::string& requestName, std::weak_ptr weak_request, ThreadPool &threadPool) + { + m_job = threadPool.Enqueue([requestName, weak_request]() { + auto request = weak_request.lock(); + if (!request) + return; - uint64_t size; - std::string created_date; - try - { - size = request_ptr->m_dataset.getBlockSize(requestName); - created_date = request_ptr->m_dataset.getCreatedDate(); + RequestStateHandler requestHandler(*request); + + if (requestHandler.isCancelledRequested()) + { + return; + } + + uint64_t size; + std::string created_date; + try + { + size = request->m_dataset.getBlockSize(requestName); + created_date = request->m_dataset.getCreatedDate(); + } + catch (const seismicdrive::SDException& ex) + { + request->m_error.code = -1; + request->m_error.string = ex.what(); + } + catch (...) + { + request->m_error.code = -1; + request->m_error.string = "Unknown exception in DMS upload"; + } - if (data && size) + if (request->m_error.code == 0) + { + request->m_handler->HandleObjectSize(size); + request->m_handler->HandleObjectLastWriteTime(created_date); + } + request->m_handler->Completed(*request, request->m_error); + }); + } + void DownloadRequestDms::run(const std::string& requestName, const IORange& range, std::weak_ptr weak_request, ThreadPool &threadPool) + { + m_job = threadPool.Enqueue([requestName, weak_request, range]() { + auto request = weak_request.lock(); + if (!request) + return; + + RequestStateHandler requestHandler(*request); + + if (requestHandler.isCancelledRequested()) + { + return; + } + + std::vector data; + try { - uint64_t offset; if (range.end) { - data->resize(range.end - range.start); - offset = range.start; + data.resize(range.end - range.start); + request->m_dataset.readBlock(requestName, (char*)data.data(), range.start, data.size()); } else { - offset = 0; - data->resize(size); + //char* read_data = nullptr; + //std::size_t read_size = 0; + //request->m_dataset.readBlock(requestName, &read_data, read_size); + //data.resize(read_size); + //memcpy(data.data(), read_data, data.size()); + //delete[] read_data; + size_t read_size = size_t(request->m_dataset.getBlockSize(requestName)); + if (read_size) + { + data.resize(read_size); + request->m_dataset.readBlock(requestName, (char*)data.data(), read_size); + } } - request_ptr->m_dataset.readBlock(requestName, (char*)data->data(), offset, data->size()); } - } - catch (const seismicdrive::SDException& ex) - { - request_ptr->m_error.code = -1; - request_ptr->m_error.string = ex.what(); - } - catch (...) - { - request_ptr->m_error.code = -1; - request_ptr->m_error.string = "Unknown exception in DMS upload"; - } - - if (request_ptr->m_error.code == 0) - { - request_ptr->m_handler->HandleObjectSize(size); - request_ptr->m_handler->HandleObjectLastWriteTime(created_date); - if (data) - request_ptr->m_handler->HandleData(std::move(*data)); - } - request_ptr->m_handler->Completed(*request_ptr, request_ptr->m_error); - } + catch (const seismicdrive::SDException& ex) + { + request->m_error.code = -1; + request->m_error.string = ex.what(); + } + catch (...) + { + request->m_error.code = -1; + request->m_error.string = "Unknown exception in DMS upload"; + } - void ReadObjectInfoRequestDms::run(const std::string& requestName, std::weak_ptr request, ThreadPool &threadPool) - { - m_job = threadPool.Enqueue([requestName, request]() { - run_request(requestName, request, IORange(), nullptr); - }); - } - void DownloadRequestDms::run(const std::string& requestName, const IORange& range, std::weak_ptr request, ThreadPool &threadPool) - { - m_job = threadPool.Enqueue([requestName, request, range]() { - std::vector data; - run_request(requestName, request, range, &data); + if (request->m_error.code == 0) + { + request->m_handler->HandleObjectSize(data.size()); + request->m_handler->HandleData(std::move(data)); + } + request->m_handler->Completed(*request, request->m_error); }); } @@ -128,7 +159,8 @@ namespace OpenVDS try { - request_ptr->m_dataset.writeBlock(requestName, (const char*)data->data(), data->size(), false); + if (data->size()) + request_ptr->m_dataset.writeBlock(requestName, (const char*)data->data(), data->size(), false); } catch (const seismicdrive::SDException& ex) { diff --git a/src/OpenVDS/IO/IOManagerRequestImpl.h b/src/OpenVDS/IO/IOManagerRequestImpl.h index 53a4128d..686cf0e7 100644 --- a/src/OpenVDS/IO/IOManagerRequestImpl.h +++ b/src/OpenVDS/IO/IOManagerRequestImpl.h @@ -46,7 +46,7 @@ public: return error.code == 0; } - void Cancel() + void Cancel() override { m_cancelledRequested = true; } diff --git a/src/OpenVDS/IO/IOManagerTransformer.h b/src/OpenVDS/IO/IOManagerTransformer.h index e30f87f4..a75d0540 100644 --- a/src/OpenVDS/IO/IOManagerTransformer.h +++ b/src/OpenVDS/IO/IOManagerTransformer.h @@ -16,7 +16,7 @@ ****************************************************************************/ #ifndef IOMANAGERTRANSFORMER_H -#define IOMANAGERTRANSFORMAT_H +#define IOMANAGERTRANSFORMER_H #include -- GitLab From d686d2cd0ff7dd78a8269056c57dff395437644a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Lind?= Date: Thu, 17 Jun 2021 10:58:13 +0200 Subject: [PATCH 2/8] Install OpenMP libs --- CMakeLists.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4e0597fd..e58ab321 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -141,7 +141,10 @@ set(LIB_TOOLSET_DIR "${LIB_TOOLSET_DIR}$<$>:/$>") set_property(GLOBAL PROPERTY USE_FOLDERS ON) find_package(Threads) if (ENABLE_OPENMP) - find_package(OpenMP) + find_package(OpenMP) +endif() +if (OpenMP_CXX_FOUND) + set(CMAKE_INSTALL_OPENMP_LIBRARIES TRUE) endif() include(CMake/SetWarnings.cmake) -- GitLab From e3a48b85798486472007cbbab9bd261b9d58abb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Lind?= Date: Thu, 17 Jun 2021 13:45:21 +0200 Subject: [PATCH 3/8] LinkuxLinkerChainHack --- src/OpenVDS/IO/IOManagerAWS.cpp | 8 ++++++++ tests/CMakeLists.txt | 7 +++++++ tests/LinuxLinkerChainHack.cpp | 17 +++++++++++++++++ 3 files changed, 32 insertions(+) create mode 100644 tests/LinuxLinkerChainHack.cpp diff --git a/src/OpenVDS/IO/IOManagerAWS.cpp b/src/OpenVDS/IO/IOManagerAWS.cpp index 5af16bce..4139ba63 100644 --- a/src/OpenVDS/IO/IOManagerAWS.cpp +++ b/src/OpenVDS/IO/IOManagerAWS.cpp @@ -36,6 +36,7 @@ #include #include #include +#include #include #include @@ -442,6 +443,13 @@ namespace OpenVDS { m_s3Client.reset(new Aws::S3::S3Client(credentials, clientConfig, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, useVirtualAddressing)); } + + //We do this to use a symbol from the transfermanager so we get the linker chain working on linux + Aws::Utils::Threading::DefaultExecutor threadExecutor; + Aws::Transfer::TransferManagerConfiguration transferConfig(&threadExecutor); + std::shared_ptr s3ClientSharedPtr(m_s3Client.get(), [](Aws::S3::S3Client*) {}); + transferConfig.s3Client = s3ClientSharedPtr; + auto manager = Aws::Transfer::TransferManager::Create(transferConfig); } IOManagerAWS::~IOManagerAWS() diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index e57033ae..298c4a74 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -51,6 +51,13 @@ function(add_test_executable name) if (OpenMP_CXX_FOUND) target_link_libraries(${name} PRIVATE OpenMP::OpenMP_CXX) endif() + if (DISABLE_AWS_IOMANAGER) + target_compile_definitions(${name} PRIVATE OPENVDS_NO_AWS_IOMANAGER) + endif() + if (DISABLE_AZURESDKFORCPP_IOMANAGER) + target_compile_definitions(${name} PRIVATE OPENVDS_NO_AZURE_SDK_FOR_CPP_IOMANAGER) + endif() + target_sources(${name} PRIVATE "LinuxLinkerChainHack.cpp") endfunction() add_test_executable(io_tests diff --git a/tests/LinuxLinkerChainHack.cpp b/tests/LinuxLinkerChainHack.cpp new file mode 100644 index 00000000..80924e1a --- /dev/null +++ b/tests/LinuxLinkerChainHack.cpp @@ -0,0 +1,17 @@ +#ifndef OPENVDS_NO_AWS_IOMANAGER +#include +#include + +void addSomeSymbolsToTheExecutableSoTheLinkerChainWorkAws() +{ + //We do this to use a symbol from the transfermanager so we get the linker chain working on linux + Aws::Auth::AWSCredentials credentials; + Aws::S3::S3Client client(credentials); + Aws::Utils::Threading::DefaultExecutor threadExecutor; + Aws::Transfer::TransferManagerConfiguration transferConfig(&threadExecutor); + std::shared_ptr s3ClientSharedPtr(&client, [](Aws::S3::S3Client*) {}); + transferConfig.s3Client = s3ClientSharedPtr; + auto manager = Aws::Transfer::TransferManager::Create(transferConfig); +} +#endif + -- GitLab From 27236924e57098d0b92986437336b60003cfceab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Lind?= Date: Fri, 18 Jun 2021 00:12:43 +0200 Subject: [PATCH 4/8] Disable the "missing metadata tags" warning with OPENVDS_DISABLE_WARNINGS --- src/OpenVDS/VDS/Env.cpp | 23 ++++++++++++++------ src/OpenVDS/VDS/VolumeDataStoreIOManager.cpp | 13 ++++++++--- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/src/OpenVDS/VDS/Env.cpp b/src/OpenVDS/VDS/Env.cpp index 0618b0ce..b3d29c9c 100644 --- a/src/OpenVDS/VDS/Env.cpp +++ b/src/OpenVDS/VDS/Env.cpp @@ -3,21 +3,30 @@ #include #include +#include +#include namespace OpenVDS { + + inline char asciitolower(char in) { + if (in <= 'Z' && in >= 'A') + return in - ('Z' - 'z'); + return in; +} + bool getBooleanEnvironmentVariable(const char *name) { - const char *var = getenv(name); - if (!var) - return false; - if (strcmp(var, "") == 0) + const char *c_var = getenv(name); + if (c_var == nullptr) return false; - if (strcmp(var, "false") == 0) + std::string var(c_var); + std::transform(var.begin(), var.end(), var.begin(), asciitolower); + if (var == "false") return false; - if (strcmp(var, "FALSE") == 0) + if (var == "0") return false; - if (strcmp(var, "0") == 0) + if (var == "off") return false; return true; } diff --git a/src/OpenVDS/VDS/VolumeDataStoreIOManager.cpp b/src/OpenVDS/VDS/VolumeDataStoreIOManager.cpp index 2ba4ff6c..0c89de02 100644 --- a/src/OpenVDS/VDS/VolumeDataStoreIOManager.cpp +++ b/src/OpenVDS/VDS/VolumeDataStoreIOManager.cpp @@ -27,6 +27,8 @@ #include "WaveletTypes.h" +#include "Env.h" + #include #include @@ -179,7 +181,7 @@ VolumeDataStoreIOManager:: VolumeDataStoreIOManager(VDS &vds, IOManager *ioManag : VolumeDataStore(ioManager->connectionType()) , m_vds(vds) , m_ioManager(ioManager) - , m_warnedAboutMissingMetadataTag(false) + , m_warnedAboutMissingMetadataTag(getBooleanEnvironmentVariable("OPENVDS_DISABLE_WARNINGS")) { } @@ -506,8 +508,13 @@ bool VolumeDataStoreIOManager::ReadChunk(const VolumeDataChunk &chunk, int adapt { if (!m_warnedAboutMissingMetadataTag) // Log once and move along. { - fmt::print(stderr, "Dataset has missing metadata tags, degraded data verification, reverting to metadata pages"); - m_warnedAboutMissingMetadataTag = true; + lock.lock(); + if (!m_warnedAboutMissingMetadataTag) + { + fmt::print(stderr, "Dataset has missing metadata tags, degraded data verification, reverting to metadata pages"); + m_warnedAboutMissingMetadataTag = true; + } + lock.unlock(); } if (moveData) -- GitLab From 09c34001f7382376a29a1057a49ff58173203faa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Lind?= Date: Fri, 18 Jun 2021 08:08:12 +0000 Subject: [PATCH 5/8] Fix Alpine build --- CMakeLists.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index e58ab321..5b71e398 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,6 +34,12 @@ else() set(LIBSUFFIX "") endif() +if (EXISTS /etc/alpine-release) #the Alpine folks desided to patch cmake but forgot to patch the FIND_LIBRARY_USE_LIB64_PATHS + if (NOT ${CMAKE_INSTALL_LIBDIR} STREQUAL "lib64") + set(LIBSUFFIX "") + endif() +endif() + set(BUILD_PYTHON_DEFAULT ON) if (Python_ROOT_DIR) set(PYTHON_ROOT_DIR_HINT "HINTS ${Python_ROOT_DIR}") -- GitLab From 1616b39997ca69196013dc5cf34e716c9df3dd3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Lind?= Date: Fri, 18 Jun 2021 13:02:59 +0200 Subject: [PATCH 6/8] add a newline to printf output --- src/OpenVDS/VDS/VolumeDataStoreIOManager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/OpenVDS/VDS/VolumeDataStoreIOManager.cpp b/src/OpenVDS/VDS/VolumeDataStoreIOManager.cpp index 0c89de02..1f16894f 100644 --- a/src/OpenVDS/VDS/VolumeDataStoreIOManager.cpp +++ b/src/OpenVDS/VDS/VolumeDataStoreIOManager.cpp @@ -511,7 +511,7 @@ bool VolumeDataStoreIOManager::ReadChunk(const VolumeDataChunk &chunk, int adapt lock.lock(); if (!m_warnedAboutMissingMetadataTag) { - fmt::print(stderr, "Dataset has missing metadata tags, degraded data verification, reverting to metadata pages"); + fmt::print(stderr, "Dataset has missing metadata tags, degraded data verification, reverting to metadata pages\n"); m_warnedAboutMissingMetadataTag = true; } lock.unlock(); -- GitLab From 826249e630d89d7bf61436ecf95c604618a17bfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Lind?= Date: Mon, 21 Jun 2021 12:34:37 +0200 Subject: [PATCH 7/8] enable read objects from objectstore without having an object name This makes uri handling in SEGYImport easier and makes it possible for SEGYImport to read seagy from dms location. This is a behavior change in the IOManager api so all backends have been manually verified --- src/OpenVDS/IO/IOManagerAzurePresigned.cpp | 19 ++++++++++++----- src/OpenVDS/IO/IOManagerDms.cpp | 24 +++++++++++++++++----- src/OpenVDS/IO/IOManagerDms.h | 1 + src/OpenVDS/IO/IOManagerGoogle.cpp | 7 ++++++- src/OpenVDS/IO/IOManagerHttp.cpp | 17 +++++++++++---- src/SEGYUtils/SEGYUtils/DataProvider.h | 16 +++++++-------- tools/SEGYImport/SEGYImport.cpp | 24 ++++++++-------------- 7 files changed, 69 insertions(+), 39 deletions(-) diff --git a/src/OpenVDS/IO/IOManagerAzurePresigned.cpp b/src/OpenVDS/IO/IOManagerAzurePresigned.cpp index 7e5c0e09..58d8ca50 100644 --- a/src/OpenVDS/IO/IOManagerAzurePresigned.cpp +++ b/src/OpenVDS/IO/IOManagerAzurePresigned.cpp @@ -34,8 +34,8 @@ namespace OpenVDS return; } - if (m_base.back() != '/') - m_base.push_back('/'); + if (m_base.back() == '/') + m_base.pop_back(); if (m_suffix.size()) { @@ -44,9 +44,18 @@ namespace OpenVDS } } + static std::string getUrl(const std::string& base, const std::string& objectName, const std::string &suffix) + { + if (objectName.empty()) + { + return base + suffix; + } + return base + "/" + objectName + suffix; + } + std::shared_ptr IOManagerAzurePresigned::ReadObjectInfo(const std::string& objectName, std::shared_ptr handler) { - std::string url = m_base + objectName + m_suffix; + std::string url = getUrl(m_base, objectName, m_suffix); std::shared_ptr request = std::make_shared(objectName, handler); std::vector headers; m_curlHandler.addDownloadRequest(request, url, headers, convertToISO8601, CurlDownloadHandler::HEADER); @@ -55,7 +64,7 @@ namespace OpenVDS } std::shared_ptr IOManagerAzurePresigned::ReadObject(const std::string& objectName, std::shared_ptr handler, const IORange& range) { - std::string url = m_base + objectName + m_suffix; + std::string url = getUrl(m_base, objectName, m_suffix); std::shared_ptr request = std::make_shared(objectName, handler); std::vector headers; if (range.start != range.end) @@ -69,7 +78,7 @@ namespace OpenVDS } std::shared_ptr IOManagerAzurePresigned::WriteObject(const std::string& objectName, const std::string& contentDispostionFilename, const std::string& contentType, const std::vector>& metadataHeader, std::shared_ptr> data, std::function completedCallback) { - std::string url = m_base + objectName + m_suffix; + std::string url = getUrl(m_base, objectName, m_suffix); std::shared_ptr request = std::make_shared(objectName, completedCallback); std::vector headers; headers.emplace_back("x-ms-blob-type: BlockBlob"); diff --git a/src/OpenVDS/IO/IOManagerDms.cpp b/src/OpenVDS/IO/IOManagerDms.cpp index 440ca885..2917867a 100644 --- a/src/OpenVDS/IO/IOManagerDms.cpp +++ b/src/OpenVDS/IO/IOManagerDms.cpp @@ -183,6 +183,18 @@ namespace OpenVDS , m_opened(false) , m_threadPool(16) { + if (openOptions.datasetPath.size()) + { + auto it = openOptions.datasetPath.rfind('/'); + if (it == openOptions.datasetPath.size() - 1) + { + it = openOptions.datasetPath.rfind('/', 1); + } + if (it != std::string::npos) + { + m_filename = openOptions.datasetPath.substr(it+1); + } + } try { m_sdManager.reset(new seismicdrive::SDManager(openOptions.sdAuthorityUrl, openOptions.sdApiKey, openOptions.logLevel)); m_sdManager->setAuthProviderFromString(openOptions.sdToken); @@ -223,15 +235,17 @@ namespace OpenVDS std::shared_ptr IOManagerDms::ReadObjectInfo(const std::string& objectName, std::shared_ptr handler) { - auto req = std::make_shared(*m_dataset, objectName, handler); - req->run(objectName, req, m_threadPool); + std::string toRead = objectName.empty() ? m_filename : objectName; + auto req = std::make_shared(*m_dataset, toRead, handler); + req->run(toRead, req, m_threadPool); return req; } - std::shared_ptr IOManagerDms::ReadObject(const std::string& requestName, std::shared_ptr handler, const IORange& range) + std::shared_ptr IOManagerDms::ReadObject(const std::string& objectName, std::shared_ptr handler, const IORange& range) { - auto req = std::make_shared(*m_dataset, requestName, handler); - req->run(requestName, range, req, m_threadPool); + std::string toRead = objectName.empty() ? m_filename : objectName; + auto req = std::make_shared(*m_dataset, toRead, handler); + req->run(toRead, range, req, m_threadPool); return req; } diff --git a/src/OpenVDS/IO/IOManagerDms.h b/src/OpenVDS/IO/IOManagerDms.h index bce5665b..4af77bc4 100644 --- a/src/OpenVDS/IO/IOManagerDms.h +++ b/src/OpenVDS/IO/IOManagerDms.h @@ -77,6 +77,7 @@ namespace OpenVDS private: std::unique_ptr m_sdManager; std::unique_ptr m_dataset; + std::string m_filename; bool m_opened; ThreadPool m_threadPool; }; diff --git a/src/OpenVDS/IO/IOManagerGoogle.cpp b/src/OpenVDS/IO/IOManagerGoogle.cpp index afa8e163..58baf91c 100644 --- a/src/OpenVDS/IO/IOManagerGoogle.cpp +++ b/src/OpenVDS/IO/IOManagerGoogle.cpp @@ -258,7 +258,12 @@ namespace OpenVDS { //std::string url = fmt::format("{}/storage/v1/b/{}/o/{}?alt=media", GOOGLEAPIS, m_bucket, objectName); //I cant make this scheme work if (pathPrefix.size()) - return fmt::format("{}/{}/{}/{}", googleapi, bucket, pathPrefix, objectName); + { + if (objectName.size()) + return fmt::format("{}/{}/{}/{}", googleapi, bucket, pathPrefix, objectName); + else + return fmt::format("{}/{}/{}", googleapi, bucket, pathPrefix); + } return fmt::format("{}/{}/{}", googleapi, bucket, objectName); } diff --git a/src/OpenVDS/IO/IOManagerHttp.cpp b/src/OpenVDS/IO/IOManagerHttp.cpp index c953e733..6168dd99 100644 --- a/src/OpenVDS/IO/IOManagerHttp.cpp +++ b/src/OpenVDS/IO/IOManagerHttp.cpp @@ -43,8 +43,8 @@ namespace OpenVDS m_base = url; } - if (m_base.back() != '/') - m_base.push_back('/'); + if (m_base.back() == '/') + m_base.pop_back(); if (m_suffix.size()) { @@ -53,9 +53,18 @@ namespace OpenVDS } } + static std::string getUrl(const std::string& base, const std::string& objectName, const std::string &suffix) + { + if (objectName.empty()) + { + return base + suffix; + } + return base + "/" + objectName + suffix; + } + std::shared_ptr IOManagerHttp::ReadObjectInfo(const std::string& objectName, std::shared_ptr handler) { - std::string url = m_base + objectName + m_suffix; + std::string url = getUrl(m_base, objectName, m_suffix); std::shared_ptr request = std::make_shared(objectName, handler); std::vector headers; m_curlHandler.addDownloadRequest(request, url, headers, convertToISO8601, CurlDownloadHandler::HEADER); @@ -64,7 +73,7 @@ namespace OpenVDS std::shared_ptr IOManagerHttp::ReadObject(const std::string& objectName, std::shared_ptr handler, const IORange& range) { - std::string url = m_base + objectName + m_suffix; + std::string url = getUrl(m_base, objectName, m_suffix); std::shared_ptr request = std::make_shared(objectName, handler); std::vector headers; if (range.start != range.end) diff --git a/src/SEGYUtils/SEGYUtils/DataProvider.h b/src/SEGYUtils/SEGYUtils/DataProvider.h index eaa8dad4..66387797 100644 --- a/src/SEGYUtils/SEGYUtils/DataProvider.h +++ b/src/SEGYUtils/SEGYUtils/DataProvider.h @@ -72,15 +72,15 @@ struct DataProvider { } - DataProvider(OpenVDS::IOManager *ioManager, const std::string &objectName, OpenVDS::Error &error) + DataProvider(const std::string &url, OpenVDS::IOManager *ioManager, OpenVDS::Error &error) : m_file(nullptr) , m_ioManager(ioManager) - , m_objectName(objectName) + , m_url(url) { if (m_ioManager) { auto syncTransfer = std::make_shared(); - auto syncRequest = m_ioManager->ReadObjectInfo(objectName, syncTransfer); + auto syncRequest = m_ioManager->ReadObjectInfo("", syncTransfer); if (syncRequest->WaitForFinish(error)) { m_size = syncTransfer->size; @@ -99,7 +99,7 @@ struct DataProvider if (m_ioManager) { auto dataTransfer = std::make_shared(); - auto request = m_ioManager->ReadObject(m_objectName, dataTransfer, { offset, offset + length}); + auto request = m_ioManager->ReadObject("", dataTransfer, {offset, offset + length}); if (!request->WaitForFinish(error)) { return false; @@ -136,17 +136,17 @@ struct DataProvider error.code = -1; error.string = "Invalid dataprovider, no file nor ioManager provided"; - return 0; + return ""; } std::string FileOrObjectName() const { - return m_file ? m_file->FileName() : m_objectName; + return m_file ? m_file->FileName() : m_url; } std::unique_ptr m_file; std::unique_ptr m_ioManager; - const std::string m_objectName; + const std::string m_url; int64_t m_size = 0; std::string m_lastWriteTime; }; @@ -174,7 +174,7 @@ struct DataView { int64_t chunk_end = std::min(i + chunk_size, end); m_transfers.push_back(std::make_shared(i - pos)); - m_requests.push_back(dataProvider.m_ioManager->ReadObject(dataProvider.m_objectName, m_transfers.back(), { i, chunk_end - 1 })); + m_requests.push_back(dataProvider.m_ioManager->ReadObject("", m_transfers.back(), {i, chunk_end - 1})); } } else diff --git a/tools/SEGYImport/SEGYImport.cpp b/tools/SEGYImport/SEGYImport.cpp index f2ad87bd..c8a05374 100644 --- a/tools/SEGYImport/SEGYImport.cpp +++ b/tools/SEGYImport/SEGYImport.cpp @@ -84,7 +84,7 @@ inline char asciitolower(char in) { return in; } -DataProvider CreateDataProviderFromFile(const std::string &filename, OpenVDS::Error &error) +static DataProvider CreateDataProviderFromFile(const std::string &filename, OpenVDS::Error &error) { std::unique_ptr file(new OpenVDS::File()); if (!file->Open(filename, false, false, false, error)) @@ -92,31 +92,23 @@ DataProvider CreateDataProviderFromFile(const std::string &filename, OpenVDS::Er return DataProvider(file.release()); } -DataProvider CreateDataProviderFromOpenOptions(const std::string &url, const std::string &connectionString, const std::string &objectId, OpenVDS::Error &error) +static DataProvider CreateDataProviderFromOpenOptions(const std::string &url, const std::string &connectionString, OpenVDS::Error &error) { std::unique_ptr ioManager(OpenVDS::IOManager::CreateIOManager(url, connectionString, OpenVDS::IOManager::AccessPattern::ReadOnly, error)); if (error.code) - return DataProvider((OpenVDS::IOManager *)nullptr, "", error); - return DataProvider(ioManager.release(), objectId, error); + return DataProvider("", (OpenVDS::IOManager*)nullptr, error); + return DataProvider(url, ioManager.release(), error); } -DataProvider CreateDataProvider(const std::string& name, const std::string& connection, OpenVDS::Error& error) +DataProvider CreateDataProvider(const std::string& url, const std::string& connection, OpenVDS::Error& error) { - if (OpenVDS::IsSupportedProtocol(name)) + if (OpenVDS::IsSupportedProtocol(url)) { - std::string dirname; - std::string basename; - std::string parameters; - splitUrl(name, dirname, basename, parameters, error); - if (error.code) - return DataProvider(nullptr); - - std::string url = dirname + parameters; - return CreateDataProviderFromOpenOptions(url, connection, basename, error); + return CreateDataProviderFromOpenOptions(url, connection, error); } else { - return CreateDataProviderFromFile(name, error); + return CreateDataProviderFromFile(url, error); } return DataProvider(nullptr); } -- GitLab From aa8699390a155da1f9cbc0e5af99a824c6fabf46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Lind?= Date: Thu, 17 Jun 2021 10:52:57 +0200 Subject: [PATCH 8/8] Mark 2.1.1 release --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5b71e398..9958256c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,7 +5,7 @@ if(CCACHE_PROGRAM) set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE "${CCACHE_PROGRAM}") endif() -set(OpenVDSVersion 2.1.0) +set(OpenVDSVersion 2.1.1) project(OpenVDS VERSION ${OpenVDSVersion}) -- GitLab