diff --git a/3rdparty/.gitignore b/3rdparty/.gitignore index 6a93006923adf97d239d308cdf8a2de8cc452de6..e8d3639a6484fade6d7ec3d4eb53b35f0cf15d24 100644 --- a/3rdparty/.gitignore +++ b/3rdparty/.gitignore @@ -16,3 +16,5 @@ java* google_nlohmann* libressl* dms* +cmakerc* +!dms-git diff --git a/CMake/Fetch3rdPartyInBuild.cmake b/CMake/Fetch3rdPartyInBuild.cmake index 2d8b0ec7ede33f2f3fe0873c6a9f84a5db0f0d7a..3a79dc24f75cb311dbb0bf93f87580ac50f28072 100644 --- a/CMake/Fetch3rdPartyInBuild.cmake +++ b/CMake/Fetch3rdPartyInBuild.cmake @@ -63,6 +63,26 @@ macro(Fetch3rdParty_Package name version url url_hash) endif() endmacro() +macro(Fetch3rdParty_Git name version url tag) + if (OPENVDS_3RD_PARTY_DIR) + set(Fetch3rdPartyDir "${OPENVDS_3RD_PARTY_DIR}") + else() + set(Fetch3rdPartyDir "${Fetch3rdPartyDirInternal}/../3rdparty") + endif() + get_filename_component(thirdParty "${Fetch3rdPartyDir}" ABSOLUTE) + set(SRC_DIR ${thirdParty}/${name}-${version}) + set(${name}_SOURCE_DIR ${SRC_DIR} PARENT_SCOPE) + set(${name}_VERSION ${version} PARENT_SCOPE) + if (NOT (EXISTS ${SRC_DIR})) + FetchContent_Populate(${name} + GIT_REPOSITORY ${url} + GIT_TAG ${url_hash} + SOURCE_DIR ${SRC_DIR} + SUBBUILD_DIR ${thirdParty}/CMakeArtifacts/${name}-sub-${version} + BINARY_DIR ${thirdParty}/CMakeArtifacts/${name}-${version}) + endif() +endmacro() + function(Fetch3rdParty) include(CMake/FetchContentLocal.cmake) @@ -82,7 +102,8 @@ function(Fetch3rdParty) Fetch3rdParty_Package(absl 20200225.2 https://codeload.github.com/abseil/abseil-cpp/tar.gz/20200225.2 SHA256=f41868f7a938605c92936230081175d1eae87f6ea2c248f41077c8f88316f111) Fetch3rdParty_Package(crc32c 1.1.1 https://codeload.github.com/google/crc32c/tar.gz/1.1.1 SHA256=a6533f45b1670b5d59b38a514d82b09c6fb70cc1050467220216335e873074e8) Fetch3rdParty_Package(google-cloud-cpp 1.14.0 https://codeload.github.com/googleapis/google-cloud-cpp/tar.gz/v1.14.0 SHA256=839b2d4dcb36a671734dac6b30ea8c298bbeaafcf7a45ee4a7d7aa5986b16569) - Fetch3rdParty_Package(dms 18a85b75c7 https://community.opengroup.org/osdu/platform/domain-data-mgmt-services/seismic/seismic-dms-suite/seismic-store-cpp-lib/-/archive/18a85b75c7c10ce1bb8788d23c79964e060cee64/seismic-store-cpp-lib-master.tar.gz SHA256=146ed18ee82277355273588b6c4c834605a86a1ec40d48c5c9c11cbd47ec0165) + Fetch3rdParty_Package(dms 172dcf6a3 https://community.opengroup.org/osdu/platform/domain-data-mgmt-services/seismic/seismic-dms-suite/seismic-store-cpp-lib/-/archive/172dcf6a3b64a4f288a11c1d0bffb0dce31b29e8/seismic-store-cpp-lib-master.tar.gz SHA256=510481a8e7ed1403290d987d949db0c392d60e9776a6d9f2b8a7e225d071f2e3) + #Fetch3rdParty_Git(dms git git@community.opengroup.org:osdu/platform/domain-data-mgmt-services/seismic/seismic-dms-suite/seismic-store-cpp-lib.git master) Fetch3rdParty_File(testng java 6.14.3 jar https://repo1.maven.org/maven2/org/testng/testng/6.14.3/testng-6.14.3.jar MD5=9f17a8f9e99165e148c42b21f4b63d7c) Fetch3rdParty_File(jcommander java 1.72 jar https://repo1.maven.org/maven2/com/beust/jcommander/1.72/jcommander-1.72.jar MD5=9fde6bc0ba1032eceb7267fd1ad1657b) Fetch3rdParty_FileTarget(google_nlohmann google/cloud/storage/internal nlohmann_json.hpp 3.4.0 https://raw.githubusercontent.com/nlohmann/json/v3.4.0/single_include/nlohmann/json.hpp MD5=27f3760c1d3a0fff7d8a2407d8db8f9d) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4e0597fd273b889c6375192e925cbcfd24dcf1b3..9958256ca3a083819d206245f31be13367e07585 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,7 +5,7 @@ if(CCACHE_PROGRAM) set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE "${CCACHE_PROGRAM}") endif() -set(OpenVDSVersion 2.1.0) +set(OpenVDSVersion 2.1.1) project(OpenVDS VERSION ${OpenVDSVersion}) @@ -34,6 +34,12 @@ else() set(LIBSUFFIX "") endif() +if (EXISTS /etc/alpine-release) #the Alpine folks desided to patch cmake but forgot to patch the FIND_LIBRARY_USE_LIB64_PATHS + if (NOT ${CMAKE_INSTALL_LIBDIR} STREQUAL "lib64") + set(LIBSUFFIX "") + endif() +endif() + set(BUILD_PYTHON_DEFAULT ON) if (Python_ROOT_DIR) set(PYTHON_ROOT_DIR_HINT "HINTS ${Python_ROOT_DIR}") @@ -141,7 +147,10 @@ set(LIB_TOOLSET_DIR "${LIB_TOOLSET_DIR}$<$>:/$>") set_property(GLOBAL PROPERTY USE_FOLDERS ON) find_package(Threads) if (ENABLE_OPENMP) - find_package(OpenMP) + find_package(OpenMP) +endif() +if (OpenMP_CXX_FOUND) + set(CMAKE_INSTALL_OPENMP_LIBRARIES TRUE) endif() include(CMake/SetWarnings.cmake) diff --git a/src/OpenVDS/IO/IOManagerAWS.cpp b/src/OpenVDS/IO/IOManagerAWS.cpp index 5af16bcea8bc804d8f36fddc7a31a21ae588a0f2..4139ba63c4a345d3dd5e4daae1210b81a6ad0cda 100644 --- a/src/OpenVDS/IO/IOManagerAWS.cpp +++ b/src/OpenVDS/IO/IOManagerAWS.cpp @@ -36,6 +36,7 @@ #include #include #include +#include #include #include @@ -442,6 +443,13 @@ namespace OpenVDS { m_s3Client.reset(new Aws::S3::S3Client(credentials, clientConfig, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, useVirtualAddressing)); } + + //We do this to use a symbol from the transfermanager so we get the linker chain working on linux + Aws::Utils::Threading::DefaultExecutor threadExecutor; + Aws::Transfer::TransferManagerConfiguration transferConfig(&threadExecutor); + std::shared_ptr s3ClientSharedPtr(m_s3Client.get(), [](Aws::S3::S3Client*) {}); + transferConfig.s3Client = s3ClientSharedPtr; + auto manager = Aws::Transfer::TransferManager::Create(transferConfig); } IOManagerAWS::~IOManagerAWS() diff --git a/src/OpenVDS/IO/IOManagerAzurePresigned.cpp b/src/OpenVDS/IO/IOManagerAzurePresigned.cpp index 7e5c0e092d3599861f6728df22cd77362dd80f1d..58d8ca505d5e75573352c11af10d321732c23778 100644 --- a/src/OpenVDS/IO/IOManagerAzurePresigned.cpp +++ b/src/OpenVDS/IO/IOManagerAzurePresigned.cpp @@ -34,8 +34,8 @@ namespace OpenVDS return; } - if (m_base.back() != '/') - m_base.push_back('/'); + if (m_base.back() == '/') + m_base.pop_back(); if (m_suffix.size()) { @@ -44,9 +44,18 @@ namespace OpenVDS } } + static std::string getUrl(const std::string& base, const std::string& objectName, const std::string &suffix) + { + if (objectName.empty()) + { + return base + suffix; + } + return base + "/" + objectName + suffix; + } + std::shared_ptr IOManagerAzurePresigned::ReadObjectInfo(const std::string& objectName, std::shared_ptr handler) { - std::string url = m_base + objectName + m_suffix; + std::string url = getUrl(m_base, objectName, m_suffix); std::shared_ptr request = std::make_shared(objectName, handler); std::vector headers; m_curlHandler.addDownloadRequest(request, url, headers, convertToISO8601, CurlDownloadHandler::HEADER); @@ -55,7 +64,7 @@ namespace OpenVDS } std::shared_ptr IOManagerAzurePresigned::ReadObject(const std::string& objectName, std::shared_ptr handler, const IORange& range) { - std::string url = m_base + objectName + m_suffix; + std::string url = getUrl(m_base, objectName, m_suffix); std::shared_ptr request = std::make_shared(objectName, handler); std::vector headers; if (range.start != range.end) @@ -69,7 +78,7 @@ namespace OpenVDS } std::shared_ptr IOManagerAzurePresigned::WriteObject(const std::string& objectName, const std::string& contentDispostionFilename, const std::string& contentType, const std::vector>& metadataHeader, std::shared_ptr> data, std::function completedCallback) { - std::string url = m_base + objectName + m_suffix; + std::string url = getUrl(m_base, objectName, m_suffix); std::shared_ptr request = std::make_shared(objectName, completedCallback); std::vector headers; headers.emplace_back("x-ms-blob-type: BlockBlob"); diff --git a/src/OpenVDS/IO/IOManagerDms.cpp b/src/OpenVDS/IO/IOManagerDms.cpp index ba3972187c039c67b53bbeaba8d949ea62e3fe1f..2917867a33719f5e0137b1b7ecb86fb46803eb4d 100644 --- a/src/OpenVDS/IO/IOManagerDms.cpp +++ b/src/OpenVDS/IO/IOManagerDms.cpp @@ -35,74 +35,105 @@ namespace OpenVDS } template - static void run_request(const std::string& requestName, std::weak_ptr request, const IORange &range, std::vector* data) + static void run_request(const std::string& requestName, std::weak_ptr weak_request, const IORange &range, std::vector* data) { - auto request_ptr = request.lock(); - if (!request_ptr) - return; + } - RequestStateHandler requestHandler(*request_ptr); - - if (requestHandler.isCancelledRequested()) - { - return; - } + void ReadObjectInfoRequestDms::run(const std::string& requestName, std::weak_ptr weak_request, ThreadPool &threadPool) + { + m_job = threadPool.Enqueue([requestName, weak_request]() { + auto request = weak_request.lock(); + if (!request) + return; - uint64_t size; - std::string created_date; - try - { - size = request_ptr->m_dataset.getBlockSize(requestName); - created_date = request_ptr->m_dataset.getCreatedDate(); + RequestStateHandler requestHandler(*request); - if (data && size) + if (requestHandler.isCancelledRequested()) + { + return; + } + + uint64_t size; + std::string created_date; + try + { + size = request->m_dataset.getBlockSize(requestName); + created_date = request->m_dataset.getCreatedDate(); + } + catch (const seismicdrive::SDException& ex) + { + request->m_error.code = -1; + request->m_error.string = ex.what(); + } + catch (...) + { + request->m_error.code = -1; + request->m_error.string = "Unknown exception in DMS upload"; + } + + if (request->m_error.code == 0) + { + request->m_handler->HandleObjectSize(size); + request->m_handler->HandleObjectLastWriteTime(created_date); + } + request->m_handler->Completed(*request, request->m_error); + }); + } + void DownloadRequestDms::run(const std::string& requestName, const IORange& range, std::weak_ptr weak_request, ThreadPool &threadPool) + { + m_job = threadPool.Enqueue([requestName, weak_request, range]() { + auto request = weak_request.lock(); + if (!request) + return; + + RequestStateHandler requestHandler(*request); + + if (requestHandler.isCancelledRequested()) + { + return; + } + + std::vector data; + try { - uint64_t offset; if (range.end) { - data->resize(range.end - range.start); - offset = range.start; + data.resize(range.end - range.start); + request->m_dataset.readBlock(requestName, (char*)data.data(), range.start, data.size()); } else { - offset = 0; - data->resize(size); + //char* read_data = nullptr; + //std::size_t read_size = 0; + //request->m_dataset.readBlock(requestName, &read_data, read_size); + //data.resize(read_size); + //memcpy(data.data(), read_data, data.size()); + //delete[] read_data; + size_t read_size = size_t(request->m_dataset.getBlockSize(requestName)); + if (read_size) + { + data.resize(read_size); + request->m_dataset.readBlock(requestName, (char*)data.data(), read_size); + } } - request_ptr->m_dataset.readBlock(requestName, (char*)data->data(), offset, data->size()); } - } - catch (const seismicdrive::SDException& ex) - { - request_ptr->m_error.code = -1; - request_ptr->m_error.string = ex.what(); - } - catch (...) - { - request_ptr->m_error.code = -1; - request_ptr->m_error.string = "Unknown exception in DMS upload"; - } - - if (request_ptr->m_error.code == 0) - { - request_ptr->m_handler->HandleObjectSize(size); - request_ptr->m_handler->HandleObjectLastWriteTime(created_date); - if (data) - request_ptr->m_handler->HandleData(std::move(*data)); - } - request_ptr->m_handler->Completed(*request_ptr, request_ptr->m_error); - } + catch (const seismicdrive::SDException& ex) + { + request->m_error.code = -1; + request->m_error.string = ex.what(); + } + catch (...) + { + request->m_error.code = -1; + request->m_error.string = "Unknown exception in DMS upload"; + } - void ReadObjectInfoRequestDms::run(const std::string& requestName, std::weak_ptr request, ThreadPool &threadPool) - { - m_job = threadPool.Enqueue([requestName, request]() { - run_request(requestName, request, IORange(), nullptr); - }); - } - void DownloadRequestDms::run(const std::string& requestName, const IORange& range, std::weak_ptr request, ThreadPool &threadPool) - { - m_job = threadPool.Enqueue([requestName, request, range]() { - std::vector data; - run_request(requestName, request, range, &data); + if (request->m_error.code == 0) + { + request->m_handler->HandleObjectSize(data.size()); + request->m_handler->HandleData(std::move(data)); + } + request->m_handler->Completed(*request, request->m_error); }); } @@ -128,7 +159,8 @@ namespace OpenVDS try { - request_ptr->m_dataset.writeBlock(requestName, (const char*)data->data(), data->size(), false); + if (data->size()) + request_ptr->m_dataset.writeBlock(requestName, (const char*)data->data(), data->size(), false); } catch (const seismicdrive::SDException& ex) { @@ -151,6 +183,18 @@ namespace OpenVDS , m_opened(false) , m_threadPool(16) { + if (openOptions.datasetPath.size()) + { + auto it = openOptions.datasetPath.rfind('/'); + if (it == openOptions.datasetPath.size() - 1) + { + it = openOptions.datasetPath.rfind('/', 1); + } + if (it != std::string::npos) + { + m_filename = openOptions.datasetPath.substr(it+1); + } + } try { m_sdManager.reset(new seismicdrive::SDManager(openOptions.sdAuthorityUrl, openOptions.sdApiKey, openOptions.logLevel)); m_sdManager->setAuthProviderFromString(openOptions.sdToken); @@ -191,15 +235,17 @@ namespace OpenVDS std::shared_ptr IOManagerDms::ReadObjectInfo(const std::string& objectName, std::shared_ptr handler) { - auto req = std::make_shared(*m_dataset, objectName, handler); - req->run(objectName, req, m_threadPool); + std::string toRead = objectName.empty() ? m_filename : objectName; + auto req = std::make_shared(*m_dataset, toRead, handler); + req->run(toRead, req, m_threadPool); return req; } - std::shared_ptr IOManagerDms::ReadObject(const std::string& requestName, std::shared_ptr handler, const IORange& range) + std::shared_ptr IOManagerDms::ReadObject(const std::string& objectName, std::shared_ptr handler, const IORange& range) { - auto req = std::make_shared(*m_dataset, requestName, handler); - req->run(requestName, range, req, m_threadPool); + std::string toRead = objectName.empty() ? m_filename : objectName; + auto req = std::make_shared(*m_dataset, toRead, handler); + req->run(toRead, range, req, m_threadPool); return req; } diff --git a/src/OpenVDS/IO/IOManagerDms.h b/src/OpenVDS/IO/IOManagerDms.h index bce5665bd14db91785a9e622ce3d113405e4c37c..4af77bc4a352f3b43441a9b801949191865dccdf 100644 --- a/src/OpenVDS/IO/IOManagerDms.h +++ b/src/OpenVDS/IO/IOManagerDms.h @@ -77,6 +77,7 @@ namespace OpenVDS private: std::unique_ptr m_sdManager; std::unique_ptr m_dataset; + std::string m_filename; bool m_opened; ThreadPool m_threadPool; }; diff --git a/src/OpenVDS/IO/IOManagerGoogle.cpp b/src/OpenVDS/IO/IOManagerGoogle.cpp index afa8e163e4bee3193d47f3c985e6a421e94df386..58baf91c453f904725478eaf07836aedf32a173a 100644 --- a/src/OpenVDS/IO/IOManagerGoogle.cpp +++ b/src/OpenVDS/IO/IOManagerGoogle.cpp @@ -258,7 +258,12 @@ namespace OpenVDS { //std::string url = fmt::format("{}/storage/v1/b/{}/o/{}?alt=media", GOOGLEAPIS, m_bucket, objectName); //I cant make this scheme work if (pathPrefix.size()) - return fmt::format("{}/{}/{}/{}", googleapi, bucket, pathPrefix, objectName); + { + if (objectName.size()) + return fmt::format("{}/{}/{}/{}", googleapi, bucket, pathPrefix, objectName); + else + return fmt::format("{}/{}/{}", googleapi, bucket, pathPrefix); + } return fmt::format("{}/{}/{}", googleapi, bucket, objectName); } diff --git a/src/OpenVDS/IO/IOManagerHttp.cpp b/src/OpenVDS/IO/IOManagerHttp.cpp index c953e733fe3915ed1ae00042f088f35f9720a076..6168dd99afa4429cf65cd7aa565b65eaa7f04579 100644 --- a/src/OpenVDS/IO/IOManagerHttp.cpp +++ b/src/OpenVDS/IO/IOManagerHttp.cpp @@ -43,8 +43,8 @@ namespace OpenVDS m_base = url; } - if (m_base.back() != '/') - m_base.push_back('/'); + if (m_base.back() == '/') + m_base.pop_back(); if (m_suffix.size()) { @@ -53,9 +53,18 @@ namespace OpenVDS } } + static std::string getUrl(const std::string& base, const std::string& objectName, const std::string &suffix) + { + if (objectName.empty()) + { + return base + suffix; + } + return base + "/" + objectName + suffix; + } + std::shared_ptr IOManagerHttp::ReadObjectInfo(const std::string& objectName, std::shared_ptr handler) { - std::string url = m_base + objectName + m_suffix; + std::string url = getUrl(m_base, objectName, m_suffix); std::shared_ptr request = std::make_shared(objectName, handler); std::vector headers; m_curlHandler.addDownloadRequest(request, url, headers, convertToISO8601, CurlDownloadHandler::HEADER); @@ -64,7 +73,7 @@ namespace OpenVDS std::shared_ptr IOManagerHttp::ReadObject(const std::string& objectName, std::shared_ptr handler, const IORange& range) { - std::string url = m_base + objectName + m_suffix; + std::string url = getUrl(m_base, objectName, m_suffix); std::shared_ptr request = std::make_shared(objectName, handler); std::vector headers; if (range.start != range.end) diff --git a/src/OpenVDS/IO/IOManagerRequestImpl.h b/src/OpenVDS/IO/IOManagerRequestImpl.h index 53a4128de2790b122ee7e97b1e74f9510514e999..686cf0e7d740ef2fbc1fd15b7a7a04766f5a1172 100644 --- a/src/OpenVDS/IO/IOManagerRequestImpl.h +++ b/src/OpenVDS/IO/IOManagerRequestImpl.h @@ -46,7 +46,7 @@ public: return error.code == 0; } - void Cancel() + void Cancel() override { m_cancelledRequested = true; } diff --git a/src/OpenVDS/IO/IOManagerTransformer.h b/src/OpenVDS/IO/IOManagerTransformer.h index e30f87f4ca566d3386ad73278ccad265e2b9723b..a75d054020ca9c2a7d403380630337406dd57e06 100644 --- a/src/OpenVDS/IO/IOManagerTransformer.h +++ b/src/OpenVDS/IO/IOManagerTransformer.h @@ -16,7 +16,7 @@ ****************************************************************************/ #ifndef IOMANAGERTRANSFORMER_H -#define IOMANAGERTRANSFORMAT_H +#define IOMANAGERTRANSFORMER_H #include diff --git a/src/OpenVDS/VDS/Env.cpp b/src/OpenVDS/VDS/Env.cpp index 0618b0cede913d45c894bbd2672096cd4caef117..b3d29c9c5a69a55b7eb2911ba056ca1c7ee28826 100644 --- a/src/OpenVDS/VDS/Env.cpp +++ b/src/OpenVDS/VDS/Env.cpp @@ -3,21 +3,30 @@ #include #include +#include +#include namespace OpenVDS { + + inline char asciitolower(char in) { + if (in <= 'Z' && in >= 'A') + return in - ('Z' - 'z'); + return in; +} + bool getBooleanEnvironmentVariable(const char *name) { - const char *var = getenv(name); - if (!var) - return false; - if (strcmp(var, "") == 0) + const char *c_var = getenv(name); + if (c_var == nullptr) return false; - if (strcmp(var, "false") == 0) + std::string var(c_var); + std::transform(var.begin(), var.end(), var.begin(), asciitolower); + if (var == "false") return false; - if (strcmp(var, "FALSE") == 0) + if (var == "0") return false; - if (strcmp(var, "0") == 0) + if (var == "off") return false; return true; } diff --git a/src/OpenVDS/VDS/VolumeDataStoreIOManager.cpp b/src/OpenVDS/VDS/VolumeDataStoreIOManager.cpp index 2ba4ff6c5fa3a1ab51ddff1152177ba339c003a8..1f16894ffbef451899a11cb2abd3a558add6c6b9 100644 --- a/src/OpenVDS/VDS/VolumeDataStoreIOManager.cpp +++ b/src/OpenVDS/VDS/VolumeDataStoreIOManager.cpp @@ -27,6 +27,8 @@ #include "WaveletTypes.h" +#include "Env.h" + #include #include @@ -179,7 +181,7 @@ VolumeDataStoreIOManager:: VolumeDataStoreIOManager(VDS &vds, IOManager *ioManag : VolumeDataStore(ioManager->connectionType()) , m_vds(vds) , m_ioManager(ioManager) - , m_warnedAboutMissingMetadataTag(false) + , m_warnedAboutMissingMetadataTag(getBooleanEnvironmentVariable("OPENVDS_DISABLE_WARNINGS")) { } @@ -506,8 +508,13 @@ bool VolumeDataStoreIOManager::ReadChunk(const VolumeDataChunk &chunk, int adapt { if (!m_warnedAboutMissingMetadataTag) // Log once and move along. { - fmt::print(stderr, "Dataset has missing metadata tags, degraded data verification, reverting to metadata pages"); - m_warnedAboutMissingMetadataTag = true; + lock.lock(); + if (!m_warnedAboutMissingMetadataTag) + { + fmt::print(stderr, "Dataset has missing metadata tags, degraded data verification, reverting to metadata pages\n"); + m_warnedAboutMissingMetadataTag = true; + } + lock.unlock(); } if (moveData) diff --git a/src/SEGYUtils/SEGYUtils/DataProvider.h b/src/SEGYUtils/SEGYUtils/DataProvider.h index eaa8dad42ef0e91d65be9e15b750ad1dd9a2b7de..6638779791d8c4bbd52f8cffe050260eb71ab08e 100644 --- a/src/SEGYUtils/SEGYUtils/DataProvider.h +++ b/src/SEGYUtils/SEGYUtils/DataProvider.h @@ -72,15 +72,15 @@ struct DataProvider { } - DataProvider(OpenVDS::IOManager *ioManager, const std::string &objectName, OpenVDS::Error &error) + DataProvider(const std::string &url, OpenVDS::IOManager *ioManager, OpenVDS::Error &error) : m_file(nullptr) , m_ioManager(ioManager) - , m_objectName(objectName) + , m_url(url) { if (m_ioManager) { auto syncTransfer = std::make_shared(); - auto syncRequest = m_ioManager->ReadObjectInfo(objectName, syncTransfer); + auto syncRequest = m_ioManager->ReadObjectInfo("", syncTransfer); if (syncRequest->WaitForFinish(error)) { m_size = syncTransfer->size; @@ -99,7 +99,7 @@ struct DataProvider if (m_ioManager) { auto dataTransfer = std::make_shared(); - auto request = m_ioManager->ReadObject(m_objectName, dataTransfer, { offset, offset + length}); + auto request = m_ioManager->ReadObject("", dataTransfer, {offset, offset + length}); if (!request->WaitForFinish(error)) { return false; @@ -136,17 +136,17 @@ struct DataProvider error.code = -1; error.string = "Invalid dataprovider, no file nor ioManager provided"; - return 0; + return ""; } std::string FileOrObjectName() const { - return m_file ? m_file->FileName() : m_objectName; + return m_file ? m_file->FileName() : m_url; } std::unique_ptr m_file; std::unique_ptr m_ioManager; - const std::string m_objectName; + const std::string m_url; int64_t m_size = 0; std::string m_lastWriteTime; }; @@ -174,7 +174,7 @@ struct DataView { int64_t chunk_end = std::min(i + chunk_size, end); m_transfers.push_back(std::make_shared(i - pos)); - m_requests.push_back(dataProvider.m_ioManager->ReadObject(dataProvider.m_objectName, m_transfers.back(), { i, chunk_end - 1 })); + m_requests.push_back(dataProvider.m_ioManager->ReadObject("", m_transfers.back(), {i, chunk_end - 1})); } } else diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index e57033ae6899eb2847c4529fa9e1d94c503dc593..298c4a749b548d4f0a3636b556c722eb5766577e 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -51,6 +51,13 @@ function(add_test_executable name) if (OpenMP_CXX_FOUND) target_link_libraries(${name} PRIVATE OpenMP::OpenMP_CXX) endif() + if (DISABLE_AWS_IOMANAGER) + target_compile_definitions(${name} PRIVATE OPENVDS_NO_AWS_IOMANAGER) + endif() + if (DISABLE_AZURESDKFORCPP_IOMANAGER) + target_compile_definitions(${name} PRIVATE OPENVDS_NO_AZURE_SDK_FOR_CPP_IOMANAGER) + endif() + target_sources(${name} PRIVATE "LinuxLinkerChainHack.cpp") endfunction() add_test_executable(io_tests diff --git a/tests/LinuxLinkerChainHack.cpp b/tests/LinuxLinkerChainHack.cpp new file mode 100644 index 0000000000000000000000000000000000000000..80924e1a51e30c59b3eb0e2895ac216317051046 --- /dev/null +++ b/tests/LinuxLinkerChainHack.cpp @@ -0,0 +1,17 @@ +#ifndef OPENVDS_NO_AWS_IOMANAGER +#include +#include + +void addSomeSymbolsToTheExecutableSoTheLinkerChainWorkAws() +{ + //We do this to use a symbol from the transfermanager so we get the linker chain working on linux + Aws::Auth::AWSCredentials credentials; + Aws::S3::S3Client client(credentials); + Aws::Utils::Threading::DefaultExecutor threadExecutor; + Aws::Transfer::TransferManagerConfiguration transferConfig(&threadExecutor); + std::shared_ptr s3ClientSharedPtr(&client, [](Aws::S3::S3Client*) {}); + transferConfig.s3Client = s3ClientSharedPtr; + auto manager = Aws::Transfer::TransferManager::Create(transferConfig); +} +#endif + diff --git a/tools/SEGYImport/SEGYImport.cpp b/tools/SEGYImport/SEGYImport.cpp index f2ad87bd392d11b67bac2d77cc322ccf76277cae..c8a053742989297c1773d382261d2db647b46809 100644 --- a/tools/SEGYImport/SEGYImport.cpp +++ b/tools/SEGYImport/SEGYImport.cpp @@ -84,7 +84,7 @@ inline char asciitolower(char in) { return in; } -DataProvider CreateDataProviderFromFile(const std::string &filename, OpenVDS::Error &error) +static DataProvider CreateDataProviderFromFile(const std::string &filename, OpenVDS::Error &error) { std::unique_ptr file(new OpenVDS::File()); if (!file->Open(filename, false, false, false, error)) @@ -92,31 +92,23 @@ DataProvider CreateDataProviderFromFile(const std::string &filename, OpenVDS::Er return DataProvider(file.release()); } -DataProvider CreateDataProviderFromOpenOptions(const std::string &url, const std::string &connectionString, const std::string &objectId, OpenVDS::Error &error) +static DataProvider CreateDataProviderFromOpenOptions(const std::string &url, const std::string &connectionString, OpenVDS::Error &error) { std::unique_ptr ioManager(OpenVDS::IOManager::CreateIOManager(url, connectionString, OpenVDS::IOManager::AccessPattern::ReadOnly, error)); if (error.code) - return DataProvider((OpenVDS::IOManager *)nullptr, "", error); - return DataProvider(ioManager.release(), objectId, error); + return DataProvider("", (OpenVDS::IOManager*)nullptr, error); + return DataProvider(url, ioManager.release(), error); } -DataProvider CreateDataProvider(const std::string& name, const std::string& connection, OpenVDS::Error& error) +DataProvider CreateDataProvider(const std::string& url, const std::string& connection, OpenVDS::Error& error) { - if (OpenVDS::IsSupportedProtocol(name)) + if (OpenVDS::IsSupportedProtocol(url)) { - std::string dirname; - std::string basename; - std::string parameters; - splitUrl(name, dirname, basename, parameters, error); - if (error.code) - return DataProvider(nullptr); - - std::string url = dirname + parameters; - return CreateDataProviderFromOpenOptions(url, connection, basename, error); + return CreateDataProviderFromOpenOptions(url, connection, error); } else { - return CreateDataProviderFromFile(name, error); + return CreateDataProviderFromFile(url, error); } return DataProvider(nullptr); }