From 75aeb3239db87cdaafa7e9634a82ada1a026869b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Lind?= Date: Mon, 21 Jun 2021 12:34:37 +0200 Subject: [PATCH] enable read objects from objectstore without having an object name This makes uri handling in SEGYImport easier and makes it possible for SEGYImport to read seagy from dms location. This is a behavior change in the IOManager api so all backends have been manually verified --- src/OpenVDS/IO/IOManagerAzurePresigned.cpp | 19 ++++++++++++----- src/OpenVDS/IO/IOManagerDms.cpp | 24 +++++++++++++++++----- src/OpenVDS/IO/IOManagerDms.h | 1 + src/OpenVDS/IO/IOManagerGoogle.cpp | 7 ++++++- src/OpenVDS/IO/IOManagerHttp.cpp | 17 +++++++++++---- src/SEGYUtils/SEGYUtils/DataProvider.h | 16 +++++++-------- tools/SEGYImport/SEGYImport.cpp | 24 ++++++++-------------- 7 files changed, 69 insertions(+), 39 deletions(-) diff --git a/src/OpenVDS/IO/IOManagerAzurePresigned.cpp b/src/OpenVDS/IO/IOManagerAzurePresigned.cpp index 7e5c0e09..58d8ca50 100644 --- a/src/OpenVDS/IO/IOManagerAzurePresigned.cpp +++ b/src/OpenVDS/IO/IOManagerAzurePresigned.cpp @@ -34,8 +34,8 @@ namespace OpenVDS return; } - if (m_base.back() != '/') - m_base.push_back('/'); + if (m_base.back() == '/') + m_base.pop_back(); if (m_suffix.size()) { @@ -44,9 +44,18 @@ namespace OpenVDS } } + static std::string getUrl(const std::string& base, const std::string& objectName, const std::string &suffix) + { + if (objectName.empty()) + { + return base + suffix; + } + return base + "/" + objectName + suffix; + } + std::shared_ptr IOManagerAzurePresigned::ReadObjectInfo(const std::string& objectName, std::shared_ptr handler) { - std::string url = m_base + objectName + m_suffix; + std::string url = getUrl(m_base, objectName, m_suffix); std::shared_ptr request = std::make_shared(objectName, handler); std::vector headers; m_curlHandler.addDownloadRequest(request, url, headers, convertToISO8601, CurlDownloadHandler::HEADER); @@ -55,7 +64,7 @@ namespace OpenVDS } std::shared_ptr IOManagerAzurePresigned::ReadObject(const std::string& objectName, std::shared_ptr handler, const IORange& range) { - std::string url = m_base + objectName + m_suffix; + std::string url = getUrl(m_base, objectName, m_suffix); std::shared_ptr request = std::make_shared(objectName, handler); std::vector headers; if (range.start != range.end) @@ -69,7 +78,7 @@ namespace OpenVDS } std::shared_ptr IOManagerAzurePresigned::WriteObject(const std::string& objectName, const std::string& contentDispostionFilename, const std::string& contentType, const std::vector>& metadataHeader, std::shared_ptr> data, std::function completedCallback) { - std::string url = m_base + objectName + m_suffix; + std::string url = getUrl(m_base, objectName, m_suffix); std::shared_ptr request = std::make_shared(objectName, completedCallback); std::vector headers; headers.emplace_back("x-ms-blob-type: BlockBlob"); diff --git a/src/OpenVDS/IO/IOManagerDms.cpp b/src/OpenVDS/IO/IOManagerDms.cpp index 440ca885..2917867a 100644 --- a/src/OpenVDS/IO/IOManagerDms.cpp +++ b/src/OpenVDS/IO/IOManagerDms.cpp @@ -183,6 +183,18 @@ namespace OpenVDS , m_opened(false) , m_threadPool(16) { + if (openOptions.datasetPath.size()) + { + auto it = openOptions.datasetPath.rfind('/'); + if (it == openOptions.datasetPath.size() - 1) + { + it = openOptions.datasetPath.rfind('/', 1); + } + if (it != std::string::npos) + { + m_filename = openOptions.datasetPath.substr(it+1); + } + } try { m_sdManager.reset(new seismicdrive::SDManager(openOptions.sdAuthorityUrl, openOptions.sdApiKey, openOptions.logLevel)); m_sdManager->setAuthProviderFromString(openOptions.sdToken); @@ -223,15 +235,17 @@ namespace OpenVDS std::shared_ptr IOManagerDms::ReadObjectInfo(const std::string& objectName, std::shared_ptr handler) { - auto req = std::make_shared(*m_dataset, objectName, handler); - req->run(objectName, req, m_threadPool); + std::string toRead = objectName.empty() ? m_filename : objectName; + auto req = std::make_shared(*m_dataset, toRead, handler); + req->run(toRead, req, m_threadPool); return req; } - std::shared_ptr IOManagerDms::ReadObject(const std::string& requestName, std::shared_ptr handler, const IORange& range) + std::shared_ptr IOManagerDms::ReadObject(const std::string& objectName, std::shared_ptr handler, const IORange& range) { - auto req = std::make_shared(*m_dataset, requestName, handler); - req->run(requestName, range, req, m_threadPool); + std::string toRead = objectName.empty() ? m_filename : objectName; + auto req = std::make_shared(*m_dataset, toRead, handler); + req->run(toRead, range, req, m_threadPool); return req; } diff --git a/src/OpenVDS/IO/IOManagerDms.h b/src/OpenVDS/IO/IOManagerDms.h index bce5665b..4af77bc4 100644 --- a/src/OpenVDS/IO/IOManagerDms.h +++ b/src/OpenVDS/IO/IOManagerDms.h @@ -77,6 +77,7 @@ namespace OpenVDS private: std::unique_ptr m_sdManager; std::unique_ptr m_dataset; + std::string m_filename; bool m_opened; ThreadPool m_threadPool; }; diff --git a/src/OpenVDS/IO/IOManagerGoogle.cpp b/src/OpenVDS/IO/IOManagerGoogle.cpp index afa8e163..58baf91c 100644 --- a/src/OpenVDS/IO/IOManagerGoogle.cpp +++ b/src/OpenVDS/IO/IOManagerGoogle.cpp @@ -258,7 +258,12 @@ namespace OpenVDS { //std::string url = fmt::format("{}/storage/v1/b/{}/o/{}?alt=media", GOOGLEAPIS, m_bucket, objectName); //I cant make this scheme work if (pathPrefix.size()) - return fmt::format("{}/{}/{}/{}", googleapi, bucket, pathPrefix, objectName); + { + if (objectName.size()) + return fmt::format("{}/{}/{}/{}", googleapi, bucket, pathPrefix, objectName); + else + return fmt::format("{}/{}/{}", googleapi, bucket, pathPrefix); + } return fmt::format("{}/{}/{}", googleapi, bucket, objectName); } diff --git a/src/OpenVDS/IO/IOManagerHttp.cpp b/src/OpenVDS/IO/IOManagerHttp.cpp index c953e733..6168dd99 100644 --- a/src/OpenVDS/IO/IOManagerHttp.cpp +++ b/src/OpenVDS/IO/IOManagerHttp.cpp @@ -43,8 +43,8 @@ namespace OpenVDS m_base = url; } - if (m_base.back() != '/') - m_base.push_back('/'); + if (m_base.back() == '/') + m_base.pop_back(); if (m_suffix.size()) { @@ -53,9 +53,18 @@ namespace OpenVDS } } + static std::string getUrl(const std::string& base, const std::string& objectName, const std::string &suffix) + { + if (objectName.empty()) + { + return base + suffix; + } + return base + "/" + objectName + suffix; + } + std::shared_ptr IOManagerHttp::ReadObjectInfo(const std::string& objectName, std::shared_ptr handler) { - std::string url = m_base + objectName + m_suffix; + std::string url = getUrl(m_base, objectName, m_suffix); std::shared_ptr request = std::make_shared(objectName, handler); std::vector headers; m_curlHandler.addDownloadRequest(request, url, headers, convertToISO8601, CurlDownloadHandler::HEADER); @@ -64,7 +73,7 @@ namespace OpenVDS std::shared_ptr IOManagerHttp::ReadObject(const std::string& objectName, std::shared_ptr handler, const IORange& range) { - std::string url = m_base + objectName + m_suffix; + std::string url = getUrl(m_base, objectName, m_suffix); std::shared_ptr request = std::make_shared(objectName, handler); std::vector headers; if (range.start != range.end) diff --git a/src/SEGYUtils/SEGYUtils/DataProvider.h b/src/SEGYUtils/SEGYUtils/DataProvider.h index eaa8dad4..66387797 100644 --- a/src/SEGYUtils/SEGYUtils/DataProvider.h +++ b/src/SEGYUtils/SEGYUtils/DataProvider.h @@ -72,15 +72,15 @@ struct DataProvider { } - DataProvider(OpenVDS::IOManager *ioManager, const std::string &objectName, OpenVDS::Error &error) + DataProvider(const std::string &url, OpenVDS::IOManager *ioManager, OpenVDS::Error &error) : m_file(nullptr) , m_ioManager(ioManager) - , m_objectName(objectName) + , m_url(url) { if (m_ioManager) { auto syncTransfer = std::make_shared(); - auto syncRequest = m_ioManager->ReadObjectInfo(objectName, syncTransfer); + auto syncRequest = m_ioManager->ReadObjectInfo("", syncTransfer); if (syncRequest->WaitForFinish(error)) { m_size = syncTransfer->size; @@ -99,7 +99,7 @@ struct DataProvider if (m_ioManager) { auto dataTransfer = std::make_shared(); - auto request = m_ioManager->ReadObject(m_objectName, dataTransfer, { offset, offset + length}); + auto request = m_ioManager->ReadObject("", dataTransfer, {offset, offset + length}); if (!request->WaitForFinish(error)) { return false; @@ -136,17 +136,17 @@ struct DataProvider error.code = -1; error.string = "Invalid dataprovider, no file nor ioManager provided"; - return 0; + return ""; } std::string FileOrObjectName() const { - return m_file ? m_file->FileName() : m_objectName; + return m_file ? m_file->FileName() : m_url; } std::unique_ptr m_file; std::unique_ptr m_ioManager; - const std::string m_objectName; + const std::string m_url; int64_t m_size = 0; std::string m_lastWriteTime; }; @@ -174,7 +174,7 @@ struct DataView { int64_t chunk_end = std::min(i + chunk_size, end); m_transfers.push_back(std::make_shared(i - pos)); - m_requests.push_back(dataProvider.m_ioManager->ReadObject(dataProvider.m_objectName, m_transfers.back(), { i, chunk_end - 1 })); + m_requests.push_back(dataProvider.m_ioManager->ReadObject("", m_transfers.back(), {i, chunk_end - 1})); } } else diff --git a/tools/SEGYImport/SEGYImport.cpp b/tools/SEGYImport/SEGYImport.cpp index f2ad87bd..c8a05374 100644 --- a/tools/SEGYImport/SEGYImport.cpp +++ b/tools/SEGYImport/SEGYImport.cpp @@ -84,7 +84,7 @@ inline char asciitolower(char in) { return in; } -DataProvider CreateDataProviderFromFile(const std::string &filename, OpenVDS::Error &error) +static DataProvider CreateDataProviderFromFile(const std::string &filename, OpenVDS::Error &error) { std::unique_ptr file(new OpenVDS::File()); if (!file->Open(filename, false, false, false, error)) @@ -92,31 +92,23 @@ DataProvider CreateDataProviderFromFile(const std::string &filename, OpenVDS::Er return DataProvider(file.release()); } -DataProvider CreateDataProviderFromOpenOptions(const std::string &url, const std::string &connectionString, const std::string &objectId, OpenVDS::Error &error) +static DataProvider CreateDataProviderFromOpenOptions(const std::string &url, const std::string &connectionString, OpenVDS::Error &error) { std::unique_ptr ioManager(OpenVDS::IOManager::CreateIOManager(url, connectionString, OpenVDS::IOManager::AccessPattern::ReadOnly, error)); if (error.code) - return DataProvider((OpenVDS::IOManager *)nullptr, "", error); - return DataProvider(ioManager.release(), objectId, error); + return DataProvider("", (OpenVDS::IOManager*)nullptr, error); + return DataProvider(url, ioManager.release(), error); } -DataProvider CreateDataProvider(const std::string& name, const std::string& connection, OpenVDS::Error& error) +DataProvider CreateDataProvider(const std::string& url, const std::string& connection, OpenVDS::Error& error) { - if (OpenVDS::IsSupportedProtocol(name)) + if (OpenVDS::IsSupportedProtocol(url)) { - std::string dirname; - std::string basename; - std::string parameters; - splitUrl(name, dirname, basename, parameters, error); - if (error.code) - return DataProvider(nullptr); - - std::string url = dirname + parameters; - return CreateDataProviderFromOpenOptions(url, connection, basename, error); + return CreateDataProviderFromOpenOptions(url, connection, error); } else { - return CreateDataProviderFromFile(name, error); + return CreateDataProviderFromFile(url, error); } return DataProvider(nullptr); } -- GitLab