Commit 1a32416b authored by Jørgen Lind's avatar Jørgen Lind
Browse files

Merge branch feature/jorgen.lind/IOManageReadWithNoName with refs/heads/master...

Merge branch feature/jorgen.lind/IOManageReadWithNoName with refs/heads/master into refs/merge-requests/426/train
parents 16635261 75aeb323
Pipeline #47100 passed with stages
in 14 minutes and 17 seconds
......@@ -34,8 +34,8 @@ namespace OpenVDS
return;
}
if (m_base.back() != '/')
m_base.push_back('/');
if (m_base.back() == '/')
m_base.pop_back();
if (m_suffix.size())
{
......@@ -44,9 +44,18 @@ namespace OpenVDS
}
}
static std::string getUrl(const std::string& base, const std::string& objectName, const std::string &suffix)
{
if (objectName.empty())
{
return base + suffix;
}
return base + "/" + objectName + suffix;
}
std::shared_ptr<Request> IOManagerAzurePresigned::ReadObjectInfo(const std::string& objectName, std::shared_ptr<TransferDownloadHandler> handler)
{
std::string url = m_base + objectName + m_suffix;
std::string url = getUrl(m_base, objectName, m_suffix);
std::shared_ptr<DownloadRequestCurl> request = std::make_shared<DownloadRequestCurl>(objectName, handler);
std::vector<std::string> headers;
m_curlHandler.addDownloadRequest(request, url, headers, convertToISO8601, CurlDownloadHandler::HEADER);
......@@ -55,7 +64,7 @@ namespace OpenVDS
}
std::shared_ptr<Request> IOManagerAzurePresigned::ReadObject(const std::string& objectName, std::shared_ptr<TransferDownloadHandler> handler, const IORange& range)
{
std::string url = m_base + objectName + m_suffix;
std::string url = getUrl(m_base, objectName, m_suffix);
std::shared_ptr<DownloadRequestCurl> request = std::make_shared<DownloadRequestCurl>(objectName, handler);
std::vector<std::string> headers;
if (range.start != range.end)
......@@ -69,7 +78,7 @@ namespace OpenVDS
}
std::shared_ptr<Request> IOManagerAzurePresigned::WriteObject(const std::string& objectName, const std::string& contentDispostionFilename, const std::string& contentType, const std::vector<std::pair<std::string, std::string>>& metadataHeader, std::shared_ptr<std::vector<uint8_t>> data, std::function<void(const Request& request, const Error& error)> completedCallback)
{
std::string url = m_base + objectName + m_suffix;
std::string url = getUrl(m_base, objectName, m_suffix);
std::shared_ptr<UploadRequestCurl> request = std::make_shared<UploadRequestCurl>(objectName, completedCallback);
std::vector<std::string> headers;
headers.emplace_back("x-ms-blob-type: BlockBlob");
......
......@@ -183,6 +183,18 @@ namespace OpenVDS
, m_opened(false)
, m_threadPool(16)
{
if (openOptions.datasetPath.size())
{
auto it = openOptions.datasetPath.rfind('/');
if (it == openOptions.datasetPath.size() - 1)
{
it = openOptions.datasetPath.rfind('/', 1);
}
if (it != std::string::npos)
{
m_filename = openOptions.datasetPath.substr(it+1);
}
}
try {
m_sdManager.reset(new seismicdrive::SDManager(openOptions.sdAuthorityUrl, openOptions.sdApiKey, openOptions.logLevel));
m_sdManager->setAuthProviderFromString(openOptions.sdToken);
......@@ -223,15 +235,17 @@ namespace OpenVDS
std::shared_ptr<Request> IOManagerDms::ReadObjectInfo(const std::string& objectName, std::shared_ptr<TransferDownloadHandler> handler)
{
auto req = std::make_shared<ReadObjectInfoRequestDms>(*m_dataset, objectName, handler);
req->run(objectName, req, m_threadPool);
std::string toRead = objectName.empty() ? m_filename : objectName;
auto req = std::make_shared<ReadObjectInfoRequestDms>(*m_dataset, toRead, handler);
req->run(toRead, req, m_threadPool);
return req;
}
std::shared_ptr<Request> IOManagerDms::ReadObject(const std::string& requestName, std::shared_ptr<TransferDownloadHandler> handler, const IORange& range)
std::shared_ptr<Request> IOManagerDms::ReadObject(const std::string& objectName, std::shared_ptr<TransferDownloadHandler> handler, const IORange& range)
{
auto req = std::make_shared<DownloadRequestDms>(*m_dataset, requestName, handler);
req->run(requestName, range, req, m_threadPool);
std::string toRead = objectName.empty() ? m_filename : objectName;
auto req = std::make_shared<DownloadRequestDms>(*m_dataset, toRead, handler);
req->run(toRead, range, req, m_threadPool);
return req;
}
......
......@@ -77,6 +77,7 @@ namespace OpenVDS
private:
std::unique_ptr<seismicdrive::SDManager> m_sdManager;
std::unique_ptr<seismicdrive::SDGenericDataset> m_dataset;
std::string m_filename;
bool m_opened;
ThreadPool m_threadPool;
};
......
......@@ -258,7 +258,12 @@ namespace OpenVDS
{
//std::string url = fmt::format("{}/storage/v1/b/{}/o/{}?alt=media", GOOGLEAPIS, m_bucket, objectName); //I cant make this scheme work
if (pathPrefix.size())
return fmt::format("{}/{}/{}/{}", googleapi, bucket, pathPrefix, objectName);
{
if (objectName.size())
return fmt::format("{}/{}/{}/{}", googleapi, bucket, pathPrefix, objectName);
else
return fmt::format("{}/{}/{}", googleapi, bucket, pathPrefix);
}
return fmt::format("{}/{}/{}", googleapi, bucket, objectName);
}
......
......@@ -43,8 +43,8 @@ namespace OpenVDS
m_base = url;
}
if (m_base.back() != '/')
m_base.push_back('/');
if (m_base.back() == '/')
m_base.pop_back();
if (m_suffix.size())
{
......@@ -53,9 +53,18 @@ namespace OpenVDS
}
}
static std::string getUrl(const std::string& base, const std::string& objectName, const std::string &suffix)
{
if (objectName.empty())
{
return base + suffix;
}
return base + "/" + objectName + suffix;
}
std::shared_ptr<Request> IOManagerHttp::ReadObjectInfo(const std::string& objectName, std::shared_ptr<TransferDownloadHandler> handler)
{
std::string url = m_base + objectName + m_suffix;
std::string url = getUrl(m_base, objectName, m_suffix);
std::shared_ptr<DownloadRequestCurl> request = std::make_shared<DownloadRequestCurl>(objectName, handler);
std::vector<std::string> headers;
m_curlHandler.addDownloadRequest(request, url, headers, convertToISO8601, CurlDownloadHandler::HEADER);
......@@ -64,7 +73,7 @@ namespace OpenVDS
std::shared_ptr<Request> IOManagerHttp::ReadObject(const std::string& objectName, std::shared_ptr<TransferDownloadHandler> handler, const IORange& range)
{
std::string url = m_base + objectName + m_suffix;
std::string url = getUrl(m_base, objectName, m_suffix);
std::shared_ptr<DownloadRequestCurl> request = std::make_shared<DownloadRequestCurl>(objectName, handler);
std::vector<std::string> headers;
if (range.start != range.end)
......
......@@ -72,15 +72,15 @@ struct DataProvider
{
}
DataProvider(OpenVDS::IOManager *ioManager, const std::string &objectName, OpenVDS::Error &error)
DataProvider(const std::string &url, OpenVDS::IOManager *ioManager, OpenVDS::Error &error)
: m_file(nullptr)
, m_ioManager(ioManager)
, m_objectName(objectName)
, m_url(url)
{
if (m_ioManager)
{
auto syncTransfer = std::make_shared<DataTransfer>();
auto syncRequest = m_ioManager->ReadObjectInfo(objectName, syncTransfer);
auto syncRequest = m_ioManager->ReadObjectInfo("", syncTransfer);
if (syncRequest->WaitForFinish(error))
{
m_size = syncTransfer->size;
......@@ -99,7 +99,7 @@ struct DataProvider
if (m_ioManager)
{
auto dataTransfer = std::make_shared<DataTransfer>();
auto request = m_ioManager->ReadObject(m_objectName, dataTransfer, { offset, offset + length});
auto request = m_ioManager->ReadObject("", dataTransfer, {offset, offset + length});
if (!request->WaitForFinish(error))
{
return false;
......@@ -136,17 +136,17 @@ struct DataProvider
error.code = -1;
error.string = "Invalid dataprovider, no file nor ioManager provided";
return 0;
return "";
}
std::string FileOrObjectName() const
{
return m_file ? m_file->FileName() : m_objectName;
return m_file ? m_file->FileName() : m_url;
}
std::unique_ptr<OpenVDS::File> m_file;
std::unique_ptr<OpenVDS::IOManager> m_ioManager;
const std::string m_objectName;
const std::string m_url;
int64_t m_size = 0;
std::string m_lastWriteTime;
};
......@@ -174,7 +174,7 @@ struct DataView
{
int64_t chunk_end = std::min(i + chunk_size, end);
m_transfers.push_back(std::make_shared<DataTransfer>(i - pos));
m_requests.push_back(dataProvider.m_ioManager->ReadObject(dataProvider.m_objectName, m_transfers.back(), { i, chunk_end - 1 }));
m_requests.push_back(dataProvider.m_ioManager->ReadObject("", m_transfers.back(), {i, chunk_end - 1}));
}
}
else
......
......@@ -84,7 +84,7 @@ inline char asciitolower(char in) {
return in;
}
DataProvider CreateDataProviderFromFile(const std::string &filename, OpenVDS::Error &error)
static DataProvider CreateDataProviderFromFile(const std::string &filename, OpenVDS::Error &error)
{
std::unique_ptr<OpenVDS::File> file(new OpenVDS::File());
if (!file->Open(filename, false, false, false, error))
......@@ -92,31 +92,23 @@ DataProvider CreateDataProviderFromFile(const std::string &filename, OpenVDS::Er
return DataProvider(file.release());
}
DataProvider CreateDataProviderFromOpenOptions(const std::string &url, const std::string &connectionString, const std::string &objectId, OpenVDS::Error &error)
static DataProvider CreateDataProviderFromOpenOptions(const std::string &url, const std::string &connectionString, OpenVDS::Error &error)
{
std::unique_ptr<OpenVDS::IOManager> ioManager(OpenVDS::IOManager::CreateIOManager(url, connectionString, OpenVDS::IOManager::AccessPattern::ReadOnly, error));
if (error.code)
return DataProvider((OpenVDS::IOManager *)nullptr, "", error);
return DataProvider(ioManager.release(), objectId, error);
return DataProvider("", (OpenVDS::IOManager*)nullptr, error);
return DataProvider(url, ioManager.release(), error);
}
DataProvider CreateDataProvider(const std::string& name, const std::string& connection, OpenVDS::Error& error)
DataProvider CreateDataProvider(const std::string& url, const std::string& connection, OpenVDS::Error& error)
{
if (OpenVDS::IsSupportedProtocol(name))
if (OpenVDS::IsSupportedProtocol(url))
{
std::string dirname;
std::string basename;
std::string parameters;
splitUrl(name, dirname, basename, parameters, error);
if (error.code)
return DataProvider(nullptr);
std::string url = dirname + parameters;
return CreateDataProviderFromOpenOptions(url, connection, basename, error);
return CreateDataProviderFromOpenOptions(url, connection, error);
}
else
{
return CreateDataProviderFromFile(name, error);
return CreateDataProviderFromFile(url, error);
}
return DataProvider(nullptr);
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment