Commit 10b39442 authored by Jørgen Lind's avatar Jørgen Lind
Browse files

SEGYImport add more source possibillities

Adding support for Azure source with connection string or using Azure
presign. Now its possible to do something like this:

SEGYImport.exe
        --azure-presign-base <BLOBSASURL>,
        --azure-presign-suffix <SASTOKEN>
        --azure-presign-source-base <SOMEOTHERBLOBSASURL>
        --azure-presign-source-suffix <SOMEOTHERSASTOKEN>
        NAME_OF_THE_SEG-Y_FILE
This will then read the SEG-Y file from the azure-presign-source-base
location and write the VDS to the azure-presign-base location
parent f6510848
......@@ -55,7 +55,7 @@ namespace OpenVDS
{
headers.emplace_back();
auto& header = headers.back();
header = fmt::format("Range: {}-{}", range.start, range.end);
header = fmt::format("x-ms-range: bytes={}-{}", range.start, range.end);
}
m_curlHandler.addDownloadRequest(request, url, headers, "x-ms-meta-", CurlDownloadHandler::GET);
return request;
......
......@@ -17,6 +17,8 @@
#include "IOManagerCurl.h"
#include <sstream>
#include <iomanip>
#include <fmt/printf.h>
namespace OpenVDS
......@@ -533,20 +535,32 @@ void CurlDownloadHandler::handleHeaderData(char* b, size_t size)
}
downloadRequest->m_handler->HandleMetadata(name, value);
std::string contentlength = name;
std::transform(contentlength.begin(), contentlength.end(), contentlength.begin(), [](char c)
std::string lowercase_name = name;
std::transform(lowercase_name.begin(), lowercase_name.end(), lowercase_name.begin(), [](char c)
{
return std::tolower(c);
});
if (contentlength == "content-length")
if (lowercase_name == "content-length")
{
char* end = 0;
long long length = strtoll(value.data(), &end, 10);
if (end > value.data() && length)
{
data.reserve(length);
if (verb == GET)
data.reserve(length);
downloadRequest->m_handler->HandleObjectSize(length);
}
}
else if (lowercase_name == "last-modified")
{
std::tm tm = {};
std::stringstream ss(value);
ss >> std::get_time(&tm, "%a, %d %b %Y %H:%M:%S");
ss = std::stringstream();
ss << std::put_time(&tm, "%Y-%m-%d %H:%M:%S");
downloadRequest->m_handler->HandleObjectLastWriteTime(ss.str());
}
return;
}
......
......@@ -1071,9 +1071,13 @@ main(int argc, char* argv[])
std::string region;
std::string connectionString;
std::string container;
std::string sourceConnectionString;
std::string sourceContainer;
int azureParallelismFactor = 0;
std::string azurePresignBase;
std::string azurePresignSuffix;
std::string azurePresignSourceBase;
std::string azurePresignSourceSuffix;
std::string prefix;
std::string sourcePrefix;
std::string persistentID;
......@@ -1097,9 +1101,13 @@ main(int argc, char* argv[])
options.add_option("", "", "region", "AWS region of bucket to upload to.", cxxopts::value<std::string>(region), "<string>");
options.add_option("", "", "connection-string", "Azure Blob Storage connection string.", cxxopts::value<std::string>(connectionString), "<string>");
options.add_option("", "", "container", "Azure Blob Storage container to upload to.", cxxopts::value<std::string>(container), "<string>");
options.add_option("", "", "source-connection-string", "Azure Blob Storage connection string.", cxxopts::value<std::string>(sourceConnectionString), "<string>");
options.add_option("", "", "source-container", "Azure Blob Storage container to upload to.", cxxopts::value<std::string>(sourceContainer), "<string>");
options.add_option("", "", "parallelism-factor", "Azure parallelism factor.", cxxopts::value<int>(azureParallelismFactor), "<value>");
options.add_option("", "", "azure-presign-base", "Base URL for presigned Azure requests", cxxopts::value(azurePresignBase), "<value>");
options.add_option("", "", "azure-presign-suffix", "Suffix of the presigned URL for Azure requests", cxxopts::value(azurePresignSuffix), "<value>");
options.add_option("", "", "azure-presign-source-base", "Base URL for presigned Azure requests", cxxopts::value(azurePresignSourceBase), "<value>");
options.add_option("", "", "azure-presign-source-suffix", "Suffix of the presigned URL for Azure requests", cxxopts::value(azurePresignSourceSuffix), "<value>");
options.add_option("", "", "prefix", "Top-level prefix to prepend to all object-keys.", cxxopts::value<std::string>(prefix), "<string>");
options.add_option("", "", "source-prefix", "Top-level prefix to prepend to all source object-keys.", cxxopts::value<std::string>(sourcePrefix), "<string>");
options.add_option("", "", "persistentID", "A globally unique ID for the VDS, usually an 8-digit hexadecimal number.", cxxopts::value<std::string>(persistentID), "<ID>");
......@@ -1213,9 +1221,9 @@ main(int argc, char* argv[])
OpenVDS::Error
error;
auto sourceBucketOpenOptions = createOpenOptions(sourcePrefix, persistentID, sourceBucket, region, container, connectionString, azureParallelismFactor, azurePresignBase, azurePresignSuffix, error);
auto sourceBucketOpenOptions = createOpenOptions(sourcePrefix, persistentID, sourceBucket, region, sourceContainer, sourceConnectionString, azureParallelismFactor, azurePresignSourceBase, azurePresignSourceSuffix, error);
DataProvider dataProvider = sourceBucket.empty() ? CreateDataProviderFromFile(fileNames[0], error) : CreateDataProviderFromOpenOptions(*sourceBucketOpenOptions, fileNames[0], error);
DataProvider dataProvider = error.code ? CreateDataProviderFromFile(fileNames[0], error) : CreateDataProviderFromOpenOptions(*sourceBucketOpenOptions, fileNames[0], error);
if (error.code != 0)
{
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment