diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index fe877ce198e11ad86ab56d25176e6db37ccd7f66..08017a8926cc156cb516314b6fb54a7d81d66ce2 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -117,7 +117,11 @@ add_test_executable(global_state add_test_executable(openvds_mixed_request OpenVDS/RequestVolumeSubsetAndPageAccessor.cpp - ) +) + +add_test_executable(tools + tools/SplitUrlTest.cpp +) if (TEST_SEGY_FILE AND TEST_URL) add_test(NAME "tools.SegyRoundtrip" diff --git a/tests/tools/SplitUrlTest.cpp b/tests/tools/SplitUrlTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..df976dcf80a5f7de2605a74b57d48dd9d87a9df3 --- /dev/null +++ b/tests/tools/SplitUrlTest.cpp @@ -0,0 +1,107 @@ + +/**************************************************************************** +** Copyright 2021 The Open Group +** Copyright 2021 Bluware, Inc. +** +** Licensed under the Apache License, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** http://www.apache.org/licenses/LICENSE-2.0 +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +****************************************************************************/ + +#include + +#include "../../tools/SEGYImport/SplitUrl.h" + +GTEST_TEST(SplitUrlOn, splitUrlOnParameters) +{ + std::string basic = "s3://url/to/some/file.json"; + std::string withquery_parameter = "http://url/to/some/other/file.json?magicparams=here"; + std::string nofile = "az://url/to/some/dir/"; + std::string nofile_parameter = "gs://url/to/some/dir/?foobar=params"; + + { + std::string out_url; + std::string out_params; + splitUrlOnParameters(basic, out_url, out_params); + ASSERT_EQ(basic, out_url); + ASSERT_EQ(std::string(""), out_params); + } + { + std::string out_url; + std::string out_params; + splitUrlOnParameters(withquery_parameter, out_url, out_params); + ASSERT_EQ(std::string("http://url/to/some/other/file.json"), out_url); + ASSERT_EQ(std::string("?magicparams=here"), out_params); + } + { + std::string out_url; + std::string out_params; + splitUrlOnParameters(nofile, out_url, out_params); + ASSERT_EQ(nofile, out_url); + ASSERT_EQ(std::string(""), out_params); + } + { + std::string out_url; + std::string out_params; + splitUrlOnParameters(nofile_parameter, out_url, out_params); + ASSERT_EQ("gs://url/to/some/dir/", out_url); + ASSERT_EQ("?foobar=params",out_params); + } +} + +GTEST_TEST(SplitUrlOn, splitUrlFile) +{ + std::string basic = "s3://url/to/some/file.json"; + std::string withquery_parameter = "http://url/to/some/other/file.json?magicparams=here"; + std::string nofile = "az://url/to/some/dir/"; + std::string nofile_parameter = "gs://url/to/some/dir/?foobar=params"; + + { + std::string dirname; + std::string basename; + std::string parameters; + OpenVDS::Error error; + splitUrl(basic, dirname, basename, parameters, error); + ASSERT_TRUE(error.code == 0); + ASSERT_EQ(parameters, ""); + ASSERT_EQ(basename, "file.json"); + ASSERT_EQ(dirname, "s3://url/to/some"); + } + { + std::string dirname; + std::string basename; + std::string parameters; + OpenVDS::Error error; + splitUrl(withquery_parameter, dirname, basename, parameters, error); + ASSERT_TRUE(error.code == 0); + ASSERT_EQ(parameters, "?magicparams=here"); + ASSERT_EQ(basename, "file.json"); + ASSERT_EQ(dirname, "http://url/to/some/other"); + } + { + std::string dirname; + std::string basename; + std::string parameters; + OpenVDS::Error error; + splitUrl(nofile, dirname, basename, parameters, error); + ASSERT_TRUE(error.code != 0); + error = OpenVDS::Error(); + } + { + std::string dirname; + std::string basename; + std::string parameters; + OpenVDS::Error error; + splitUrl(nofile_parameter, dirname, basename, parameters, error); + ASSERT_TRUE(error.code != 0); + } + +} diff --git a/tools/SEGYExport/SEGYExport.cpp b/tools/SEGYExport/SEGYExport.cpp index fafa05c77e88f4414a556d224891719da3a86716..8914e0fdc91a79b108cfa3644dac2daf033716e9 100644 --- a/tools/SEGYExport/SEGYExport.cpp +++ b/tools/SEGYExport/SEGYExport.cpp @@ -41,14 +41,13 @@ main(int argc, char *argv[]) std::string url; std::string connection; - std::string vdsFileName; std::string persistentID; std::string fileName; bool help = false; - options.add_option("", "", "url", "Url with vendor specific protocol.", cxxopts::value(url), ""); + options.add_option("", "", "url", "Url with vendor specific protocol or VDS file name.", cxxopts::value(url), ""); options.add_option("", "", "connection", "Vendor specific connection string.", cxxopts::value(connection), ""); - options.add_option("", "", "vdsfile", "Input VDS file name.", cxxopts::value(vdsFileName), ""); + options.add_option("", "", "vdsfile", "Input VDS file name.", cxxopts::value(url), ""); options.add_option("", "", "persistentID", "A globally unique ID for the VDS, usually an 8-digit hexadecimal number.", cxxopts::value(persistentID), ""); options.add_option("", "h", "help", "Print this help information", cxxopts::value(help), ""); @@ -99,13 +98,13 @@ main(int argc, char *argv[]) OpenVDS::VDSHandle handle; - if(vdsFileName.empty()) + if(OpenVDS::IsSupportedProtocol(url)) { handle = OpenVDS::Open(url, connection, openError); } else { - handle = OpenVDS::Open(OpenVDS::VDSFileOpenOptions(vdsFileName), openError); + handle = OpenVDS::Open(OpenVDS::VDSFileOpenOptions(url), openError); } if(openError.code != 0) diff --git a/tools/SEGYImport/SEGYImport.cpp b/tools/SEGYImport/SEGYImport.cpp index 49b815527dc22b03647f06d84f8fa1e2cf0ecefa..e79c6650ad1f329f9c8079d8a7f9243c4a753983 100644 --- a/tools/SEGYImport/SEGYImport.cpp +++ b/tools/SEGYImport/SEGYImport.cpp @@ -33,6 +33,8 @@ #include #include +#include "IO/IOManager.h" + #include #include #include @@ -43,6 +45,8 @@ #include #include +#include "SplitUrl.h" + #include #include @@ -72,7 +76,7 @@ int64_t GetTotalSystemMemory() } #endif -static DataProvider CreateDataProviderFromFile(const std::string &filename, OpenVDS::Error &error) +DataProvider CreateDataProviderFromFile(const std::string &filename, OpenVDS::Error &error) { std::unique_ptr file(new OpenVDS::File()); if (!file->Open(filename, false, false, false, error)) @@ -80,7 +84,7 @@ static DataProvider CreateDataProviderFromFile(const std::string &filename, Open return DataProvider(file.release()); } -static DataProvider CreateDataProviderFromOpenOptions(const std::string &url, const std::string &connectionString, const std::string &objectId, OpenVDS::Error &error) +DataProvider CreateDataProviderFromOpenOptions(const std::string &url, const std::string &connectionString, const std::string &objectId, OpenVDS::Error &error) { std::unique_ptr ioManager(OpenVDS::IOManager::CreateIOManager(url, connectionString, OpenVDS::IOManager::AccessPattern::ReadOnly, error)); if (error.code) @@ -88,6 +92,46 @@ static DataProvider CreateDataProviderFromOpenOptions(const std::string &url, co return DataProvider(ioManager.release(), objectId, error); } +DataProvider CreateDataProvider(const std::string& name, const std::string& connection, OpenVDS::Error& error) +{ + if (OpenVDS::IsSupportedProtocol(name)) + { + std::string dirname; + std::string basename; + std::string parameters; + splitUrl(name, dirname, basename, parameters, error); + if (error.code) + return DataProvider(nullptr); + + std::string url = dirname + parameters; + return CreateDataProviderFromOpenOptions(url, connection, basename, error); + } + else + { + return CreateDataProviderFromFile(name, error); + } + return DataProvider(nullptr); +} + +std::vector CreateDataProviders(const std::vector &fileNames, const std::string &connection, OpenVDS::Error &error) +{ + std::vector + dataProviders; + + for (const auto& fileName : fileNames) + { + error = OpenVDS::Error(); + dataProviders.push_back(CreateDataProvider(fileName, connection, error)); + + if (error.code != 0) + { + dataProviders.clear(); + break; + } + } + return dataProviders; +} + Json::Value SerializeSEGYBinInfo(SEGYBinInfo const& binInfo) { @@ -303,24 +347,24 @@ HeaderFieldFromJson(Json::Value const& jsonHeaderField) } bool -ParseHeaderFormatFile(OpenVDS::File const& file, std::map& traceHeaderFields, SEGY::Endianness& headerEndianness, OpenVDS::Error &error) +ParseHeaderFormatFile(DataProvider &dataProvider, std::map& traceHeaderFields, SEGY::Endianness& headerEndianness, OpenVDS::Error &error) { - int64_t fileSize = file.Size(error); + int64_t dataSize = dataProvider.Size(error); if (error.code != 0) { return false; } - if (fileSize > INT_MAX) + if (dataSize > INT_MAX) { return false; } std::unique_ptr - buffer(new char[fileSize]); + buffer(new char[dataSize]); - file.Read(buffer.get(), 0, (int32_t)fileSize, error); + dataProvider.Read(buffer.get(), 0, (int32_t)dataSize, error); if (error.code != 0) { @@ -344,7 +388,7 @@ ParseHeaderFormatFile(OpenVDS::File const& file, std::mapparse(buffer.get(), buffer.get() + fileSize, &root, &errs); + success = reader->parse(buffer.get(), buffer.get() + dataSize, &root, &errs); if (!success) { @@ -921,11 +965,11 @@ createImportInformationMetadata(const std::vector &dataProviders, } bool -parseSEGYFileInfoFile(OpenVDS::File const& file, SEGYFileInfo& fileInfo) +parseSEGYFileInfoFile(DataProvider &dataProvider, SEGYFileInfo& fileInfo) { OpenVDS::Error error; - int64_t fileSize = file.Size(error); + int64_t fileSize = dataProvider.Size(error); if (error.code != 0) { @@ -940,7 +984,7 @@ parseSEGYFileInfoFile(OpenVDS::File const& file, SEGYFileInfo& fileInfo) std::unique_ptr buffer(new char[fileSize]); - file.Read(buffer.get(), 0, (int32_t)fileSize, error); + dataProvider.Read(buffer.get(), 0, (int32_t)fileSize, error); if (error.code != 0) { @@ -1254,35 +1298,6 @@ findFirstTrace(TraceDataManager& traceDataManager, const SEGYSegmentInfo& segmen return findFirstTrace(traceDataManager, segment.m_primaryKey, secondaryKey, fileInfo, segment.m_traceStart, segment.m_traceStop, secondaryStart, secondaryStop, error); } -static std::vector CreateDataProviders(const std::vector &fileNames, - const std::string &url, const std::string &connection, - OpenVDS::Error &error) -{ - std::vector - dataProviders; - - for (const auto& fileName : fileNames) - { - error = OpenVDS::Error(); - if (url.empty()) - { - dataProviders.push_back(CreateDataProviderFromFile(fileName, error)); - } - else - { - dataProviders.push_back(CreateDataProviderFromOpenOptions(url, connection, fileName, error)); - } - - if (error.code != 0) - { - dataProviders.clear(); - break; - } - } - - return dataProviders; -} - int SecondaryKeyDimension(const SEGYFileInfo& fileInfo) { @@ -1330,10 +1345,8 @@ main(int argc, char* argv[]) bool force = false; bool ignoreWarnings = false; std::string url; - std::string connection; - std::string vdsFileName; - std::string sourceUrl; - std::string sourceConnection; + std::string urlConnection; + std::string inputConnection; std::string persistentID; bool uniqueID = false; bool disablePersistentID = false; @@ -1357,11 +1370,10 @@ main(int argc, char* argv[]) options.add_option("", "b", "brick-size", "The brick size for the volume data store.", cxxopts::value(brickSize)->default_value("64"), ""); options.add_option("", "f", "force", "Continue on upload error.", cxxopts::value(force), ""); options.add_option("", "", "ignore-warnings", "Ignore warnings about import parameters.", cxxopts::value(ignoreWarnings), ""); - options.add_option("", "", "url", "Url with cloud vendor scheme used for target location.", cxxopts::value(url), ""); - options.add_option("", "", "connection", "Connection string used for additional parameters to the target connection", cxxopts::value(connection), ""); - options.add_option("", "", "vdsfile", "File name of output VDS file.", cxxopts::value(vdsFileName), ""); - options.add_option("", "", "source-url", "Url with cloud vendor scheme used for source location.", cxxopts::value(sourceUrl), ""); - options.add_option("", "", "source-connection", "Connection string used for additional parameters to the source connection", cxxopts::value(sourceConnection), ""); + options.add_option("", "", "url", "Url with cloud vendor scheme used for target location or file name of output VDS file.", cxxopts::value(url), ""); + options.add_option("", "", "url-connection", "Connection string used for additional parameters to the url connection", cxxopts::value(urlConnection), ""); + options.add_option("", "", "vdsfile", "File name of output VDS file.", cxxopts::value(url), ""); + options.add_option("", "", "input-connection", "Connection string used for additional parameters to the input connection", cxxopts::value(inputConnection), ""); options.add_option("", "", "persistentID", "A globally unique ID for the VDS, usually an 8-digit hexadecimal number.", cxxopts::value(persistentID), ""); options.add_option("", "", "uniqueID", "Generate a new globally unique ID when scanning the input SEG-Y file.", cxxopts::value(uniqueID), ""); options.add_option("", "", "disable-persistentID", "Disable the persistentID usage, placing the VDS directly into the url location.", cxxopts::value(disablePersistentID), ""); @@ -1454,13 +1466,9 @@ main(int argc, char* argv[]) if (!headerFormatFileName.empty()) { - OpenVDS::File - headerFormatFile; - OpenVDS::Error error; - - headerFormatFile.Open(headerFormatFileName, false, false, false, error); + DataProvider headerFormatDataProvider = CreateDataProvider(headerFormatFileName, inputConnection, error); if (error.code != 0) { @@ -1468,7 +1476,7 @@ main(int argc, char* argv[]) return EXIT_FAILURE; } - ParseHeaderFormatFile(headerFormatFile, g_traceHeaderFields, headerEndianness, error); + ParseHeaderFormatFile(headerFormatDataProvider, g_traceHeaderFields, headerEndianness, error); if (error.code != 0) { @@ -1505,7 +1513,7 @@ main(int argc, char* argv[]) OpenVDS::Error error; - auto dataProviders = CreateDataProviders(fileNames, sourceUrl, sourceConnection, error); + auto dataProviders = CreateDataProviders(fileNames, inputConnection, error); if (error.code != 0) { // TODO need to name which file failed to open @@ -1562,48 +1570,78 @@ main(int argc, char* argv[]) } else { - OpenVDS::File - fileInfoFile; - OpenVDS::Error error; - fileInfoFile.Open(fileInfoFileName.c_str(), true, false, true, error); - - if (error.code != 0) + if (OpenVDS::IsSupportedProtocol(fileInfoFileName)) { - std::cerr << std::string("Could not create file: ") << fileInfoFileName; - return EXIT_FAILURE; + std::string dirname; + std::string basename; + std::string parameters; + splitUrl(fileInfoFileName, dirname, basename, parameters, error); + if (error.code) + { + fmt::print(stderr, "Failed to creating IOManager for {}: {}\n", fileInfoFileName, error.string); + return EXIT_FAILURE; + } + std::string scanUrl = dirname + parameters; + std::unique_ptr ioManager(OpenVDS::IOManager::CreateIOManager(scanUrl, urlConnection, OpenVDS::IOManager::ReadWrite, error)); + if (error.code) + { + fmt::print(stderr, "Failed to creating IOManager for {}: {}\n", fileInfoFileName, error.string); + return EXIT_FAILURE; + } + auto shared_data = std::make_shared>(); + shared_data->insert(shared_data->end(), document.begin(), document.end()); + auto req = ioManager->WriteObject(basename, "", "text/plain", {}, shared_data, {}); + req->WaitForFinish(); + req->IsSuccess(error); + if (error.code) + { + fmt::print(stderr, "Failed to write {}: {}\n", fileInfoFileName, error.string); + return EXIT_FAILURE; + } } + else + { + OpenVDS::File + fileInfoFile; - fileInfoFile.Write(document.data(), 0, (int32_t)document.size(), error); + fileInfoFile.Open(fileInfoFileName.c_str(), true, false, true, error); - if (error.code != 0) - { - std::cerr << std::string("Could not write to file: ") << fileInfoFileName; - return EXIT_FAILURE; + if (error.code != 0) + { + std::cerr << std::string("Could not create file: ") << fileInfoFileName; + return EXIT_FAILURE; + } + + fileInfoFile.Write(document.data(), 0, (int32_t)document.size(), error); + + if (error.code != 0) + { + std::cerr << std::string("Could not write to file: ") << fileInfoFileName; + return EXIT_FAILURE; + } } + } return EXIT_SUCCESS; } } else if (!fileInfoFileName.empty()) { - OpenVDS::File - fileInfoFile; - OpenVDS::Error error; - fileInfoFile.Open(fileInfoFileName.c_str(), false, false, false, error); + DataProvider fileInfoDataProvider = CreateDataProvider(fileInfoFileName, inputConnection, error); if (error.code != 0) { - std::cerr << std::string("Could not open file: ") << fileInfoFileName; + fmt::print(stderr, "Could not create data provider for {}: {}.\n", fileInfoFileName, error.string); return EXIT_FAILURE; } - bool success = parseSEGYFileInfoFile(fileInfoFile, fileInfo); + bool success = parseSEGYFileInfoFile(fileInfoDataProvider, fileInfo); if (!success) { @@ -1779,24 +1817,27 @@ main(int argc, char* argv[]) OpenVDS::Error createError; - if (!persistentID.empty()) - { - if (!url.empty() && url.back() != '/') - { - url.push_back('/'); - } - url.insert(url.end(), persistentID.begin(), persistentID.end()); - } - OpenVDS::VDSHandle handle; - if(vdsFileName.empty()) + if(OpenVDS::IsSupportedProtocol(url)) { - handle = OpenVDS::Create(url, connection, layoutDescriptor, axisDescriptors, channelDescriptors, metadataContainer, createError); + if (!persistentID.empty()) + { + std::string baseUrl; + std::string parameters; + splitUrlOnParameters(url, baseUrl, parameters); + if (baseUrl.back() != '/') + { + baseUrl.push_back('/'); + } + baseUrl.insert(baseUrl.end(), persistentID.begin(), persistentID.end()); + url = baseUrl + parameters; + } + handle = OpenVDS::Create(url, urlConnection, layoutDescriptor, axisDescriptors, channelDescriptors, metadataContainer, createError); } else { - handle = OpenVDS::Create(OpenVDS::VDSFileOpenOptions(vdsFileName), layoutDescriptor, axisDescriptors, channelDescriptors, metadataContainer, createError); + handle = OpenVDS::Create(OpenVDS::VDSFileOpenOptions(url), layoutDescriptor, axisDescriptors, channelDescriptors, metadataContainer, createError); } if (createError.code != 0) diff --git a/tools/SEGYImport/SplitUrl.h b/tools/SEGYImport/SplitUrl.h new file mode 100644 index 0000000000000000000000000000000000000000..d50929ab17b05e48a2a46d16aebae322fdc1978a --- /dev/null +++ b/tools/SEGYImport/SplitUrl.h @@ -0,0 +1,31 @@ +#ifndef SPLIT_URL_H +#define SPLIT_URL_H + +#include +#include +#include + +void splitUrlOnParameters(const std::string& full_url, std::string& url, std::string& parameters) +{ + auto parameters_start = std::find(full_url.begin(), full_url.end(), '?'); + + url.insert(url.end(), full_url.begin(), parameters_start); + parameters.insert(parameters.end(), parameters_start, full_url.end()); +} + +void splitUrl(const std::string& url, std::string &dirName, std::string &baseName, std::string ¶meters, OpenVDS::Error& error) +{ + std::string url_no_param; + splitUrlOnParameters(url, url_no_param, parameters); + if (url_no_param.back() == '/') + { + error.code = -1; + error.string = fmt::format("Missing file argument for url {}.", url); + return; + } + auto last_slash = std::find(url_no_param.rbegin(), url_no_param.rend(), '/').base(); + dirName.insert(dirName.end(), url_no_param.begin(), last_slash - 1); + baseName.insert(baseName.end(), last_slash, url_no_param.end()); +} + +#endif \ No newline at end of file diff --git a/tools/VDSInfo/VDSInfo.cpp b/tools/VDSInfo/VDSInfo.cpp index f21ac915d9015360a48c7e554527c544fe7c439a..126a5d27f1f05e0efa4691fe21ee0a9c03cd0154 100644 --- a/tools/VDSInfo/VDSInfo.cpp +++ b/tools/VDSInfo/VDSInfo.cpp @@ -194,8 +194,6 @@ int main(int argc, char **argv) std::vector urlarg; std::string connection; - std::string vdsFileName; - std::string persistentID; std::string metadataPrintName; std::string metadataPrintCategory; @@ -212,7 +210,6 @@ int main(int argc, char **argv) //connection options options.add_option("", "", "urlpos", "Url with vendor specific protocol.", cxxopts::value>(urlarg), ""); options.add_option("", "", "connection", "Vendor specific connection string.", cxxopts::value(connection), ""); - options.add_option("", "", "persistentID", "A globally unique ID for the VDS, usually an 8-digit hexadecimal number.", cxxopts::value(persistentID), ""); ///action options.add_option("", "", "axis", "Print axis descriptors.", cxxopts::value(axisDescriptors), ""); @@ -227,8 +224,7 @@ int main(int argc, char **argv) options.add_option("", "e", "metadata-autodecode", "Autodetect EBCDIC and decode to ASCII for blobs.", cxxopts::value(metadataAutoDecodeEBCDIC), ""); options.add_option("", "w", "metadata-force-width", "Force output width.", cxxopts::value(textDecodeWidth), ""); - options.add_option("", "", "url", "Url with vendor specific protocol. (Available as positional argument as well).", cxxopts::value>(urlarg), ""); - options.add_option("", "", "vdsfile", "VDS file.", cxxopts::value(vdsFileName), ""); + options.add_option("", "", "url", "Url with vendor specific protocol or VDS file. (Available as positional argument as well).", cxxopts::value>(urlarg), ""); options.add_option("", "h", "help", "Print this help information", cxxopts::value(help), ""); @@ -256,33 +252,16 @@ int main(int argc, char **argv) return EXIT_SUCCESS; } - if (urlarg.empty() && vdsFileName.empty()) + if (urlarg.empty()) { std::cout << "\nFailed - missing url/vdsfile argument\n\n"; std::cout << options.help(); return EXIT_FAILURE; } - if (urlarg.size() + vdsFileName.size() > 1) - { - std::cout << "\nFailed - can only specify one url/vdsfile argument\n\n"; - std::cout << options.help(); - return EXIT_FAILURE; - } - std::string url; if(!urlarg.empty()) url = urlarg[0]; - // Open the VDS - if (!persistentID.empty()) - { - if (!url.empty() && url.back() != '/') - { - url.push_back('/'); - } - url.insert(url.end(), persistentID.begin(), persistentID.end()); - } - if (!axisDescriptors && !channelDescriptors && !volumeDataLayout && !metaKeys && metadataPrintName.empty() && metadataPrintCategory.empty() && !metadataAll) { axisDescriptors = true; @@ -295,13 +274,13 @@ int main(int argc, char **argv) OpenVDS::VDSHandle handle; - if(vdsFileName.empty()) + if(OpenVDS::IsSupportedProtocol(url)) { handle = OpenVDS::Open(url, connection, openError); } else { - handle = OpenVDS::Open(OpenVDS::VDSFileOpenOptions(vdsFileName), openError); + handle = OpenVDS::Open(OpenVDS::VDSFileOpenOptions(url), openError); } if(openError.code != 0)