Commit 623fd567 authored by Paal Kvamme's avatar Paal Kvamme
Browse files

Additional unit tests for updating ZGY files on Seismic Store.

parent 32538d8a
......@@ -187,8 +187,10 @@ ifneq ($(strip $(ZFP_LIBRARY)),)
/bin/cp -a -t $(BIN_DIR) $(strip $(ZFP_LIBRARY))*
endif
# SD_LIBRARY is only needed because of test/sdutils.cpp doing direct access
# to SDAPI and bypassing OpenZGY entirely.
$(BIN_DIR)/test_all: $(TEST_OBJ) $(LIBDSO) $(SD_SENTINEL) $(ZFP_SENTINEL)
$(CXX) -o $@ $(CXXFLAGS) $(ORIGIN) $(TEST_OBJ) $(LIBDSO)
$(CXX) -o $@ $(CXXFLAGS) $(ORIGIN) $(TEST_OBJ) $(LIBDSO) $(SD_LIBRARY)
#$(BIN_DIR)/zgycopyc: $(OBJ_DIR)/tools/zgycopyc.o $(OBJ_DIR)/test/mock.o $(LIBDSO)
# $(CXX) -o $@ $(CXXFLAGS) $(ORIGIN) $^ -fopenmp
......
......@@ -21,6 +21,7 @@
#include "test_all.h"
#include "test_utils.h"
#include "test_sdutils.h"
#include "../api.h"
#include "../iocontext.h"
//#include "../exception.h"
......@@ -93,6 +94,15 @@ get_testdata(const std::string& name)
#endif
}
static std::string
get_sdtestdata(const std::string& name)
{
std::string testfolder = InternalZGY::Environment::getStringEnv("OPENZGY_SDTESTDATA", "sd://sntc/testdata");
if (testfolder.back() != '/')
testfolder += "/";
return testfolder + name;
}
static void
do_write_twice(const std::string& filename, const IOContext* context = nullptr)
{
......@@ -1821,6 +1831,57 @@ test_reopen_zgypublic()
}
}
#ifdef HAVE_SD
/**
* Opening an empty file created by the old ZGY accessor and cloud plug-in
* is not allowed.
*/
void
test_reopen_zgycloud()
{
Test_Utils::CloudFileAutoDelete cad
("reopen_zgycloud.zgy", Test_Utils::default_sd_context());
{
// Expected to fail because the header segment is not padded.
// Otherwise this file, containing no bricks, would have worked.
const std::string oldname = get_sdtestdata("EmptyOldFile.zgy");
Test_Utils::copy_sd_to_sd(oldname, cad.name());
std::shared_ptr<IZgyWriter> writer;
must_throw("Only files uploaded by OpenZGY", [&](){
writer = IZgyWriter::reopen(ZgyWriterArgs()
.filename(cad.name())
.iocontext(Test_Utils::default_sd_context()));
});
}
{
// Expected to fail because the first segment has both header and bulk.
const std::string oldname = get_sdtestdata("Synt2.zgy");
Test_Utils::copy_sd_to_sd(oldname, cad.name());
std::shared_ptr<IZgyWriter> writer;
must_throw("Only files uploaded by OpenZGY", [&](){
writer = IZgyWriter::reopen(ZgyWriterArgs()
.filename(cad.name())
.iocontext(Test_Utils::default_sd_context()));
});
}
{
// Expected to fail because only the current version (3/4) can be updated,
// and that check should be done first because that error is more clear.
// Without the check it would still fail the "uploaded by OpenZGY" test.
const std::string oldname = get_sdtestdata("Synt2-v1.zgy");
Test_Utils::copy_sd_to_sd(oldname, cad.name());
std::shared_ptr<IZgyWriter> writer;
must_throw("version is too old for this library", [&](){
writer = IZgyWriter::reopen(ZgyWriterArgs()
.filename(cad.name())
.iocontext(Test_Utils::default_sd_context()));
});
}
}
#endif
static std::vector<std::int16_t>
make_sequence(std::int64_t start, std::int64_t count, std::int64_t step)
{
......@@ -1851,10 +1912,8 @@ array_to_string(const std::vector<T>& vec, double scale)
* Extract the segment information from FileStatistics -> xx_segments()
* as the abbreviated 3-element vector of (first, all_middle, last)
* and expand that to a list of all the segment sizes.
* Yes this is way too roundabout.
* TODO-@@@-Test: For a unit test the code should really have checked
* each and every segment using SDAPI to see that all except the first
* and last are the same size.
* Yes this is way too roundabout. Also it obviously cannot check
* that the segments that are expected to be the same size actually are.
*
* Note that if the FileStatistics come from a file opened for write
* then the result depends on the segsplit setting. Data still in the
......@@ -1891,11 +1950,22 @@ list_segments(std::shared_ptr<const OpenZGY::FileStatistics> fs, int verbose)
static std::vector<std::int64_t>
list_segments(const std::string& name, int verbose)
{
#if 0
std::shared_ptr<OpenZGY::IZgyReader> reader =
OpenZGY::IZgyReader::open(name, Test_Utils::default_sd_context());
std::shared_ptr<const OpenZGY::FileStatistics> fs = reader->filestats();
reader->close();
return list_segments(fs, verbose);
#else
// Bypass OpenZGY completely. This will get the actual size of each
// segment instead of assuming that all but the first and last have
// the same size.
std::vector<std::int64_t> result = Test_Utils::get_segsizes(name);
if (verbose)
std::cout << "segments " << array_to_string(result, 512*1024)
<< std::endl;
return result;
#endif
}
static void
......@@ -2008,6 +2078,26 @@ test_reopen_sd()
}
}
{
// Same as above but more thorough as it bypasses OpenZGY.
std::vector<std::int64_t> check =
list_segments(cad.name(), verbose());
if (TEST_EQUAL(check.size(), (std::size_t)12)) {
TEST_EQUAL(check[0], 512*1024);
TEST_EQUAL(check[1], 4*512*1024);
TEST_EQUAL(check[2], 4*512*1024);
TEST_EQUAL(check[3], 4*512*1024);
TEST_EQUAL(check[4], 4*512*1024);
TEST_EQUAL(check[5], 4*512*1024);
TEST_EQUAL(check[6], 4*512*1024);
TEST_EQUAL(check[7], 4*512*1024);
TEST_EQUAL(check[8], 4*512*1024);
TEST_EQUAL(check[9], 4*512*1024);
TEST_EQUAL(check[10], 4*512*1024);
TEST_EQUAL(check[11], 3*512*1024);
}
}
std::vector<std::int16_t> check1(11*64);
std::vector<std::int16_t> check2(11*64);
reader->read(size3i_t{0,0,0}, size3i_t{1,1,11*64}, check1.data(), 0);
......@@ -2051,6 +2141,7 @@ public:
#ifdef HAVE_SD
register_test("reopen.plain_sd", test_reopen_plain_sd);
register_test("reopen.sd", test_reopen_sd);
register_test("reopen.zgycloud", test_reopen_zgycloud);
#endif
}
} dummy_reopen;
......
// Copyright 2017-2021, Schlumberger
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Based on Salmon/Shared/TestUtils/TempFileAutoDelete.cpp
// And wrapper/test_utils.py
#ifdef HAVE_SD // Rest of file
#include "test_sdutils.h"
#include "../impl/environment.h"
#define TEST_NO_MAIN
#define NOMINMAX // cutest includes Windows.h. Ugh!
#include "cutest.h"
#ifndef _WIN32
#include <SDManager.h>
#include <SDGenericDataset.h>
#else
#include <SDAPI/SDManager.h>
#include <SDAPI/SDGenericDataset.h>
#endif
#include <stdexcept>
namespace Test_Utils {
#if 0
}
#endif
static std::shared_ptr<seismicdrive::SDManager>
get_manager()
{
using InternalZGY::Environment;
const std::string sdurl{Environment::getStringEnv("OPENZGY_SDURL")};
const std::string sdapikey{Environment::getStringEnv("OPENZGY_SDAPIKEY")};
const std::string sdtoken{Environment::getStringEnv("OPENZGY_TOKEN")};
if (sdurl.empty() || sdapikey.empty() || sdtoken.empty())
throw std::runtime_error("Missing $OPENZGY_{SDURL,SDAPIKEY,TOKEN}");
std::shared_ptr<seismicdrive::SDManager> sdmanager
(new seismicdrive::SDManager(sdurl, sdapikey));
sdmanager->setAuthProviderFromString(sdtoken);
return sdmanager;
}
/**
* Copy a file on seismic store to another file on seismic store.
* This uses direct calls to SDAPI only; there is no OpenZGY involved.
* To be used for setting up unit test data. Caveat: No legaltag
* is provided, so the destination subproject must have a default set.
*/
void
copy_sd_to_sd(const std::string& srcname, const std::string& dstname)
{
std::shared_ptr<seismicdrive::SDManager> manager = get_manager();
seismicdrive::SDGenericDataset src(manager.get(), srcname);
src.open(seismicdrive::SDDatasetDisposition::READ_ONLY);
const std::uint64_t nblocks = src.getBlockNum();
std::vector<std::string> names;
for (std::uint64_t ii = 0; ii < nblocks; ++ii)
names.push_back(std::to_string(ii));
const std::vector<long long> sizearray = src.getBlocksSize(names);
for (long long segsize : sizearray)
if (segsize < 0)
throw std::runtime_error("Segment size must be > 0");
seismicdrive::SDGenericDataset dst(manager.get(), dstname);
dst.open(seismicdrive::SDDatasetDisposition::OVERWRITE);
for (std::uint64_t segnum = 0; segnum < nblocks; ++segnum) {
//std::cout << "copy " << segnum << " size " << sizearray[segnum] << "\n";
std::unique_ptr<char[]> data(new char[sizearray[segnum]]);
src.readBlock((int)segnum, data.get(), 0, (size_t)sizearray[segnum]);
dst.writeBlock((int)segnum, data.get(), (size_t)sizearray[segnum]);
}
dst.close();
src.close();
}
std::vector<std::int64_t>
get_segsizes(const std::string& name)
{
std::shared_ptr<seismicdrive::SDManager> manager = get_manager();
seismicdrive::SDGenericDataset src(manager.get(), name);
src.open(seismicdrive::SDDatasetDisposition::READ_ONLY);
const std::uint64_t nblocks = src.getBlockNum();
std::vector<std::string> names;
for (std::uint64_t ii = 0; ii < nblocks; ++ii)
names.push_back(std::to_string(ii));
const std::vector<long long> sizearray = src.getBlocksSize(names);
src.close();
std::vector<std::int64_t> result;
for (auto it : sizearray)
result.push_back(it);
return result;
}
} // namespace Test_Utils
#endif // HAVE_SD
// Copyright 2017-2021, Schlumberger
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <string>
#include <vector>
#include <cstdint>
namespace Test_Utils {
#if 0
}
#endif
// Free functions
#ifdef HAVE_SD
void copy_sd_to_sd(const std::string& srcname, const std::string& dstname);
std::vector<std::int64_t> get_segsizes(const std::string& name);
#endif
} // namespace
......@@ -84,6 +84,7 @@
<ClInclude Include="..\src\test\cutest.h" />
<ClInclude Include="..\src\test\mock.h" />
<ClInclude Include="..\src\test\test_all.h" />
<ClInclude Include="..\src\test\test_sdutils.h" />
<ClInclude Include="..\src\test\test_utils.h" />
</ItemGroup>
<ItemGroup>
......@@ -103,6 +104,7 @@
<ClCompile Include="..\src\test\test_mtguard.cpp" />
<ClCompile Include="..\src\test\test_reopen.cpp" />
<ClCompile Include="..\src\test\test_roundandclip.cpp" />
<ClCompile Include="..\src\test\test_sdutils.cpp" />
<ClCompile Include="..\src\test\test_smallcache.cpp" />
<ClCompile Include="..\src\test\test_structaccess.cpp" />
<ClCompile Include="..\src\test\test_timer.cpp" />
......@@ -129,4 +131,4 @@
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
</Project>
\ No newline at end of file
</Project>
......@@ -20,6 +20,9 @@
<ClInclude Include="..\src\test\test_all.h">
<Filter>src\test</Filter>
</ClInclude>
<ClInclude Include="..\src\test\test_sdutils.h">
<Filter>src\test</Filter>
</ClInclude>
<ClInclude Include="..\src\test\test_utils.h">
<Filter>src\test</Filter>
</ClInclude>
......@@ -73,6 +76,9 @@
<ClCompile Include="..\src\test\test_roundandclip.cpp">
<Filter>src\test</Filter>
</ClCompile>
<ClCompile Include="..\src\test\test_sdutils.cpp">
<Filter>src\test</Filter>
</ClCompile>
<ClCompile Include="..\src\test\test_smallcache.cpp">
<Filter>src\test</Filter>
</ClCompile>
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment