Commit 878690f2 authored by Paal Kvamme's avatar Paal Kvamme
Browse files

Unit tests for incremental finalize.

parent 94ef360c
......@@ -48,6 +48,34 @@ namespace Test_API {
}
#endif
namespace {
/**
* Pass a SilentProgress instance to finalize() to get back the "total"
* field that will be the same for all invocations. I happen to know
* that this is the expected number of bricks read or written.
*/
class SilentProgress
{
private:
std::mutex _mutex;
std::int64_t _done;
std::int64_t _total;
public:
SilentProgress(const SilentProgress&) = delete;
SilentProgress& operator=(const SilentProgress&) = delete;
SilentProgress(): _done(0), _total(0) {}
std::int64_t done() const { return _done; }
std::int64_t total() const { return _total; }
void reset() { _done = _total = 0; }
bool operator()(std::int64_t done, std::int64_t total) {
std::lock_guard<std::mutex> lk(_mutex);
_done = std::max(_done, done);
_total = total;
return true;
}
};
} // anonymous namespace
static std::string
get_testdata(const std::string& name)
{
......@@ -196,6 +224,7 @@ enum class TestTwiceFlags : int
step2_rmw = 1<<5, // OFF/on. Only with both write, both uncomp.
step2_replace = 1<<6, // OFF/on. All step1_write samples overwritten.
step2_finalize = 1<<7, // off/ON. If on, use "Decimate" algorithm.
step2_fin_incr = 1<<10, // OFF/on. BuildIncremental used.
};
inline TestTwiceFlags
......@@ -227,6 +256,7 @@ std::ostream& operator<<(std::ostream& os, TestTwiceFlags value)
<< (((value & F::step2_rmw) != F::nothing) ? "F::step2_rmw | " : "")
<< (((value & F::step2_replace) != F::nothing) ? "F::step2_replace | " : "")
<< (((value & F::step2_finalize) != F::nothing) ? "F::step2_finalize | " : "")
<< (((value & F::step2_fin_incr) != F::nothing) ? "F::step2_fin_incr | " : "")
<< "F::nothing";
}
}
......@@ -371,11 +401,39 @@ do_test_reopen(const std::string& filename, TestTwiceFlags flags)
(expect_15 * -15 * -15) +
(expect_10 * -10 * -10);
//const double expect_stat_min = !expect_finalized ? 0 :
// (expect_15 + expect_10 == 0 ? 0 : -15);
//const double expect_stat_max = !expect_finalized ? 0 :
// (expect_41 + expect_42 == 0 ? 0 : 42);
const double expect_stat_min = !expect_finalized ? 0 :
(expect_15 + expect_10 == 0 ? 0 : -15);
// Note the quirk with incremental finalize. If there were ever
// some 42's written that number remains in the statistical
// range even if those samples were removed later.
// Unless of course the caller asked for an incremental finalize
// but didn't get it because step 1 did not finalize.
const double expect_stat_max = !expect_finalized ? 0 :
(flagset(TestTwiceFlags::step1_write) &&
flagset(TestTwiceFlags::step1_finalize) &&
flagset(TestTwiceFlags::step2_replace) &&
flagset(TestTwiceFlags::step2_finalize) &&
flagset(TestTwiceFlags::step2_fin_incr)) ? 42 :
(expect_41 + expect_42 == 0 ? 0 : 42);
// Even if asking for incremental build, might not get it.
// TODO-@@@ Verify by hand the expected block count.
// Not useful right now because the algorithms are still
// under development.
SilentProgress p;
const bool any_step2_write =
((flags & (TestTwiceFlags::step2_write |
TestTwiceFlags::step2_rmw |
TestTwiceFlags::step2_replace))
!= TestTwiceFlags::nothing);
const std::int64_t expect_brickrw =
!flagset(TestTwiceFlags::step2_finalize) ? 0 :
!flagset(TestTwiceFlags::step1_finalize) ? 88 :
!any_step2_write ? 0 :
!flagset(TestTwiceFlags::step2_fin_incr) ? 88 :
any_step2_write ? 46 :
0;
ZgyWriterArgs firstargs = ZgyWriterArgs()
.iocontext(Test_Utils::default_context())
......@@ -460,7 +518,9 @@ do_test_reopen(const std::string& filename, TestTwiceFlags flags)
// Decide whether the end result will have statistics and metadata.
if (flagset(TestTwiceFlags::step2_finalize)) {
writer->finalize(std::vector<OpenZGY::DecimationType>{OpenZGY::DecimationType::Decimate}, nullptr);
writer->finalize(std::vector<OpenZGY::DecimationType>{OpenZGY::DecimationType::Decimate},
std::ref(p),
flagset(TestTwiceFlags::step2_fin_incr) ? FinalizeAction::BuildIncremental : FinalizeAction::BuildFull);
writer->close();
}
else {
......@@ -560,12 +620,11 @@ do_test_reopen(const std::string& filename, TestTwiceFlags flags)
ok = TEST_EQUAL(actual_stat.cnt, expect_stat_cnt) && ok;
ok = TEST_EQUAL_FLOAT(actual_stat.sum, expect_stat_sum, 0.1) && ok;
ok = TEST_EQUAL_FLOAT(actual_stat.ssq, expect_stat_ssq, 0.1) && ok;
// TODO-@@@ FAILS because this is one place where incremental rebuild
// does make a difference. The range can only grow. Actually I should
// modify this test to try out both incremental and full rebuild,
// at least for some of the cases.
//FAILS: ok = TEST_EQUAL(actual_stat.min, expect_stat_min) && ok;
//FAILS: ok = TEST_EQUAL(actual_stat.max, expect_stat_max) && ok;
ok = TEST_EQUAL(actual_stat.min, expect_stat_min) && ok;
ok = TEST_EQUAL(actual_stat.max, expect_stat_max) && ok;
ok = TEST_EQUAL(p.total(), expect_brickrw) && ok;
ok = TEST_EQUAL(p.done(), p.total()) && ok;
// Not testing histogram min/max because it is rather unclear
// when it will be too wide (because samples were erased) or too
// narrow (due to incremental finalize). What I can verify is
......@@ -1323,6 +1382,7 @@ test_reopen_track_changes()
{
// Create empty file with 6 lod levels
SilentProgress p;
auto writer = OpenZGY::IZgyWriter::open
(ZgyWriterArgs()
.iocontext(Test_Utils::default_context())
......@@ -1334,11 +1394,14 @@ test_reopen_track_changes()
const float fortytwo{42};
writer->writeconst(size3i_t{0,0,0}, writer->size(), &fortytwo);
writer->finalize(std::vector<OpenZGY::DecimationType>
{OpenZGY::DecimationType::Average});
{OpenZGY::DecimationType::Average}, std::ref(p));
writer->close();
TEST_EQUAL(p.total(), 2142 + 388);
TEST_EQUAL(p.done(), p.total());
}
{
SilentProgress p;
auto writer = OpenZGY::IZgyWriter::reopen
(ZgyWriterArgs()
.iocontext(Test_Utils::default_context())
......@@ -1354,8 +1417,10 @@ test_reopen_track_changes()
// will need to be re-read assuming the simpler genlod algorithm.
writer->finalize(std::vector<OpenZGY::DecimationType>
{OpenZGY::DecimationType::Average},
nullptr, FinalizeAction::BuildIncremental);
std::ref(p), FinalizeAction::BuildIncremental);
writer->close();
TEST_EQUAL(p.total(), 322); // difficult to verify by hand.
TEST_EQUAL(p.done(), p.total());
}
{
......@@ -1508,6 +1573,64 @@ test_reopen()
TEST_EQUAL(runs, 5+32+12);
}
/**
* Extensive test of the reopen() feature.
* Requesting incremental build of derived data.
*/
static void
test_reopen_incr()
{
typedef TestTwiceFlags F;
LocalFileAutoDelete lad("reopen_incr.zgy");
// Run roughly the same tests as in test_reopen().
// Skip those that don't have both finalize step1 and step2 turned on,
// and skip compressed files because incremental is not allowed there.
// Paranoia: Ideally I'd like to test a few int16_t cubes here as well
// because histograms for those work slightly differently.
TEST_CHECK(do_test_reopen(lad.name(),
F::step1_write |
F::step2_write |
F::step2_rmw |
F::step2_fin_incr | F::step2_finalize));
TEST_CHECK(do_test_reopen(lad.name(),
F::step1_compress |
F::step2_compress |
F::step1_write |
F::step2_write |
F::step2_fin_incr |F::step2_finalize));
TEST_CHECK(do_test_reopen(lad.name(),
F::step1_vt_int8 | // Only tested here.
F::step2_nometa | // Only tested here.
F::step1_write |
F::step2_write |
F::step2_rmw |
F::step2_fin_incr | F::step2_finalize));
// here will be 32 tests for uncompressed and 12 for compressed.
int runs = 3; // for the tests above.
for (int ii=0; ii<128; ++ii) {
TestTwiceFlags flags = static_cast<TestTwiceFlags>(ii);
flags = flags | TestTwiceFlags::step2_fin_incr | TestTwiceFlags::step2_finalize;
// Only append is allowed for compressed files. For that reason a double
// finalize is also bad because it would need to replaces lowres bricks.
if ((flags & (F::step1_compress | F::step2_compress)) != F::nothing)
continue;
try {
++runs;
TEST_CHECK_(do_test_reopen(lad.name(), flags), "case 0x%02x", ii);
}
catch(const std::exception& ex) {
TEST_CHECK_(false, "case 0x%02x raised %s", ii, ex.what());
}
}
// Just in case I messed up the "valid" test.
TEST_EQUAL(runs, 3+32);
}
/**
* Opening an empty file created by the old ZGY accessor has some
* challenges with respect to alignment.
......@@ -1615,6 +1738,7 @@ public:
register_test("reopen.empty", test_reopen_empty);
register_test("reopen.bad_histogram", test_reopen_bad_histogram);
register_test("reopen.reopen", test_reopen);
register_test("reopen.reopen_incr", test_reopen_incr);
register_test("reopen.zgypublic", test_reopen_zgypublic);
register_test("reopen.track_changes", test_reopen_track_changes);
#ifdef HAVE_SD
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment