Commit 43f2971e authored by Paal Kvamme's avatar Paal Kvamme
Browse files

Checkpoint for the intricate is_finalized test.

parent 5e3b74e2
......@@ -1192,15 +1192,7 @@ public:
//std::cout << "Was nlods " << nl1 << " code " << code1
// << " is nlods " << nl2 << " code " << code2 << std::endl;
// 4. Change the version number to 4, to signify that the old
// ZGY reader should no try to access this file. Low resolution
// bricks would appear to be present but have all zero samples.
// TODO-Low: Enhancement: With update support, when a file is
// later opened and properly finalized it may be possible to
// change the version number back to 3. Issue goes away when
// we upgrade the ZGY file format.
_meta_rw->fh().set_version
(std::max(_meta_rw->fh().version(), (std::uint32_t)4));
// 4. Change the version number to 4 is now done in _close_internal().
}
/**
......@@ -1435,6 +1427,43 @@ public:
this->_dirty = false;
}
/**
* Does the file use features the old ZGY-Public doesn't understand?
*
* ZFP-compressed data is definitely out. As is missing lowres.
* Testing for lowres is trivial but the old accesor just doesn't.
* Missing histogram and/or statistics can probably be tolerated.
*
* Tiny files consisting of just a single brick triggers several corner
* cases. In practice we shold never see such files but ideally they
* ought to be handled.
*
* The tiny files have no lowres bricks so they doesn't need a finalize
* for lowres. But it is needed for statistics. Use relaxed checks
* treat as finalized in the context of checking for v4. Treat as
* not finalized when deciding to allow re-open of a compressed
* finalized file. Tiny files should never have incremental rebuild.
*/
bool _has_v4_features()
{
const InternalZGY::IHistHeaderAccess& hh = this->_meta_rw->hh();
const InternalZGY::IInfoHeaderAccess& ih = this->_meta_rw->ih();
const std::int32_t usable_lods = InternalZGY::LookupTable::usableBrickLOD
(ih.lodsizes(),
ih.brickoffsets(),
this->_meta_rw->blup().lup(),
this->_meta_rw->blup().lupend());
const bool has_compression = InternalZGY::LookupTable::hasBrickCompression
(this->_meta_rw->blup().lup(),
this->_meta_rw->blup().lupend());
//const bool has_statistics =
// (ih.scnt() != 0 && ih.smin() <= ih.smax() &&
// hh.samplecount() != 0 && hh.minvalue() <= hh.maxvalue());
const bool has_lowres = (ih.nlods() == usable_lods);
const bool has_v4_features = has_compression || !has_lowres;
return has_v4_features;
}
/**
* \brief Flush the file to disk and close it.
*
......@@ -1453,6 +1482,9 @@ public:
if (!this->_fd || !this->_accessor_rw || !this->_meta_rw)
return; // looks like _close_internal alrady called.
// Prevent ZGY-Public from opening the file if appropriate.
this->_meta_rw->fh().set_version(_has_v4_features() ? 4 : 3);
if (!this->errorflag()) {
this->_meta_rw->flushMeta(this->_fd);
}
......
......@@ -484,6 +484,21 @@ LookupTable::usableBrickLOD(
(std::int32_t)lodsizes.size();
}
bool
LookupTable::hasBrickCompression(
const std::vector<std::uint64_t>& blup,
const std::vector<std::uint64_t>& bend)
{
constexpr std::int64_t bytesperbrick{1}; // Bogus, we only care about type.
for (std::size_t ix = 0; ix < blup.size(); ++ix) {
LookupTable::LutInfo info =
LookupTable::getBrickFilePositionFromIndex(ix, blup, bend, bytesperbrick);
if (info.status == BrickStatus::Compressed)
return true;
}
return false;
}
/**
* \brief Logically erase all low resolution bricks.
*
......
......@@ -157,6 +157,10 @@ public:
const std::vector<std::uint64_t>& blup,
const std::vector<std::uint64_t>& bend);
static bool hasBrickCompression(
const std::vector<std::uint64_t>& blup,
const std::vector<std::uint64_t>& bend);
static void setNoBrickLOD(
const std::vector<std::array<std::int64_t,3>>& lodsizes,
const std::vector<std::int64_t>& brickoffsets,
......
......@@ -2285,15 +2285,14 @@ ZgyInternalMeta::initFromReopen(const ZgyInternalWriterArgs& args_in, bool compr
// allowed to re-finalize it. That counts as an update. Need to catch
// this error early. If the application is allowed to start writing but
// not finalize at the end then the file would essentially be corrupted.
const std::vector<std::uint64_t>& blup = this->_blup->lup();
const std::vector<std::uint64_t>& bend = this->_blup->lupend();
const std::int64_t perbrick = ih.bytesperbrick();
for (std::size_t ii=0; ii<blup.size(); ++ii) {
if (LookupTable::getBrickFilePositionFromIndex
(ii, blup, bend, perbrick).status == BrickStatus::Compressed)
throw OpenZGY::Errors::ZgyUserError
("A finalized compressed file cannot be opened for update.");
}
// TODO-@@@-Medium: It should still be possible to open a compressed
// file for the sole purpose of changing annotation and world coords.
// So ideally the test should be deferred to the first write. But there
// are other problems such as also deferring the re-open segment.
if (InternalZGY::LookupTable::hasBrickCompression
(this->_blup->lup(), this->_blup->lupend()))
throw OpenZGY::Errors::ZgyUserError
("A finalized compressed file cannot be opened for update.");
}
}
......
......@@ -1171,7 +1171,7 @@ test_compress_noop()
std::shared_ptr<const FileStatistics> stats =
do_test_copy_slurp_8(filename, lad.name(), "Null");
TEST_CHECK(stats->fileVersion() != 3); // v4 set even if no actual compressed
TEST_CHECK(stats->fileVersion() == 3); // v4 only set if actual compressed
TEST_CHECK(stats->isCompressed() == false); // looked for compressed bricks.
TEST_CHECK(stats->fileSize() == 4*64*64*64*4); // header plus 3 normal bricks
TEST_CHECK(stats->brickNormalCount() == 3); // 1 lod0, 1 lod1, 1 lod2
......
......@@ -1743,6 +1743,318 @@ test_reopen_rwlod()
#endif
}
/**
* The short version: When OpenZGY saves a file that isn't finalized
* then it is flagged as v4 because the old reader cannot handle that.
* Just like compressed data will do. Also there will be no
* incremental finalize on the next close since there is no starting
* point. On the positive side, compressed files that are not
* finalized can still be re-opened and appended to.
*
* The long version: The above is complicated by the fact that
* finalize does more than one thing so there exists a "halfway
* finalized" state where low resolution data is present and
* statistics are not. Or vice versa. Or low resolution is missing
* because it is not needed for tiny files.
*
* The version number (3 or 4) should reflect the current state of the
* file, not its history. So e.g. a file that was v4 only due to
* missing low resolution data should be changed to v3 when finalized.
* The tests that determine the version number are similar to the tests
* that decide whether incremental builds are allowed and whether a
* compressed file may be re-opened.
*
* Why does this matter?
*
* Petrel creates an empty file that will be written to later. This
* file must not be finalized because that would prevent it from being
* later written with compressed data. And if the finalize was done
* with compression it would prevent even ubcompressed writes. If no
* compression at all is involved then a finalize at this point "only"
* a serious performance issue.
*
* So:
* - Create empty file not finalized. It will be version 4.
* Except the special single-brick case which will be v3
* because there isn't any lowres to be generated.
* It doesn't matter whether compression is requested;
* the file will be uncompressed because it has no data.
*
* Features:
* (ZFP) Has compressed data.
* (LOD) Has low resolution data (N/A for single brick file).
* (S/H) Has statistics and histogram (empty histogram due to wrong range fails)
*
* Behavior depending on the above:
* - Mark as V4 if ZFP || !LOD.
* - Not allowed to reopen if ZFP && LOD.
* - Track changes for BuildIncremental if LOD && S/H
*
* Behavior for single-brick file which cannot have LOD:
* - Mark as V4 if ZFP
* - Not allowed to reopen if ZFP.
* - Never track changes.
*
* Notes:
* - Applications using the old readers are assumed to handle missing
* statistics and histogram correctly, so there is no need to flag
* as v4 only for that purpose.
*
* - A single brick file cannot have lowres, this lack will not trigger
* the v4 flag either.
*
* - Allowing BuildIncremental has a stricter check. Err on the size
* of forcing a BuildFull where an incremental might have sufficed.
* Here, missing S/H makes us assume the file is not finalized yet.
* A single brick file always fails this test because incremental
* builds would be pointless. (test: See if the statistics can shrink).
*
* - Reopen of a compressed file is not allowed if finalized because the
* low resolution bricks (just like the full resolution) can only be
* written once. Reopen of a compressed single brick file is allowed
* for consistency even though it would be pointless. Not due to the
* double finalize but because if the single data brick is already
* written then there are no more fullres bricks to write. And if it
* is not written then the file cannot be compressed.
*/
static void
test_reopen_setversion()
{
LocalFileAutoDelete lad("reopen_setversion.zgy");
typedef OpenZGY::IZgyWriter::size3i_t size3i_t;
using namespace OpenZGY;
constexpr double inf{std::numeric_limits<double>::infinity()};
{
// Empty file finalized, request compression
std::shared_ptr<OpenZGY::IZgyWriter> writer =
OpenZGY::IZgyWriter::open(ZgyWriterArgs()
.filename(lad.name())
.zfp_compressor(99)
.size(33, 128, 92));
writer->close();
}
{
// In spite of compression being requested there wasn't actually
// any compressed data written. So the version should still be 3.
std::shared_ptr<OpenZGY::IZgyReader> reader =
OpenZGY::IZgyReader::open(lad.name());
TEST_EQUAL(reader->nlods(), 2);
TEST_EQUAL(reader->filestats()->fileVersion(), 3);
TEST_EQUAL(reader->statistics().cnt, 33*128*92);
TEST_EQUAL(reader->statistics().min, 0);
TEST_EQUAL(reader->statistics().max, 0);
TEST_EQUAL(reader->histogram().samplecount, reader->statistics().cnt);
// Histogram range is unspecified when there is just a single value.
//TEST_EQUAL(reader->histogram().minvalue, reader->statistics().min);
//TEST_EQUAL(reader->histogram().maxvalue, reader->statistics().max);
}
{
// NEW -> Empty file not finalized
std::shared_ptr<OpenZGY::IZgyWriter> writer =
OpenZGY::IZgyWriter::open(ZgyWriterArgs()
.filename(lad.name())
.size(33, 128, 92));
writer->close_incomplete();
}
{
// Check 1 lod i.e. only fullres, and v4 for that reason.
std::shared_ptr<OpenZGY::IZgyReader> reader =
OpenZGY::IZgyReader::open(lad.name());
TEST_EQUAL(reader->nlods(), 1);
TEST_EQUAL(reader->filestats()->fileVersion(), 4);
TEST_EQUAL(reader->statistics().cnt, 0);
TEST_EQUAL(reader->statistics().min, inf);
TEST_EQUAL(reader->statistics().max, -inf);
TEST_EQUAL(reader->histogram().samplecount, reader->statistics().cnt);
}
{
// REOPEN -> Empty file finalized
std::shared_ptr<OpenZGY::IZgyWriter> writer =
OpenZGY::IZgyWriter::reopen(ZgyWriterArgs()
.filename(lad.name()));
writer->close();
}
{
// Should now be ZGY-Public compatible. Stats show only zeros.
std::shared_ptr<OpenZGY::IZgyReader> reader =
OpenZGY::IZgyReader::open(lad.name());
TEST_EQUAL(reader->nlods(), 2);
TEST_EQUAL(reader->filestats()->fileVersion(), 3);
TEST_EQUAL(reader->statistics().cnt, 33*128*92);
TEST_EQUAL(reader->statistics().min, 0);
TEST_EQUAL(reader->statistics().max, 0);
TEST_EQUAL(reader->histogram().samplecount, reader->statistics().cnt);
}
{
// NEW -> Start over, make a file with compressed data not finalized.
std::shared_ptr<OpenZGY::IZgyWriter> writer =
OpenZGY::IZgyWriter::open(ZgyWriterArgs()
.filename(lad.name())
.zfp_compressor(99)
.size(33, 128, 92));
std::vector<float> data(64*64*64, 1000.0f);
data[0] = 42.0f;
writer->write(size3i_t{0,0,0}, size3i_t{64,64,64}, data.data());
writer->close_incomplete();
}
{
// Check 1 lod i.e. only fullres, and v4 for two reasons.
std::shared_ptr<OpenZGY::IZgyReader> reader =
OpenZGY::IZgyReader::open(lad.name());
TEST_EQUAL(reader->nlods(), 1);
TEST_EQUAL(reader->filestats()->fileVersion(), 4);
TEST_EQUAL(reader->statistics().cnt, 0);
TEST_EQUAL(reader->statistics().min, inf);
TEST_EQUAL(reader->statistics().max, -inf);
TEST_EQUAL(reader->histogram().samplecount, reader->statistics().cnt);
// Histogram range might be the value ranges seen till now,
// but this is an implementation detail since the histogram
// is still empty.
TEST_EQUAL(reader->histogram().minvalue, 42.0);
TEST_EQUAL(reader->histogram().maxvalue, 1000.0);
}
{
// REOPEN -> Write more compressed data and then finalize.
std::shared_ptr<OpenZGY::IZgyWriter> writer =
OpenZGY::IZgyWriter::reopen(ZgyWriterArgs()
.filename(lad.name())
.zfp_compressor(99));
const float fifteen{1500.0f};
writer->write(size3i_t{0,64,0}, size3i_t{1,1,1}, &fifteen);
writer->close();
}
{
// Check all lods, still v4 because of the compression.
std::shared_ptr<OpenZGY::IZgyReader> reader =
OpenZGY::IZgyReader::open(lad.name());
TEST_EQUAL(reader->nlods(), 2);
TEST_EQUAL(reader->filestats()->fileVersion(), 4);
TEST_EQUAL(reader->statistics().cnt, 33*128*92);
TEST_EQUAL(reader->statistics().min, 0.0);
TEST_EQUAL(reader->statistics().max, 1500.0);
TEST_EQUAL(reader->histogram().samplecount, reader->statistics().cnt);
TEST_EQUAL(reader->histogram().minvalue, reader->statistics().min);
TEST_EQUAL(reader->histogram().maxvalue, reader->statistics().max);
}
{
// NEW -> Start over, tiny file (one brick), written not finalized.
// Not finalized means the statistics and histogram are unset.
// Low resolution data is N/A. For the v4 test the lowres is treated
// as if present i.e. not v4.
std::shared_ptr<OpenZGY::IZgyWriter> writer =
OpenZGY::IZgyWriter::open(ZgyWriterArgs()
.filename(lad.name())
.size(33, 22, 11));
const float fortytwo{42.0f};
writer->write(size3i_t{1,2,3}, size3i_t{1,1,1}, &fortytwo);
writer->close_incomplete();
}
{
std::shared_ptr<OpenZGY::IZgyReader> reader =
OpenZGY::IZgyReader::open(lad.name());
TEST_EQUAL(reader->nlods(), 1);
TEST_EQUAL(reader->filestats()->fileVersion(), 3);
TEST_EQUAL(reader->statistics().cnt, 0);
TEST_EQUAL(reader->statistics().min, inf);
TEST_EQUAL(reader->statistics().max, -inf);
TEST_EQUAL(reader->histogram().samplecount, 0);
// Histogram range might be the value ranges seen till now,
// but this is an implementation detail since the histogram
// is still empty.
TEST_EQUAL(reader->histogram().minvalue, 0.0);
TEST_EQUAL(reader->histogram().maxvalue, 42.0);
}
{
// REOPEN -> Finalize the tiny file.
// One twist: Explicitly request an incremental build.
// The code should switch to a full build since there
// was no previous finalize but that might not be
// obvious at this point.
// Another thing is that incremental rebuild makes
// absolutely no sense in this case.
std::shared_ptr<OpenZGY::IZgyWriter> writer =
OpenZGY::IZgyWriter::reopen(ZgyWriterArgs()
.filename(lad.name()));
writer->finalize(std::vector<OpenZGY::DecimationType>
{OpenZGY::DecimationType::Average},
nullptr, FinalizeAction::BuildIncremental);
writer->close();
}
{
std::shared_ptr<OpenZGY::IZgyReader> reader =
OpenZGY::IZgyReader::open(lad.name());
TEST_EQUAL(reader->nlods(), 1);
TEST_EQUAL(reader->filestats()->fileVersion(), 3);
// Off topic: This also verifies that statistics and histogram were
// created even though genlod() didn't actually have anything to do.
TEST_EQUAL(reader->statistics().cnt, 33*22*11);
TEST_EQUAL(reader->statistics().min, 0);
TEST_EQUAL(reader->statistics().max, 42);
TEST_EQUAL(reader->histogram().samplecount, reader->statistics().cnt);
TEST_EQUAL(reader->histogram().minvalue, reader->statistics().min);
TEST_EQUAL(reader->histogram().maxvalue, reader->statistics().max);
}
{
// NEW -> Start over, as above but compressed data. tiny file (one brick),
// written compressed not finalized. Statistics and histogram are unset.
// Low resolution data is N/A. For the v4 test the lowres is treated
// as if present i.e. not v4. For compressed re-open lowres is
// treated as not present i.e. reopen ok.
std::shared_ptr<OpenZGY::IZgyWriter> writer =
OpenZGY::IZgyWriter::open(ZgyWriterArgs()
.filename(lad.name())
.zfp_compressor(99)
.size(33, 22, 11));
const float fortytwo{42.0f};
writer->write(size3i_t{1,2,3}, size3i_t{1,1,1}, &fortytwo);
writer->close_incomplete();
}
{
// REOPEN -> Finalize the compressed tiny file with compressed lowres.
std::shared_ptr<OpenZGY::IZgyWriter> writer =
OpenZGY::IZgyWriter::reopen(ZgyWriterArgs()
.zfp_compressor(99)
.filename(lad.name()));
writer->finalize(std::vector<OpenZGY::DecimationType>
{OpenZGY::DecimationType::Average},
nullptr, FinalizeAction::BuildIncremental);
writer->close();
}
#if 0 // Don't really care either way. For consistency maybe throw,
{
// REOPEN -> Finalize the compressed tiny file one more time.
// Normally this is not allowed, but since the finalize does not
// actually store any data in this case it should be ok.
std::shared_ptr<OpenZGY::IZgyWriter> writer =
OpenZGY::IZgyWriter::reopen(ZgyWriterArgs()
.zfp_compressor(99)
.filename(lad.name()));
writer->finalize(std::vector<OpenZGY::DecimationType>
{OpenZGY::DecimationType::Average},
nullptr, FinalizeAction::BuildIncremental);
writer->close();
}
#endif
}
/**
* Opening an empty file created by the old ZGY accessor has some
* challenges with respect to alignment.
......@@ -2138,6 +2450,7 @@ public:
register_test("reopen.incr", test_reopen_incr);
register_test("reopen.keepopen", test_reopen_keepopen);
register_test("reopen.rwlod", test_reopen_rwlod);
register_test("reopen.setversion", test_reopen_setversion);
register_test("reopen.zgypublic", test_reopen_zgypublic);
register_test("reopen.track_changes", test_reopen_track_changes);
#ifdef HAVE_SD
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment