Commit 0c08043e authored by Paal Kvamme's avatar Paal Kvamme
Browse files

Checkpoint implementing partial rebuild of low resolution data.

parent 3496b1cf
......@@ -827,6 +827,102 @@ ZgyInternalBulk::readToNewBuffer(
return result;
}
/**
* \brief Mark bricks as dirty.
*
* \details
* Mark all affected bricks as dirty, both those written now and those
* at a higher lod level that will need to be re-generated.
* - lod=0, flag bit 0x01: This fullres brick written by the user.
* - lod>0, flag bit 0x01: Brick depends on a fullres that was written.
* - lod>0, flag bit 0x02: This lowres brick was written by genlod
* The second one is redundant as it could be calculated on the fly,
* and the third one might not be useful outside of debugging.
*
* Unlike trackedBricksDirty() the region has already been converted
* to a list of bricks but the numbers ate still in samples not bricks.
*/
void
ZgyInternalBulk::_trackedBricksSetDirty(
const std::vector<index3_t>& work,
int32_t lod)
{
if (_modified_bricks.size() != 0) {
const IInfoHeaderAccess& ih = this->_metadata->ih();
const index3_t bs = ih.bricksize();
const std::int32_t nlods = ih.nlods();
int new_dirty{0}, old_dirty{0};
// number of lods we will build, not the current count.
for (const index3_t& pos : work) {
int ii = pos[0]/bs[0], jj = pos[1]/bs[1], kk = pos[2]/bs[2];
for (std::int32_t mylod = lod; mylod < nlods; ++mylod) {
const std::int64_t bix = LookupTable::getBrickLookupIndex
(ii, jj, kk, mylod, ih.lodsizes(), ih.brickoffsets());
if (bix >= (std::int64_t)_modified_bricks.size())
throw OpenZGY::Errors::ZgyInternalError("overflow _modified_bricks.");
std::uint8_t flagval = (lod==0) ? 1 : 2;
if ((_modified_bricks[bix] & flagval) == 0)
++new_dirty;
else
++old_dirty;
_modified_bricks[bix] |= flagval;
ii /= 2;
jj /= 2;
kk /= 2;
}
}
if (_logger(5)) {
auto clean = std::count_if(_modified_bricks.begin(),
_modified_bricks.end(),
[](std::uint8_t x){return (x&1) == 0;});
_logger(5, std::stringstream()
<< "dirty: " << new_dirty << " new, " << old_dirty
<< " already soiled, " << clean << "/"
<< _modified_bricks.size() << " still clean");
}
}
}
/**
* \brief Check if any brick in the region has been modified.
*
* \details
* Used by incremental low resolution computation.
* The input start and size is in sample coordinates.
*/
std::uint8_t
ZgyInternalBulk::trackedBricksDirty(
const std::array<std::int64_t,3>& start,
const std::array<std::int64_t,3>& size,
int32_t lod) const
{
if (_modified_bricks.size() == 0)
return 0xFF; // Or maybe treat this as an error.
const IInfoHeaderAccess& ih = this->_metadata->ih();
std::uint8_t result{0};
const std::array<std::int64_t,3> bs = ih.bricksize();
const std::array<std::int64_t,3> bstart
{start[0] / bs[0], start[1] / bs[1], start[2] / bs[2]};
const std::array<std::int64_t,3> bend
{(start[0] + size[0] + bs[0] - 1) / bs[0],
(start[1] + size[1] + bs[1] - 1) / bs[1],
(start[2] + size[2] + bs[2] - 1) / bs[2]};
for (std::int64_t ii = bstart[0]; ii < bend[0]; ++ii) {
for (std::int64_t jj = bstart[1]; jj < bend[1]; ++jj) {
for (std::int64_t kk = bstart[2]; kk < bend[2]; ++kk) {
const std::int64_t bix = LookupTable::getBrickLookupIndex
(ii, jj, kk, lod, ih.lodsizes(), ih.brickoffsets());
// "cannot happen" because getBrickLookupIndex() should have checked.
if (bix >= (std::int64_t)_modified_bricks.size())
throw OpenZGY::Errors::ZgyInternalError("overflow _modified_bricks.");
result |= _modified_bricks[bix];
}
}
}
return result;
}
/**
* Mitigate the problem that ZgyInternalBulk is used both for reading
* and writing, making it difficult to keep track of thread safety.
......@@ -1866,39 +1962,7 @@ ZgyInternalBulk::_writeAlignedRegion(
// Mark all affected bricks as dirty, both those written now and those
// at a higher lod level that will need to be re-generated.
if (_modified_bricks.size() != 0) {
int new_dirty{0}, old_dirty{0};
// Possible lod count, not affected by whether lowres is available.
const std::int32_t nlods = this->_metadata->ih().nlods();
for (const index3_t& pos : work) {
int ii = pos[0]/bs[0], jj = pos[1]/bs[1], kk = pos[2]/bs[2];
for (std::int32_t mylod = lod; mylod < nlods; ++mylod) {
const std::int64_t bix = LookupTable::getBrickLookupIndex
(ii, jj, kk, mylod,
this->_metadata->ih().lodsizes(),
this->_metadata->ih().brickoffsets());
if (bix >= (std::int64_t)_modified_bricks.size())
throw OpenZGY::Errors::ZgyInternalError("overflow _modified_bricks.");
if ((_modified_bricks[bix] & 1) == 0)
++new_dirty;
else
++old_dirty;
_modified_bricks[bix] |= 1;
ii /= 2;
jj /= 2;
kk /= 2;
}
}
if (_logger(5)) {
auto clean = std::count_if(_modified_bricks.begin(),
_modified_bricks.end(),
[](std::uint8_t x){return (x&1) == 0;});
_logger(5, std::stringstream()
<< "dirty: " << new_dirty << " new, " << old_dirty
<< " already soiled, " << clean << "/"
<< _modified_bricks.size() << " still clean");
}
}
this->_trackedBricksSetDirty(work, lod);
// If the application asked to write just a single brick than we can
// skip the multithreading logic and in most cases skip the copying
......
......@@ -134,9 +134,17 @@ public: // actually internal
const std::vector<std::uint8_t>& trackedBricks() const {
return _modified_bricks;
}
std::uint8_t trackedBricksDirty(
const std::array<std::int64_t,3>& start,
const std::array<std::int64_t,3>& size,
int32_t lod) const;
private:
void _trackedBricksSetDirty(
const std::vector<index3_t>& work,
int32_t lod);
bool _logger(int priority, const std::string& ss = std::string()) const;
bool _logger(int priority, const std::ios& ss) const;
......
......@@ -117,6 +117,9 @@ GenLodBase::GenLodBase(
throw OpenZGY::Errors::ZgyInternalError("GenLod error in nlods computation");
this->_nlods = nlods;
this->_total = total;
// TODO-@@@-Low: in incremental builds "total" will be incorrect.
// probably need a proper dry run to get the actual total reads and
// writes. That would be invoked by call().
this->_report(nullptr);
}
......@@ -140,6 +143,21 @@ GenLodBase::_report(const DataBuffer* data)
throw OpenZGY::Errors::ZgyAborted("Computation of low resolution data was aborted");
}
/**
* Return true if the requested data exists on the file or (for plan D)
* in the application. This is always the case for LOD 0. The method
* may be overridden to implement imcremental rebuild where some
* low reaolution data might also be available on the file.
*/
bool
GenLodBase::_canread(
std::int32_t lod,
const std::array<std::int64_t,3>& /*pos*/,
const std::array<std::int64_t,3>& /*size*/)
{
return lod==0;
}
/**
* This is a stub that must be redefined except for low level unit tests.
* Read a block from the ZGY file (plans B and C) or the application
......@@ -293,6 +311,11 @@ GenLodImpl::_accumulateT(const std::shared_ptr<const DataBuffer>& data_in)
/**
* Keep a running tally of statistics and histogram.
*
* TODO-@@@-Low: Performance: When doing an incremental
* build this data will be discarded so we might as well
* not collect it in the first place.
* Add a bool GenLodImpl::_incremental
*/
void
GenLodImpl::_accumulate(const std::shared_ptr<const DataBuffer>& data)
......@@ -319,6 +342,16 @@ GenLodImpl::_accumulate(const std::shared_ptr<const DataBuffer>& data)
* 2* bs * 2^N where bs is the file's brick size in that dimension,
* Clipped to the survey boundaries. This might give an empty result.
*
* If doing an incremental build the minimum size is also the maximum
* size because the buffer size determines the granularity of the
* "dirty" test and that needs to be as small as possible.
* So, size will end up as 2x2 brick columns or less at survey edge.
* The current code does this also for full rebuilds because it is
* simpled (no need to figure out the available memory size) and also
* unlikely to make a noticeable difference in performance. Except
* _possibly_ if the file is optimized for slice access and resides
* on the cloud.
*
* TODO-Performance: Allow the application to configure how much memory
* we are allowed to use. Increase the block size accordingly.
* Larger bricks might help the bulk layer to become more efficient.
......@@ -373,9 +406,20 @@ GenLodImpl::_calculate(const std::array<std::int64_t,3>& readpos_in, std::int32_
std::shared_ptr<const DataBuffer> data;
if(readlod == 0) {
data = this->_read(readlod,readpos,readsize);
// Fullres bricks are always read, not calculated.
data = this->_read(readlod, readpos, readsize);
this->_accumulate(data);
}
else if (this->_canread(readlod, readpos, readsize)) {
// Lowres bricks can sometimes be read if building incrementally.
// Note that the code always calculates full brick-columns to
// avoid r/m/w cycles. So if any of the 4 brick columns we
// depend on are dirty then all 4 needs to be read or computed.
// And the converse: If _canread() returns true then *all*
// requested data is already in file so there is no write back.
// Just like in the LOD 0 case.
data = this->_read(readlod, readpos, readsize);
}
else {
const std::array<std::int64_t,3> bs = this->_bricksize;
std::array<std::int64_t,3> offsets[4] =
......@@ -391,15 +435,13 @@ GenLodImpl::_calculate(const std::array<std::int64_t,3>& readpos_in, std::int32_
// could also have consolidated the 4 lod0 read requests but that
// would mean quite a bit of refactoring.
// Worry: The assumption that _write is a no-op is important,
// so it probably needs to be checked to avoid paralell writes.
// so it probably needs to be checked to avoid parallel writes.
// Alternatively, and I suspect this will be more difficult, the code
// might be refactored so the bulk layer sees just a single request
// that is 4x larger. The bulk layer can then do multi threading itself.
// The reason this is better than the first approach is that the bulk
// layer in the cloud case might be able to consolidate more bricks.
// On the flip size, while I plan to implement the cloud access case
// I might not actually be doing the split and parallelize.
// Or, can I just check if readlod==1 make just one call and skip paste?
// There are probably some more survey-edge cases I need to handle then.
......@@ -417,8 +459,16 @@ GenLodImpl::_calculate(const std::array<std::int64_t,3>& readpos_in, std::int32_
std::shared_ptr<DataBuffer> result;
if (readlod == this->_nlods - 1) {
result = nullptr; // Caller will discard it anyway.
if (this->_done != this->_total)
throw OpenZGY::Errors::ZgyInternalError("GenLodImpl: Missing or excess data.");
if (this->_done != this->_total) {
#if 0 // TODO-@@@ need a dry run to get required read/write count.
throw OpenZGY::Errors::ZgyInternalError
("GenLodImpl: Expected " + std::to_string(this->_total) +
" reads and writes but saw " + std::to_string(this->_done) + ".");
#else
this->_done = this->_total;
this->_report(nullptr);
#endif
}
}
else if (data->isScalar()) {
result = DataBuffer::makeScalarBuffer3d
......@@ -437,7 +487,8 @@ GenLodImpl::_calculate(const std::array<std::int64_t,3>& readpos_in, std::int32_
if (this->_verbose)
std::cout << "@" << _prefix(readlod)
<< "calculate returns(lod="
<< readlod+1 << ", pos=" << readpos
<< readlod+1 << ", pos=" << readpos << " / 2"
<< ", size=" << writesize
<< ", data=" << data->toString() << ")\n";
return result;
}
......@@ -739,6 +790,29 @@ GenLodC::GenLodC(
<< "\n";
}
/**
* See base class for details.
*/
bool
GenLodC::_canread(
std::int32_t lod,
const std::array<std::int64_t,3>& pos,
const std::array<std::int64_t,3>& size)
{
if (lod == 0) {
return true; // Fullres always read from file.
}
else if (this->_accessor->trackedBricks().size() == 0) {
return false; // Not tracking dirty bits => no BuildIncremental.
}
else if ((this->_accessor->trackedBricksDirty(pos, size, lod) & 0x01) != 0) {
return false; // Something dirty somewhere in the input.
}
else {
return true; // Hurra! We can read from file instead of recomputing,
}
}
/**
* See base class for details.
*/
......
......@@ -81,6 +81,8 @@ public:
const std::function<bool(std::int64_t,std::int64_t)>& progress,
bool verbose);
protected:
virtual bool _canread(
std::int32_t lod, const index3_t& pos, const index3_t& size);
virtual std::shared_ptr<DataBuffer> _read(
std::int32_t lod, const index3_t& pos, const index3_t& size);
virtual void _write(
......@@ -156,6 +158,8 @@ public:
const std::function<bool(std::int64_t,std::int64_t)>& progress,
bool verbose);
protected:
bool _canread(
std::int32_t lod, const index3_t& pos, const index3_t& size) override;
std::shared_ptr<DataBuffer> _read(
std::int32_t lod, const index3_t& pos, const index3_t& size) override;
void _write(
......
......@@ -1293,6 +1293,95 @@ test_reopen_empty()
}
}
/**
* Meant to run manually after enabling some debug logging in
* ZgyWriter::_create_serived() but has some use even without the
* manual verification. The test focuses on how the library keeps
* track of dirty bricks.
*/
static void
test_reopen_track_changes()
{
typedef OpenZGY::IZgyWriter::size3i_t size3i_t;
LocalFileAutoDelete lad("reopen_track_changes.zgy");
// Survey is slightly larger than 16x16 bricks so it triggers
// the close-to-edge logic. Vertically it is much smaller
// than horizontally which might also cause issues.
// Brick size is 16 x 32 x 32 just to make things interesting,
// inline: 16 full bricks @ 16 lines/brick plus 5 lines = 261
// xline: 17 full bricks @ 32 lines/brick plus 3 lines = 547
// time: 6 full bricks @ 32 samples/brick plus 10 = 202
//
// LOD0: 17 x 18 x 7 261 x 547 x 202
// LOD1: 9 x 9 x 4 131 x 274 x 101
// LOD2: 5 x 5 x 2 66 x 137 x 51
// LOD3: 3 x 3 x 1 33 x 69 x 26
// LOD4: 2 x 2 x 1 17 x 35 x 13
// LOD5: 1 x 1 x 1 9 x 18 x 7
{
// Create empty file with 6 lod levels
auto writer = OpenZGY::IZgyWriter::open
(ZgyWriterArgs()
.iocontext(Test_Utils::default_context())
.filename(lad.name())
.bricksize(16,32,32)
.size(261, 547, 202));
if (!TEST_CHECK(writer != nullptr))
return;
const float fortytwo{42};
writer->writeconst(size3i_t{0,0,0}, writer->size(), &fortytwo);
writer->finalize(std::vector<OpenZGY::DecimationType>
{OpenZGY::DecimationType::Average});
writer->close();
}
{
auto writer = OpenZGY::IZgyWriter::reopen
(ZgyWriterArgs()
.iocontext(Test_Utils::default_context())
.filename(lad.name()));
if (!TEST_CHECK(writer != nullptr))
return;
const float thirteen{13}, seven{7};
// One single brick is dirty.
writer->writeconst(size3i_t{0,32,64}, size3i_t{16,32,32}, &thirteen);
// Four brick-columns of 7 bricks, because writes are not aligned
writer->writeconst(size3i_t{241,513,0}, size3i_t{16,32,202}, &seven);
// A total of 29 bricks are dirty, and 5 brick-columns at LOD0
// will need to be re-read assuming the simpler genlod algorithm.
writer->finalize(std::vector<OpenZGY::DecimationType>
{OpenZGY::DecimationType::Average},
nullptr, FinalizeAction::BuildIncremental);
writer->close();
}
{
auto reader = OpenZGY::IZgyReader::open
(lad.name(), Test_Utils::default_context());
if (!TEST_CHECK(reader != nullptr))
return;
TEST_EQUAL(reader->nlods(), 6);
TEST_EQUAL(reader->statistics().cnt, 261*547*202);
TEST_EQUAL(reader->statistics().sum,
42*(261*547*202 - 16*32*32 - 16*32*202) +
13*(16*32*32) + 7*(16*32*202));
// Check lod1 to verify that at least the dirty data caused lowres
// to be re-calculated. To also verify that the code din't do more
// than it needed then some logging needs to be turned on.
float data1{0};
reader->read(size3i_t{0,16,32}, size3i_t{1,1,1}, &data1, /*lod=*/1);
TEST_EQUAL_FLOAT(data1, 13.0, 0.001);
float data2{0};
// lod0:(241,513) maps to lod1:(120,256) but so does the 3 neighbor
// brick columns which were not overwritten by seven.
reader->read(size3i_t{120,256,0}, size3i_t{1,1,1}, &data2, /*lod=*/1);
TEST_EQUAL_FLOAT(data2, (2*7.0 + 6*42.0)/8, 0.001);
}
}
/**
* This is one of the test cases run in test_reopen() but I'll also
* run it explicitly here. Because it needs some explanation.
......@@ -1526,6 +1615,7 @@ public:
register_test("reopen.reopen_bad_histo", test_reopen_bad_histo);
register_test("reopen.reopen", test_reopen);
register_test("reopen.zgypublic", test_reopen_zgypublic);
register_test("reopen.track_changes", test_reopen_track_changes);
#ifdef HAVE_SD
register_test("reopen.plain_sd", test_reopen_plain_sd);
#endif
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment