Commit 076deb1a authored by Paal Kvamme's avatar Paal Kvamme
Browse files

Refactoring before complicated change to make diffing simpler. Defer computing expected bricks.

parent c0501f7d
......@@ -88,7 +88,7 @@ GenLodBase::GenLodBase(
double defaultstorage,
const std::function<bool(std::int64_t,std::int64_t)>& progress,
bool verbose)
: _nlods(0)
: _nlods(nlods_in)
, _total(0)
, _done(0)
, _surveysize(size)
......@@ -102,25 +102,16 @@ GenLodBase::GenLodBase(
, _progress(progress)
, _verbose(verbose)
{
// Re-compute this. Should already be known unless running a unit test.
// Do a sanity check on the supplied nlods.
std::int32_t nlods = 1; // The loop stops before counting final level
std::int64_t total = 1; // Total number of bricks in all levels.
std::array<std::int64_t,3> bs = bricksize;
std::array<std::int64_t,3> sz = size;
while (sz[0]>bs[0] || sz[1] > bs[1] || sz[2] > bs[2]) {
std::array<std::int64_t,3> bricks = (sz + bs - std::int64_t(1)) / bs;
nlods += 1;
total += (bricks[0] * bricks[1] * bricks[2]);
sz = (sz + std::int64_t(1)) / std::int64_t(2);
}
if (nlods_in > 0 && nlods_in != nlods)
throw OpenZGY::Errors::ZgyInternalError("GenLod error in nlods computation");
this->_nlods = nlods;
this->_total = total;
// TODO-@@@-Low: in incremental builds "total" will be incorrect.
// probably need a proper dry run to get the actual total reads and
// writes. That would be invoked by call().
this->_report(nullptr);
}
/**
......@@ -143,6 +134,20 @@ GenLodBase::_report(const DataBuffer* data) const
throw OpenZGY::Errors::ZgyAborted("Computation of low resolution data was aborted");
}
/**
* Return the number of bricks that will be accessed (either read or written)
* in order to generate low resolution bricks. If the version in the base
* class is not overridden the assumption is that no lowres will be stored.
*
* Note that since this is a virtual method it must not be invoked from
* the base class constructor.
*/
std::int64_t
GenLodBase::_willneed() const
{
return 0;
}
/**
* Return true if the requested data exists on the file or (for plan D)
* in the application. This is always the case for LOD 0. The method
......@@ -281,6 +286,8 @@ GenLodImpl::call()
std::cout << "@ GenLod is running."
<< " Histogram range " << _histogram_range
<< "\n";
this->_reporttotal(_willneed());
this->_report(nullptr);
this->_calculate(std::array<std::int64_t,3>{0,0,0}, this->_nlods-1);
this->_savestats(); // Remove?
return std::make_tuple(this->_stats, this->_histo);
......@@ -460,11 +467,13 @@ GenLodImpl::_calculate(const std::array<std::int64_t,3>& readpos_in, std::int32_
if (readlod == this->_nlods - 1) {
result = nullptr; // Caller will discard it anyway.
if (this->_done != this->_total) {
#if 0 // TODO-@@@ need a dry run to get required read/write count.
#if 0 // If trusting _willneed()
throw OpenZGY::Errors::ZgyInternalError
("GenLodImpl: Expected " + std::to_string(this->_total) +
" reads and writes but saw " + std::to_string(this->_done) + ".");
#else
std::cout << "Warning: GenLodImpl: Expected " << this->_total
<< " reads and writes but saw " << this->_done << ".\n";
this->_done = this->_total;
this->_report(nullptr);
#endif
......@@ -790,6 +799,47 @@ GenLodC::GenLodC(
<< "\n";
}
/**
* Return the number of bricks that will be accessed (either read or written)
* in order to generate low resolution bricks. With plan C doing a full rebuild
* all the lod0 bricks will be read exactly once and all the lowres bricks
* will be written exactly once.
*
* Incremental builds are trickier. Fullres bricks do not directly affect
* the read+write count. Any lowres brick marked as clean means that it
* 8 input bricks will no longer be read (subtract 8) and that this
* brick changes from potentially being written to potentially being read
* (no change to the total). The brick might end up neither read nor
* written if all its 7 siblings also are clean. But that adjustment is
* done by the lowres brick above. With one exception: If the single
* brick in the last level is clean, implying that the entire data set
* is clean, this saves not just the input but also the brick itself
* because there is never any reason to read it. Only write.
*
* To complicate the above, fewer than 8 bricks might be involved if
* close to the survey border.
*
* To complicate even furtger, when _calculate chooses how much data
* to process then it will read 4 brick-columns at "readlod".
* I suspect one brick-column would have worked just as well.
*/
std::int64_t
GenLodC::_willneed() const
{
// meta->lodsizes() is not available, I guess I could have captutred it.
// But it is no big deal to re-compute that information.
std::int64_t total = 0; // Total number of bricks in all levels.
std::array<std::int64_t,3> bs = this->_bricksize;
std::array<std::int64_t,3> sz = this->_surveysize;
while (sz[0] > bs[0] || sz[1] > bs[1] || sz[2] > bs[2]) {
std::array<std::int64_t,3> bricks = (sz + bs - std::int64_t(1)) / bs;
total += (bricks[0] * bricks[1] * bricks[2]);
sz = (sz + std::int64_t(1)) / std::int64_t(2);
}
++total; // Loop stopped short of the last level, by definition one brick.
return total;
}
/**
* See base class for details.
*/
......@@ -824,6 +874,10 @@ GenLodC::_read(
{
std::shared_ptr<DataBuffer> result = this->_accessor->readToNewBuffer
(pos, size, lod, /*as_float=*/false, /*extra_checking*/true);
if (this->_verbose)
std::cout << "@" << _prefix(lod)
<< "read(lod=" << lod << ", pos=" << pos
<< ", " << result->toString() << ")\n";
_report(result.get());
return result;
}
......
......@@ -81,6 +81,7 @@ public:
const std::function<bool(std::int64_t,std::int64_t)>& progress,
bool verbose);
protected:
virtual std::int64_t _willneed() const;
virtual bool _canread(
std::int32_t lod, const index3_t& pos, const index3_t& size) const;
virtual std::shared_ptr<DataBuffer> _read(
......@@ -90,6 +91,7 @@ protected:
const std::shared_ptr<const DataBuffer>& data) const;
virtual void _savestats();
void _report(const DataBuffer* data) const;
void _reporttotal(std::int64_t total) { _total = total; }
std::string _prefix(std::int32_t lod) const;
static std::string _format_result(const std::shared_ptr<DataBuffer>& data);
};
......@@ -158,6 +160,7 @@ public:
const std::function<bool(std::int64_t,std::int64_t)>& progress,
bool verbose);
protected:
std::int64_t _willneed() const override;
bool _canread(
std::int32_t lod, const index3_t& pos, const index3_t& size) const override;
std::shared_ptr<DataBuffer> _read(
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment