Commit 282789ca authored by Paal Kvamme's avatar Paal Kvamme
Browse files

Even if genlod can do an incremental build that doesn't imply it should.

parent 0de14d12
......@@ -1235,6 +1235,7 @@ public:
std::tie(stats, histo) =
InternalZGY::GenLodC(_accessor_rw, _meta_rw, _lodcompressor,
EnumMapper::mapDecimationTypeToLodAlgorithm(decimation),
action == FinalizeAction::BuildIncremental,
progress, /*verbose=*/false).call();
// If doing an incremental rebuild: The statistics collected by
......
......@@ -86,6 +86,7 @@ GenLodBase::GenLodBase(
const std::vector<LodAlgorithm>& decimation,
const std::shared_ptr<HistogramData>& histogram,
double defaultstorage,
bool incremental,
const std::function<bool(std::int64_t,std::int64_t)>& progress,
bool verbose)
: _nlods(nlods_in)
......@@ -99,6 +100,7 @@ GenLodBase::GenLodBase(
LodAlgorithm::LowPass, LodAlgorithm::WeightedAverage} : decimation)
, _wa_histogram(histogram) // Might point to self._histo later.
, _wa_defaultstorage(defaultstorage)
, _incremental(incremental)
, _progress(progress)
, _verbose(verbose)
{
......@@ -253,10 +255,11 @@ GenLodImpl::GenLodImpl(
const std::vector<LodAlgorithm>& decimation,
const std::shared_ptr<HistogramData>& histogram,
double defaultstorage,
bool incremental,
const std::function<bool(std::int64_t,std::int64_t)>& progress,
bool verbose)
: GenLodBase(size, bricksize, dtype, histogram_range, nlods_in, decimation,
histogram, defaultstorage, progress, verbose)
histogram, defaultstorage, incremental, progress, verbose)
, _stats(std::make_shared<StatisticData>())
, _histo(std::make_shared<HistogramData>(256, histogram_range[0], histogram_range[1]))
{
......@@ -329,16 +332,16 @@ GenLodImpl::_accumulateT(const std::shared_ptr<const DataBuffer>& data_in)
/**
* Keep a running tally of statistics and histogram.
*
* TODO-@@@-Low: Performance: When doing an incremental
* build this data will be discarded so we might as well
* not collect it in the first place.
* Add a bool GenLodImpl::_incremental
* When doing an incremental build this data will be discarded
* so it might as well not be collected it in the first place.
*/
void
GenLodImpl::_accumulate(const std::shared_ptr<const DataBuffer>& data)
{
if (!data)
return;
if (_incremental)
return;
switch (data->datatype()) {
case RawDataType::SignedInt8: _accumulateT<std::int8_t>(data); break;
case RawDataType::UnsignedInt8: _accumulateT<std::uint8_t>(data); break;
......@@ -399,6 +402,8 @@ GenLodImpl::_accumulate(const std::shared_ptr<const DataBuffer>& data)
* impractical. At least make sure the histogram updfate is done early
* enough and the decimation late enough that the chunk of data being
* decimated has already been added to the histogram.
*
* TODO-@@@: if (_incremental) may want to reduce the block size.
*/
std::shared_ptr<DataBuffer>
GenLodImpl::_calculate(const std::array<std::int64_t,3>& readpos_in, std::int32_t readlod)
......@@ -791,6 +796,7 @@ GenLodC::GenLodC(
const std::shared_ptr<ZgyInternalMeta>& meta,
const compressor_t& lodcompressor,
const std::vector<LodAlgorithm>& decimation,
bool incremental,
const std::function<bool(std::int64_t,std::int64_t)>& progress,
bool verbose)
: GenLodImpl(meta->ih().size(),
......@@ -802,11 +808,17 @@ GenLodC::GenLodC(
decimation,
nullptr, // Will use computed histogram so far.
meta->ih().defaultstorage(),
incremental,
progress,
verbose)
, _accessor(accessor)
, _lodcompressor(lodcompressor)
{
// Doing a full build even when incremental changes have been tracked
// is ok; that is up to the application to choose. The inverse won't
// work and should have been tested for earlier.
if (incremental && accessor->trackedBricks().size() == 0)
throw OpenZGY::Errors::ZgyInternalError("GenLodC inconsistent parameters.");
if (this->_verbose)
std::cout << "@ GenLod is created."
<< " Written range " << accessor->valueRangeWritten()
......@@ -859,14 +871,11 @@ GenLodC::_willneed() const
// rebuilds touch the data they ought to. Even though only the final
// count is checked.
// TODO-@@@ A virtual method or a data member telling us when an
// incremental buid is in progress. If not, this entire code block
// can be skipped. Knowing about incremental builds might also suggest
// other changes. Such as reducing the block size in _calculate().
// Subtract the bricks skipped because of incremental build.
// If full build then _canread() returns false for all lod>0
// and the code below, even without a short cut, would do nothing.
if (!_incremental)
return total;
sz = this->_surveysize;
std::array<std::int64_t,3> max = sz; // Limit imposed by survey size @ lod-1
......@@ -933,8 +942,8 @@ GenLodC::_canread(
if (lod == 0) {
return true; // Fullres always read from file.
}
else if (this->_accessor->trackedBricks().size() == 0) {
return false; // Not tracking dirty bits => no BuildIncremental.
else if (!_incremental) {
return false; // Not tracking dirty bits, or caller wants full anyway.
}
else if ((this->_accessor->trackedBricksDirty(pos, size, lod) & 0x01) != 0) {
return false; // Something dirty somewhere in the input.
......
......@@ -66,6 +66,7 @@ protected:
std::vector<LodAlgorithm> _decimation;
std::shared_ptr<HistogramData> _wa_histogram;
double _wa_defaultstorage;
bool _incremental;
std::function<bool(std::int64_t,std::int64_t)> _progress;
bool _verbose;
......@@ -78,6 +79,7 @@ public:
const std::vector<LodAlgorithm>& decimation,
const std::shared_ptr<HistogramData>& histogram,
double defaultvalue,
bool incremental,
const std::function<bool(std::int64_t,std::int64_t)>& progress,
bool verbose);
protected:
......@@ -116,6 +118,7 @@ public:
const std::vector<LodAlgorithm>& decimation,
const std::shared_ptr<HistogramData>& histogram,
double defaultvalue,
bool incremental,
const std::function<bool(std::int64_t,std::int64_t)>& progress,
bool verbose);
std::tuple<std::shared_ptr<StatisticData>, std::shared_ptr<HistogramData>>
......@@ -157,6 +160,7 @@ public:
const std::shared_ptr<ZgyInternalMeta>& meta,
const compressor_t& lodcompressor,
const std::vector<LodAlgorithm>& decimation,
bool incremental,
const std::function<bool(std::int64_t,std::int64_t)>& progress,
bool verbose);
protected:
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment