Commit 5f79b241 authored by Paal Kvamme's avatar Paal Kvamme
Browse files

Add --update option to zgycopyc for testing the new functionality.

parent 448188e3
......@@ -83,7 +83,7 @@ public:
std::string myname;
int verbose;
bool nolod;
bool update; // Still unused.
bool update;
bool randomize;
bool alpha; // Still unused.
bool dumpsqnr; // Still unused.
......@@ -135,7 +135,7 @@ public:
, algorithm()
, lod()
, threads(1)
, brickcount(0)
, brickcount(-1)
, sqnr(0)
, omp_nest(-1)
, noisefactor(0)
......@@ -162,7 +162,7 @@ public:
"-v, --verbose: Verbose output. May be repeated",
"-q, --quiet: No output except errors.",
"-G, --nolod: Completely disable lod generation on write.",
//"-u, --update: *Update (don't truncate) an existing file.",
"-u, --update: *Update (don't truncate) an existing file.",
"-r, --randomize: Read and write bricks in random order.",
//"-a, --alpha: *Also copy the alpha plane.",
//"-D, --dumpsqnr: *Dump table of sqnr vs. compression ratio.",
......@@ -265,7 +265,7 @@ public:
os << "--lod=" << lod << " ";
if (threads != 1)
os << "--threads=" << threads << " ";
if (brickcount != 0)
if (brickcount >= 0)
os << "--brickcount=" << brickcount << " ";
if (sqnr > 0)
os << "--sqnr=" << sqnr << " ";
......@@ -419,7 +419,7 @@ public:
case 'v': ++verbose; break;
case 'q': --verbose; break;
case 'G': nolod = true; break;
case 'u': throw std::runtime_error("--update not supported"); //update = true; break;
case 'u': update = true; break;
case 'r': randomize = true; break;
case 'a': throw std::runtime_error("--alpha not supported"); //alpha = true; break;
case 'D': throw std::runtime_error("--dumpsqnr not supported"); //dumpsqnr = true; break;
......@@ -515,6 +515,10 @@ public:
osamplesize != OpenZGY::SampleDataType::unknown) {
throw std::runtime_error("command line: Don't use --osamplesize with compressed output files");
}
if (update && osamplesize != OpenZGY::SampleDataType::unknown)
throw std::runtime_error("command line: Don't use --osamplesize with --update");
if (update && noisefactor)
throw std::runtime_error("command line: Don't use --noise with --update");
}
#ifdef HAVE_GETOPT
......@@ -983,15 +987,18 @@ copy(const Options& opt, SummaryPrintingTimerEx& rtimer, SummaryPrintingTimerEx&
std::shared_ptr<IZgyReader> r = !opt.input.empty() ?
IZgyReader::open(opt.input, &rcontext):
Test::ZgyReaderMock::mock(opt.fakesize);
args.metafrom(r).filename(opt.output);
if (opt.osamplesize != OpenZGY::SampleDataType::unknown)
args.datatype(opt.osamplesize);
if (!opt.update) {
args.metafrom(r);
if (opt.osamplesize != OpenZGY::SampleDataType::unknown)
args.datatype(opt.osamplesize);
if (opt.obricksize[0]>0 && opt.obricksize[1]>0 && opt.obricksize[2]>0)
args.bricksize(opt.obricksize[0], opt.obricksize[1], opt.obricksize[2]);
}
args.filename(opt.output);
if (opt.sqnr > 0)
args.zfp_compressor(static_cast<float>(opt.sqnr))
.zfp_lodcompressor(static_cast<float>(opt.sqnr))
.datatype(OpenZGY::SampleDataType::float32);
if (opt.obricksize[0]>0 && opt.obricksize[1]>0 && opt.obricksize[2]>0)
args.bricksize(opt.obricksize[0], opt.obricksize[1], opt.obricksize[2]);
args.iocontext(&wcontext);
// Existing files with integral storage and all samples set to
......@@ -1001,6 +1008,9 @@ copy(const Options& opt, SummaryPrintingTimerEx& rtimer, SummaryPrintingTimerEx&
r->datatype() == OpenZGY::SampleDataType::int16) &&
r->raw_datarange()[0] == r->raw_datarange()[1] &&
!opt.output.empty()) {
if (opt.update)
throw std::runtime_error
("Update from an all-constant file is pointless and not implemented");
const float value = r->raw_datarange()[0];
float lo=0, hi=0;
// TODO-Low, should be w->datatype() in case the user overrides
......@@ -1021,9 +1031,10 @@ copy(const Options& opt, SummaryPrintingTimerEx& rtimer, SummaryPrintingTimerEx&
return;
}
std::shared_ptr<IZgyWriter> w = !opt.output.empty() ?
IZgyWriter::open(args) :
Test::ZgyWriterMock::mock(args);
std::shared_ptr<IZgyWriter> w =
opt.output.empty() ? Test::ZgyWriterMock::mock(args) :
opt.update ? IZgyWriter::reopen(args) :
IZgyWriter::open(args);
// I normally want to read full bricks as stored on file. So, while
// I need to ensure that I don't read too far past the survey's edge
......@@ -1051,11 +1062,45 @@ copy(const Options& opt, SummaryPrintingTimerEx& rtimer, SummaryPrintingTimerEx&
for (pos[2] = 0; pos[2] < size[2]; pos[2] += bs[2])
tasklist.push_back(pos);
}
if (opt.update) {
std::vector<std::array<std::int64_t,3>> newlist;
for (const std::array<std::int64_t,3>& pos : tasklist) {
std::pair<bool,double> r_isconst = r->readconst(pos, bs, 0, false);
std::pair<bool,double> w_isconst = w->readconst(pos, bs, false);
// Subtle issue: Technically this might turn a constant-0 brick in
// the source into an unwritten brick in the target. Which creates
// all kinds of risk because the replacement for a missing brick
// is chosen when the file is read, not when it us written.
// But the targes ws probably free of missing bricks already.
// Not-so-subtle issue: Chunk size must not change, and I cannot
// detect whether it has or not.
if (r_isconst.first && w_isconst.first && r_isconst.second == w_isconst.second) {
// Already has the correct value.
}
else if (!w_isconst.first) {
// If any part of the chunk has been written with non-const data,
// assume that the entire chunk has already been copied and might
// be skipped. This is why it is important to (a) use the same
// source file ehan updating, and (b) use the same chunk size.
}
else {
newlist.push_back(pos);
}
}
if (opt.verbose >= 2)
std::cerr << tasklist.size() - newlist.size()
<< " of " << tasklist.size()
<< " are empty or have already been copied."
<< std::endl;
tasklist = newlist;
}
if (opt.randomize)
randomize(tasklist);
// How many bricks to copy might be changed by the user.
const std::int64_t total = opt.brickcount > 0 ?
const std::int64_t total = opt.brickcount >= 0 ?
std::min((std::int64_t)tasklist.size(), (std::int64_t)opt.brickcount) :
(std::int64_t)tasklist.size();
std::atomic<std::int64_t> done(0);
......@@ -1183,7 +1228,10 @@ copy(const Options& opt, SummaryPrintingTimerEx& rtimer, SummaryPrintingTimerEx&
// Yes it does actually have a (tiny) cost but the user won't
// expect to see finalize reported at all when discarding the output.
SimpleTimerEx ft(ftimer);
w->finalize(opt.algorithm, std::ref(p2));
w->finalize(opt.algorithm, std::ref(p2),
opt.update ? FinalizeAction::BuildIncremental :
FinalizeAction::BuildDefault,
false);
w->close();
// If Timer logging is also enabled inside OpenZGY there will now
// be multiple lines of output when the output file is finalized
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment