Commit 8899fcdc authored by Paal Kvamme's avatar Paal Kvamme
Browse files

Update version number for ZFP compressed files from 3 to 4.

parent a783447c
Pipeline #29135 passed with stages
in 5 minutes and 12 seconds
......@@ -600,7 +600,7 @@ public:
// we need to hold on to _fd ourselves and provide it when it is time
// to flush metadata to disk.
InternalZGY::ZgyInternalWriterArgs iargs = EnumMapper::mapWriterArgs(args);
_meta_rw.reset(new InternalZGY::ZgyInternalMeta(iargs));
_meta_rw.reset(new InternalZGY::ZgyInternalMeta(iargs, compress));
_meta = _meta_rw; // in base class
// The file creation was deferred until after the consistency checks.
......@@ -1305,6 +1305,8 @@ IZgyWriter::open(const ZgyWriterArgs& args)
* There are several restrictions on using this feature.
* Some of those might be relaxed in the future.
*
* \li The file must have the latest version. Currently not enforced.
*
* \li size, bricksize, datatype, and datarange are not allowed to change.
*
* \li filename, iocontext, compressor, and lodcompressor are not stored
......@@ -1338,6 +1340,11 @@ IZgyWriter::open(const ZgyWriterArgs& args)
* If you ignore this advice then he new compression mode will only
* apply to subsequent writes.
*
* \li If the file does not contain bulk data but is flagged as v4
* anyway, it is ok to reopen it as uncompressed. In that case
* it is unspecifed whether the version is changed back from 4 to 3
* (which is the current behavior) or not.
*
* \li If the file does contain bulk data (which currently isn't allowed)
* then it is strongly advised to not update any already existing data.
* Including overwrite due to a read/modify/write cycle. With a cloud
......
......@@ -324,7 +324,7 @@ class FileHeaderPOD
{
public:
std::uint8_t _magic[4]; // Always VBS\0.
std::uint32_t _version; // Current version is 3.
std::uint32_t _version; // Current version is 3, or 4 if ZFP used.
};
#pragma pack()
......@@ -464,7 +464,7 @@ OffsetHeaderV1Access::byteswap()
// Size in storage: 1 bytes
#pragma pack(1)
// Thread safety: None. The user of this type is responsible for that.
/** \brief Physical layout of Offser Header version 2 and 3 (empty). */
/** \brief Physical layout of Offser Header version 2 and 3 and 4 (empty). */
class OffsetHeaderV2POD
{
public:
......@@ -530,6 +530,7 @@ HeaderAccessFactory::createOffsetHeader(std::uint32_t version)
case 1: return std::shared_ptr<IOffsetHeaderAccess>(new OffsetHeaderV1Access());
case 2: return std::shared_ptr<IOffsetHeaderAccess>(new OffsetHeaderV2Access());
case 3: return std::shared_ptr<IOffsetHeaderAccess>(new OffsetHeaderV2Access());
case 4: return std::shared_ptr<IOffsetHeaderAccess>(new OffsetHeaderV2Access());
default: throw OpenZGY::Errors::ZgyFormatError("Unsupported ZGY version");
}
}
......@@ -891,7 +892,7 @@ InfoHeaderV1Access::byteswap()
// Size in storage: 337 bytes
#pragma pack(1)
// Thread safety: None. The user of this type is responsible for that.
/** \brief Physical layout of Info Header version 2 and 3. */
/** \brief Physical layout of Info Header version 2 and 3 and 4. */
class InfoHeaderV2POD
{
public:
......@@ -1085,6 +1086,7 @@ HeaderAccessFactory::createInfoHeader(std::uint32_t version)
case 1: return std::shared_ptr<IInfoHeaderAccess>(new InfoHeaderV1Access());
case 2: return std::shared_ptr<IInfoHeaderAccess>(new InfoHeaderV2Access());
case 3: return std::shared_ptr<IInfoHeaderAccess>(new InfoHeaderV2Access());
case 4: return std::shared_ptr<IInfoHeaderAccess>(new InfoHeaderV2Access());
default: throw OpenZGY::Errors::ZgyFormatError("Unsupported ZGY version");
}
}
......@@ -1183,7 +1185,7 @@ HistHeaderV1Access::byteswap()
// Size in storage: 2064 bytes
#pragma pack(1)
// Thread safety: None. The user of this type is responsible for that.
/** \brief Physical layout of Histogram Header version 2 and 3. */
/** \brief Physical layout of Histogram Header version 2 and 3 and 4. */
class HistHeaderV2POD
{
public:
......@@ -1267,6 +1269,7 @@ HeaderAccessFactory::createHistHeader(std::uint32_t version)
case 1: return std::shared_ptr<IHistHeaderAccess>(new HistHeaderV1Access());
case 2: return std::shared_ptr<IHistHeaderAccess>(new HistHeaderV2Access());
case 3: return std::shared_ptr<IHistHeaderAccess>(new HistHeaderV2Access());
case 4: return std::shared_ptr<IHistHeaderAccess>(new HistHeaderV2Access());
default: throw OpenZGY::Errors::ZgyFormatError("Unsupported ZGY version");
}
}
......@@ -1659,11 +1662,11 @@ ZgyInternalMeta::ZgyInternalMeta(const std::shared_ptr<FileADT>& file, const Log
initFromOpenFile(file);
}
ZgyInternalMeta::ZgyInternalMeta(const ZgyInternalWriterArgs& args_in, const LoggerFn& logger)
ZgyInternalMeta::ZgyInternalMeta(const ZgyInternalWriterArgs& args_in, bool compress, const LoggerFn& logger)
: _loggerfn(logger ? logger : LoggerBase::standardCallback(LoggerBase::getVerboseFromEnv("OPENZGY_VERBOSE"), "openzgy-meta:", ""))
, _is_bad(false)
{
initFromScratch(args_in);
initFromScratch(args_in, compress);
}
/**
......@@ -1711,7 +1714,7 @@ ZgyInternalMeta::initFromOpenFile(const std::shared_ptr<FileADT>& file_in)
std::shared_ptr<FileADT> file(new FileWithSmallCache(file_in, 1024*256));
// The file header contains only a magic string and the file version.
// In V2 and V3 the headers are stored consecutively:
// In V2 and V3 and V4 the headers are stored consecutively:
// FileHeader OffsetHeader InfoHeader StringList HistHeader
// AlphaLUT BrickLUT
// In V1 and most likely in the upcoming V4 only FileHeader and
......@@ -1725,7 +1728,7 @@ ZgyInternalMeta::initFromOpenFile(const std::shared_ptr<FileADT>& file_in)
throw OpenZGY::Errors::ZgyFormatError("Old ZGY compressed files are not supported");
else if (this->_fh->magic() != std::array<std::uint8_t,4>{'V','B','S',0})
throw OpenZGY::Errors::ZgyFormatError("This is not a ZGY file");
else if (this->_fh->version() < 1 || this->_fh->version() > 3)
else if (this->_fh->version() < 1 || this->_fh->version() > 4)
throw OpenZGY::Errors::ZgyFormatError("Unsupported ZGY version " +
std::to_string(this->_fh->version()));
......@@ -1851,22 +1854,28 @@ ZgyInternalMeta::validateCreateNewFile(const ZgyInternalWriterArgs& args_in) con
* zunitdim, hunitdim, zunitname, hunitname, zunitfactor, hunitfactor,
* zstart, zinc, annotstart, annotinc, corners.
*
* We also need to know whether ZFP or other compression is in use,
* which means that the user provided a compressor or we are updating
* a file already flagged as potentially compressed (i.e. version >= 4)
*
* The api currently does not allow setting hprjsys, srcname, srcdesc.
* If this is desired then those need to be added as well.
*/
void
ZgyInternalMeta::initFromScratch(const ZgyInternalWriterArgs& args_in)
ZgyInternalMeta::initFromScratch(const ZgyInternalWriterArgs& args_in, bool compress)
{
if (_logger(2))
_logger(2, std::stringstream()
<< "\nCreating new file \"" << args_in.filename << "\"\n");
<< "\nCreating new file \"" << args_in.filename << "\""
<< (compress ? " with compression" : "")
<< "\n");
ZgyInternalWriterArgs args = validateCreateNewFile(args_in);
std::shared_ptr<FileHeaderAccess> fh(new FileHeaderAccess());
std::uint8_t magic[4]{'V', 'B', 'S', 0};
memcpy(fh->_pod._magic, magic, 4);
fh->_pod._version = 3;
fh->_pod._version = compress ? 4 : 3;
std::shared_ptr<OffsetHeaderV2Access> oh(new OffsetHeaderV2Access());
......
......@@ -425,6 +425,7 @@ public:
explicit ZgyInternalMeta(const std::shared_ptr<FileADT>& file,
const LoggerFn& logger = LoggerFn());
explicit ZgyInternalMeta(const ZgyInternalWriterArgs& args,
bool compress,
const LoggerFn& logger = LoggerFn());
public:
void dump(std::ostream& out, const std::string& prefix = "");
......@@ -440,7 +441,7 @@ private:
bool _logger(int priority, const std::string& ss = std::string()) const;
bool _logger(int priority, const std::ios& ss) const;
void initFromOpenFile(const std::shared_ptr<FileADT>&);
void initFromScratch(const ZgyInternalWriterArgs&);
void initFromScratch(const ZgyInternalWriterArgs&, bool compress);
ZgyInternalWriterArgs validateCreateNewFile(const ZgyInternalWriterArgs&) const;
class ErrorsWillCorruptFile;
};
......
......@@ -740,7 +740,7 @@ class ZgyWriter(ZgyMetaAndTools):
# which in Python is a no-op but in C++ the parent might
# have its own _meta that we shadow here. Or not.
super().__init__(self._meta)
self._create(filename, **kwargs)
self._create(filename, compressed = bool(compressor or lodcompressor), **kwargs)
# Now self._meta._ih and friends will all be allocated.
# Prove that all the tests for "._ih is not None" are redundant.
self._meta._assert_all_headers_allocated()
......@@ -777,7 +777,7 @@ class ZgyWriter(ZgyMetaAndTools):
# suppressing exceptions if the file has already been flagged bad.
self.close()
def _create(self, filename, *, size = None,
def _create(self, filename, *, size = None, compressed = False,
bricksize = None,
datatype = SampleDataType.float, datarange = None,
zunitdim = UnitDimension.unknown,
......@@ -794,6 +794,7 @@ class ZgyWriter(ZgyMetaAndTools):
"""
self._meta._init_from_scratch(filename = filename,
size = size,
compressed = compressed, # If True this is V4.
bricksize = bricksize,
datatype = _map_SampleDataTypeToDataType(datatype),
datarange = datarange,
......
......@@ -23,12 +23,12 @@ the public API.
is allowed to use directly.
FileHeader
OffsetHeader{V1,V2,V3}
InfoHeader{V1,V2,V3}
StringList{V1,V2,V3}
HistHeader{V1,V2,V3}
AlphaLUP{V1,V2,V3}
BrickLUP{V1,V2,V3}:
OffsetHeader{V1,V2,V3,V4}
InfoHeader{V1,V2,V3,V4}
StringList{V1,V2,V3,V4}
HistHeader{V1,V2,V3,V4}
AlphaLUP{V1,V2,V3,V4}
BrickLUP{V1,V2,V3,V4}:
* All these represent a particular version of one of the headers
that make up a complete ZGY file. Each header knows about the
......@@ -276,24 +276,26 @@ class FileHeader(HeaderBase):
Caller is responsible for all I/O, so we don't need an iocontext.
"""
def __init__(self, buf = None):
def __init__(self, buf = None, *, compressed = None):
assert self.headersize() == 8
# Python isn't really good at overloaded methods.
assert (buf is None and compressed is not None) or (buf is not None and compressed is None)
if buf:
self.unpack(buf)
assert buf == self.pack()
if self._magic == b'VCS\x00': raise ZgyFormatError("Old ZGY compressed files are not supported")
if self._magic != b'VBS\x00': raise ZgyFormatError("Not an uncompressed ZGY file")
if self._version < 1 or self._version > 3: raise ZgyFormatError("Unsupported ZGY version " + str(version))
if self._version < 1 or self._version > 4: raise ZgyFormatError("Unsupported ZGY version " + str(version))
else:
# Prepare for writing a new file.
self._magic = b'VBS\x00'
self._version = 3
self._version = 4 if compressed else 3
@staticmethod
def _formats():
return [
('_magic', '4s', 'uint8[4]', 'Always VBS\\0.'),
('_version', 'I', 'uint32', 'Current version is 3.'),
('_version', 'I', 'uint32', 'Current version is 3 or 4.'),
]
#****************************************************************************
......@@ -434,7 +436,7 @@ class OffsetHeaderV2:
This allows adding more data fields at the end of the header.
Older readers will just unpack the fields they know about.
For ZGY V2 and V3 this is moot, as all the offsets are implicit
For ZGY V{2,3,4} this is moot, as all the offsets are implicit
with all the headers written sequentially. So the size needs
to match exactly or the headers following this will be corrupt.
"""
......@@ -469,9 +471,12 @@ class OffsetHeaderV2:
class OffsetHeaderV3(OffsetHeaderV2):
pass
class OffsetHeaderV4(OffsetHeaderV3):
pass
def OffsetHeaderFactory(version):
try:
return [OffsetHeaderV1, OffsetHeaderV2, OffsetHeaderV3][version-1]
return [OffsetHeaderV1, OffsetHeaderV2, OffsetHeaderV3, OffsetHeaderV4][version-1]
except IndexError:
raise ZgyFormatError("Version " + str(version) + " is not supported")
......@@ -854,9 +859,12 @@ class InfoHeaderV2(HeaderBase):
class InfoHeaderV3(InfoHeaderV2):
pass
class InfoHeaderV4(InfoHeaderV3):
pass
def InfoHeaderFactory(version):
try:
return [InfoHeaderV1, InfoHeaderV2, InfoHeaderV3][version-1]
return [InfoHeaderV1, InfoHeaderV2, InfoHeaderV3, InfoHeaderV4][version-1]
except IndexError:
raise ZgyFormatError("Version " + str(version) + " is not supported")
......@@ -944,9 +952,12 @@ class StringListV2:
class StringListV3(StringListV2):
pass
class StringListV4(StringListV3):
pass
def StringListFactory(version):
try:
return [StringListV1, StringListV2, StringListV3][version-1]
return [StringListV1, StringListV2, StringListV3, StringListV4][version-1]
except IndexError:
raise ZgyFormatError("Version " + str(version) + " is not supported")
......@@ -1015,9 +1026,12 @@ class HistHeaderV2(HeaderBase):
class HistHeaderV3(HistHeaderV2):
pass
class HistHeaderV4(HistHeaderV3):
pass
def HistHeaderFactory(version):
try:
return [HistHeaderV1, HistHeaderV2, HistHeaderV3][version-1]
return [HistHeaderV1, HistHeaderV2, HistHeaderV3, HistHeaderV4][version-1]
except IndexError:
raise ZgyFormatError("Version " + str(version) + " is not supported")
......@@ -1200,9 +1214,12 @@ class AlphaLUPV2(LookupTable):
class AlphaLUPV3(AlphaLUPV2):
pass
class AlphaLUPV4(AlphaLUPV3):
pass
def AlphaLUPFactory(version):
try:
return [AlphaLUPV1, AlphaLUPV2, AlphaLUPV3][version-1]
return [AlphaLUPV1, AlphaLUPV2, AlphaLUPV3, AlphaLUPV4][version-1]
except IndexError:
raise ZgyFormatError("Version " + str(version) + " is not supported")
......@@ -1221,9 +1238,12 @@ class BrickLUPV2(LookupTable):
class BrickLUPV3(BrickLUPV2):
pass
class BrickLUPV4(BrickLUPV3):
pass
def BrickLUPFactory(version):
try:
return [BrickLUPV1, BrickLUPV2, BrickLUPV3][version-1]
return [BrickLUPV1, BrickLUPV2, BrickLUPV3, BrickLUPV4][version-1]
except IndexError:
raise ZgyFormatError("Version " + str(version) + " is not supported")
......@@ -1278,7 +1298,7 @@ class ZgyInternalMeta:
"""
# The file header contains only a magic string and the file version.
# In V2 and V3 the headers are stored consecutively:
# In V{2,3,4} the headers are stored consecutively:
# FileHeader OffsetHeader InfoHeader StringList HistHeader
# AlphaLUT BrickLUT
# In V1 and most likely in the upcoming V4 only FileHeader and
......@@ -1297,12 +1317,12 @@ class ZgyInternalMeta:
# of other sections.
oh = OffsetHeaderFactory(fh._version).read(f)
# 'oh' at this point may be incomplete (V2, V3) but the offset
# 'oh' at this point may be incomplete (V{2,3,4}) but the offset
# to the InfoHeader should be known by now. In V1 is is mostly
# complete but is missing a few section sizes.
ih = InfoHeaderFactory(fh._version).read(f, oh._infoff)
# For V2 and V3, fill in the rest of the offsets now that
# For V{2,3,4}, fill in the rest of the offsets now that
# the InfoHeader is known.
oh.calculate(ih)
......@@ -1323,7 +1343,7 @@ class ZgyInternalMeta:
self._init_from_headers(fh, oh, ih, sl, hh, alup, blup)
def _init_from_scratch(self, filename, *, size = None,
def _init_from_scratch(self, filename, *, size = None, compressed = False,
bricksize = None,
datatype = impl_enum.RawDataType.Float32,
datarange = None,
......@@ -1380,7 +1400,7 @@ class ZgyInternalMeta:
#--- End sanity checks and defaults --#
fh = FileHeader() # Inits to version 3.
fh = FileHeader(buf = None, compressed = compressed) # Inits to version 3, or 4 if potentially compressed.
oh = OffsetHeaderFactory(fh._version)() # Sets infoff, infsize only.
ih = InfoHeaderFactory(fh._version)() # Inits to all zero
......@@ -1527,7 +1547,7 @@ class ZgyInternalMeta:
self._first_data = hdrsize + padsize
# If not contiguous then just write the blocks one at a time.
# For V2 and V3 this will never happen. So I won't add code
# For V{2,3,4} this will never happen. So I won't add code
# for that case (which I will never be able to test).
assert contig
with ErrorsWillCorruptFile(self):
......@@ -1651,9 +1671,11 @@ def checkAllFormats(*, verbose = False, file = None):
InfoHeaderV1.checkformats(verbose=verbose, file=file)
InfoHeaderV2.checkformats(verbose=verbose, file=file)
InfoHeaderV3.checkformats(verbose=verbose, file=file)
InfoHeaderV4.checkformats(verbose=verbose, file=file)
HistHeaderV1.checkformats(verbose=verbose, file=file)
HistHeaderV2.checkformats(verbose=verbose, file=file)
HistHeaderV3.checkformats(verbose=verbose, file=file)
HistHeaderV4.checkformats(verbose=verbose, file=file)
# For now, run the consistency checks also when the module is imported.
checkAllFormats()
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment