Commit 06dd5e92 authored by Paal Kvamme's avatar Paal Kvamme
Browse files

New virtual FileADT::xx_segments(), to be used in upcoming changes.

parent 20154566
......@@ -222,6 +222,28 @@ public:
*/
virtual std::int64_t xx_eof() const = 0;
/**
* Return the size of each segment of the file, if the backend
* has a concept of multiple segments in one file. Otherwise
* just return xx_eof() in the first and only slot.
*
* If complete=false return at most 3 numbers: The first, second,
* and last segment size. Currently all segments except the
* first and last are required to have the same size, so by
* combining the results of xx_segments() and xx_eof() it is
* possible to compute the rest of the information.
*
* If the file is open for write then the last number will be
* the in-memory buffer. That can be zero and it can also be
* larger than the preferred segment size.
*
* Currently the only reason this is needed (apart from debug)
* is a couple of consistency checks when opening a cloud file
* for update. This is unfortunate because I really wanted to
* keep this api as small as possible.
*/
virtual std::vector<std::int64_t> xx_segments(bool complete) const = 0;
/**
* Return true if the file is on the cloud.
* This might trigger some optimizations.
......
......@@ -79,6 +79,7 @@ public:
static std::shared_ptr<FileADT> xx_make_instance(const std::string& filename, OpenMode mode, const OpenZGY::IOContext *iocontext);
virtual void xx_close() override;
virtual std::int64_t xx_eof() const override;
virtual std::vector<std::int64_t> xx_segments(bool complete) const override;
virtual bool xx_iscloud() const override;
virtual void xx_read(void *data, std::int64_t offset, std::int64_t size, UsageHint usagehint=UsageHint::Unknown) override;
virtual void xx_readv(const ReadList& requests, bool parallel_ok=false, bool immutable_ok=false, bool transient_ok=false, UsageHint usagehint=UsageHint::Unknown) override;
......@@ -210,6 +211,15 @@ LocalFileLinux::xx_eof() const
return this->_eof;
}
/**
* \details: Thread safety: Yes, called method is thread safe.
*/
std::vector<std::int64_t>
LocalFileLinux::xx_segments(bool /*complete*/) const
{
return std::vector<std::int64_t>{this->xx_eof()};
}
/**
* \details: Thread safety: Yes.
*/
......
......@@ -114,6 +114,12 @@ FileWithPerformanceLogger::xx_eof() const
return _relay->xx_eof();
}
std::vector<std::int64_t>
FileWithPerformanceLogger::xx_segments(bool complete) const
{
return _relay->xx_segments(complete);
}
bool
FileWithPerformanceLogger::xx_iscloud() const
{
......
......@@ -69,6 +69,7 @@ public:
virtual void xx_write(const void* data, std::int64_t offset, std::int64_t size, UsageHint usagehint) override;
virtual void xx_close() override;
virtual std::int64_t xx_eof() const override;
virtual std::vector<std::int64_t> xx_segments(bool complete) const override;
virtual bool xx_iscloud() const override;
public:
void add(const Timer& timer, std::int64_t blocksize);
......
......@@ -63,6 +63,10 @@ public:
virtual std::int64_t xx_eof() const override {
return _relay->xx_eof();
}
virtual std::vector<std::int64_t> xx_segments(bool complete) const override
{
return _relay->xx_segments(complete);
}
virtual bool xx_iscloud() const override {
return _relay->xx_iscloud();
}
......
......@@ -160,6 +160,7 @@ public:
virtual void xx_write(const void* data, std::int64_t offset, std::int64_t size, UsageHint usagehint=UsageHint::Unknown) override;
virtual void xx_close();
virtual std::int64_t xx_eof() const;
virtual std::vector<std::int64_t> xx_segments(bool complete) const override;
virtual bool xx_iscloud() const override;
// Functions from FileUtilsSeismicStore
virtual void deleteFile(const std::string& filename, bool missing_ok) const;
......@@ -173,10 +174,6 @@ public:
// instance as well, or need the logger passed in each call.
static bool _logger(int priority, const std::string& message = std::string());
static bool _logger(int priority, const std::ios& ss);
public:
// For use by debug_trace, allow SeismicStoreFileDelayedWrite() access
// to details of the file.
std::shared_ptr<const DatasetInformation> debug_info();
private:
/**
* This class is used by _split_by_segment to describe a request as seen by
......@@ -293,6 +290,7 @@ public:
virtual void xx_write(const void* data, std::int64_t offset, std::int64_t size, UsageHint usagehint=UsageHint::Unknown) override;
virtual void xx_close() override;
virtual std::int64_t xx_eof() const override;
virtual std::vector<std::int64_t> xx_segments(bool complete) const override;
virtual bool xx_iscloud() const override;
private:
......@@ -358,7 +356,7 @@ public:
public:
std::int64_t totalSize() const;
std::vector<std::int64_t> allSizes(std::int64_t open_size) const;
std::vector<std::int64_t> allSizes(bool complete) const;
void getLocalOffset(std::int64_t offset, std::int64_t size, std::int64_t *blocknum, std::int64_t *local_offset, std::int64_t *local_size) const;
void checkOnWrite(std::int64_t blocknum, std::int64_t blocksize) const;
void updateOnWrite(std::int64_t blocknum, std::int64_t blocksize);
......@@ -479,21 +477,27 @@ DatasetInformation::totalSize() const
}
/**
* Return the total file size broken down into segments, including
* the "open" segment which size needs to be provided explicitly.
* This functon is currently only used for debugging.
* Return the total file size broken down into segments, not including
* the "open" segment which DatasetInformation doesn't know about.
*/
std::vector<std::int64_t>
DatasetInformation::allSizes(std::int64_t open_size) const
DatasetInformation::allSizes(bool complete) const
{
std::vector<std::int64_t> result;
for (int ii = 0; ii < block_count_; ++ii)
result.push_back(ii == 0 ? block0_size_ :
ii == block_count_-1 ? last_block_size_ :
block1_size_);
if (open_size >= 0)
result.push_back(open_size);
return result;
switch (block_count_) {
case 0: return std::vector<std::int64_t>{};
case 1: return std::vector<std::int64_t>{block0_size_};
case 2: return std::vector<std::int64_t>{block0_size_, last_block_size_};
default: {
std::vector<std::int64_t> result;
result.push_back(block0_size_);
result.push_back(block1_size_);
if (complete)
for (int ii = 0; ii < block_count_ - 3; ++ii)
result.push_back(block1_size_);
result.push_back(last_block_size_);
return result;
}
}
}
/**
......@@ -552,7 +556,7 @@ DatasetInformation::checkOnWrite(std::int64_t blocknum, std::int64_t blocksize)
* Update cached size information after data is successfully written.
* checkOnWrite() must have been called already.
*
* Thread safety: NOT thred safe.
* Thread safety: NOT thread safe.
* Do not invoke SDGenericDatasetWrapper::info()->updateOnWrite() directly.
* Call the thread safe SDGenericDatasetWrapper::updateOnWrite() instead.
* That one wll make sure the smart pointer being updated is unique.
......@@ -1134,7 +1138,7 @@ SeismicStoreFile::xx_read(void *data, std::int64_t offset, std::int64_t size, Us
ReadRequest request(offset, size, nullptr);
RawList split = this->_split_by_segment(ReadList{request});
if (this->_config->_debug_trace)
this->_config->_debug_trace("read", /*need=*/size, /*want=*/size,/*parts*/ split.size(), this->_dataset->info()->allSizes(-1));
this->_config->_debug_trace("read", /*need=*/size, /*want=*/size,/*parts*/ split.size(), this->xx_segments(true));
for (const RawRequest& it : split) {
// TODO-Low: port _cached_read ?
SimpleTimerEx tt(*this->_rtimer);
......@@ -1222,7 +1226,7 @@ SeismicStoreFile::xx_readv(const ReadList& requests, bool parallel_ok, bool immu
std::shared_ptr<char> data(new char[realsize], std::default_delete<char[]>());
if (this->_config->_debug_trace)
this->_config->_debug_trace("readv", /*need=*/asked, /*want=*/realsize,/*parts*/ work.size(), this->_dataset->info()->allSizes(-1));
this->_config->_debug_trace("readv", /*need=*/asked, /*want=*/realsize,/*parts*/ work.size(), this->xx_segments(true));
// Do the actual reading of the consolidated chunks, possibly using
// multiple threads.
......@@ -1358,7 +1362,7 @@ SeismicStoreFile::xx_write(const void* data, std::int64_t offset, std::int64_t s
if (this->_config->_debug_trace)
this->_config->_debug_trace
(offset == current_eof ? "append" : "write",
size, size, 1, this->_dataset->info()->allSizes(-1));
size, size, 1, this->xx_segments(true));
}
/**
......@@ -1538,6 +1542,22 @@ SeismicStoreFile::xx_eof() const
return _dataset->info()->totalSize();
}
/**
* \brief Return the size of each segment of the file.
* \details: Thread safety: Not if writes may be in progress. Could be fixed.
*
* If complete=false return at most 3 numbers: The first, second,
* and last segment size. Currently all segments except the
* first and last are required to have the same size, so by
* combining the results of xx_segments() and xx_eof() it is
* possible to compute the rest of the information.
*/
std::vector<std::int64_t>
SeismicStoreFile::xx_segments(bool complete) const
{
return this->_dataset->info()->allSizes(complete);
}
/**
* \details: Thread safety: Yes.
*/
......@@ -1614,15 +1634,6 @@ SeismicStoreFile::altUrl(const std::string& filename) const
return url;
}
/**
* Thread safety: Yes.
*/
std::shared_ptr<const DatasetInformation>
SeismicStoreFile::debug_info()
{
return this->_dataset->info();
}
/**
* Given one or more (offset, size, ...) tuples, convert these
* to (segment_number, offset_in_seg, size_in_seg, outpos).
......@@ -1882,7 +1893,7 @@ SeismicStoreFileDelayedWrite::xx_write(const void* data, std::int64_t offset, st
if (offset == 0 || this->_config->_segsize <= 0 || offset < committed) {
this->_relay->xx_write(data, offset, size, usagehint);
if (this->_config->_debug_trace)
this->_config->_debug_trace("flush", size, size, 1, this->_relay->debug_info()->allSizes(this->_open_segment.size()));
this->_config->_debug_trace("flush", size, size, 1, this->xx_segments(true));
return;
}
......@@ -1927,7 +1938,7 @@ SeismicStoreFileDelayedWrite::xx_write(const void* data, std::int64_t offset, st
this->_usage_hint = UsageHint::Mixed;
if (this->_config->_debug_trace)
this->_config->_debug_trace("queue", size, size, 1, this->_relay->debug_info()->allSizes(this->_open_segment.size()));
this->_config->_debug_trace("queue", size, size, 1, this->xx_segments(true));
this->_flush(false);
}
......@@ -1953,6 +1964,26 @@ SeismicStoreFileDelayedWrite::xx_eof() const
static_cast<std::int64_t>(this->_open_segment.size()));
}
/**
* \brief Return the size of each segment of the file.
* \details: Thread safety: Not if writes may be in progress. Could be fixed.
*
* If complete=false return at most 3 numbers: The first, second,
* and last segment size. Currently all segments except the
* first and last are required to have the same size, so by
* combining the results of xx_segments() and xx_eof() it is
* possible to compute the rest of the information.
*/
std::vector<std::int64_t>
SeismicStoreFileDelayedWrite::xx_segments(bool complete) const
{
std::vector<std::int64_t> result = this->_relay->xx_segments(complete);
if (!complete && result.size() >= 3)
result.resize(2);
result.push_back(this->_open_segment.size());
return result;
}
/**
* \details: Thread safety: Yes.
*/
......@@ -1977,7 +2008,7 @@ SeismicStoreFileDelayedWrite::_flush_part(std::int64_t this_segsize)
this->_relay->xx_write(seg.data(), this->_relay->xx_eof(), this_segsize, this->_usage_hint);
seg.erase(seg.begin(), seg.begin() + this_segsize);
if (this->_config->_debug_trace)
this->_config->_debug_trace("flush", this_segsize, this_segsize, 1, this->_relay->debug_info()->allSizes(seg.size()));
this->_config->_debug_trace("flush", this_segsize, this_segsize, 1, this->xx_segments(true));
}
/**
......
......@@ -93,6 +93,12 @@ FileWithSmallCache::xx_eof() const
return _relay->xx_eof();
}
std::vector<std::int64_t>
FileWithSmallCache::xx_segments(bool complete) const
{
return _relay->xx_segments(complete);
}
bool
FileWithSmallCache::xx_iscloud() const
{
......
......@@ -100,6 +100,7 @@ public:
virtual void xx_write(const void* data, std::int64_t offset, std::int64_t size, UsageHint usagehint) override;
virtual void xx_close() override;
virtual std::int64_t xx_eof() const override;
virtual std::vector<std::int64_t> xx_segments(bool complete) const override;
virtual bool xx_iscloud() const override;
};
......
......@@ -78,6 +78,7 @@ public:
static std::shared_ptr<FileADT> xx_make_instance(const std::string& filename, OpenMode mode, const OpenZGY::IOContext *iocontext);
virtual void xx_close() override;
virtual std::int64_t xx_eof() const override;
virtual std::vector<std::int64_t> xx_segments(bool complete) const override;
virtual bool xx_iscloud() const override;
virtual void xx_read(void *data, std::int64_t offset, std::int64_t size, UsageHint usagehint=UsageHint::Unknown) override;
virtual void xx_readv(const ReadList& requests, bool parallel_ok=false, bool immutable_ok=false, bool transient_ok=false, UsageHint usagehint=UsageHint::Unknown) override;
......@@ -211,6 +212,15 @@ LocalFileWindows::xx_eof() const
return this->_eof;
}
/**
* \details: Thread safety: Yes, called method is thread safe.
*/
std::vector<std::int64_t>
LocalFileWindows::xx_segments(bool /*complete*/) const
{
return std::vector<std::int64_t>{this->xx_eof()};
}
/**
* \details: Thread safety: Yes.
*/
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment