Commit cc6b425d authored by Paal Kvamme's avatar Paal Kvamme
Browse files

More information output to CSV log.

parent cc507911
......@@ -21,12 +21,16 @@
#include <fstream>
#include <sstream>
#include <cmath>
#include <atomic>
#include <algorithm>
namespace InternalZGY {
#if 0
}
#endif
std::atomic<int> FileWithPerformanceLogger::_last_id{0};
FileWithPerformanceLogger::FileWithPerformanceLogger(std::shared_ptr<FileADT> relay, std::shared_ptr<std::ostream> outfile, std::int64_t chunksize, int hist_bincount, double hist_min, double hist_max, int interval)
: _relay(relay)
, _outfile(outfile)
......@@ -45,15 +49,15 @@ FileWithPerformanceLogger::FileWithPerformanceLogger(std::shared_ptr<FileADT> re
, _sumtimerend(std::numeric_limits<double>::lowest())
, _suminterval(interval)
, _sumbytes(0)
, _first(true)
, _id(0)
{
_id = 1 + _last_id.fetch_add(1);
}
FileWithPerformanceLogger::~FileWithPerformanceLogger()
{
std::string str1 = dumpThroughput(true);
std::string str2 = dumpLatency(true);
if (_outfile && (!str1.empty() || !str2.empty()))
*_outfile << str1 + str2 << std::flush;
dumpToFile("destructed");
}
void
......@@ -102,10 +106,7 @@ void
FileWithPerformanceLogger::xx_close()
{
_relay->xx_close();
std::string str1 = dumpThroughput(true);
std::string str2 = dumpLatency(true);
if (_outfile && (!str1.empty() || !str2.empty()))
*_outfile << str1 + str2 << std::flush;
dumpToFile("closed");
}
std::int64_t
......@@ -194,9 +195,9 @@ FileWithPerformanceLogger::dumpLatency(bool clear)
const int nbins = static_cast<int>(_histbins.size());
const double binwidth = (_histmax - _histmin) / (nbins - 1);
ss << "CSV1,Samplecount,Histogram min,Histogram max"
ss << "CSV1,ID,Samplecount,Histogram min,Histogram max"
<< ",Statistic min,Statistic max,Statistic average,END\n"
<< "CSV2," << _nsamples
<< "CSV2," << _id << "," << _nsamples
<< "," << _histmin << "," << _histmax
<< "," << _statmin << "," << _statmax
<< "," << _statsum / _nsamples
......@@ -206,19 +207,19 @@ FileWithPerformanceLogger::dumpLatency(bool clear)
// Note that CSV3, CSV4, CSV5 can all be computed in a
// spreadsheet using a simple formula. Bit I'd like to
// have an (almost) single-click way of making the graph.
ss << "CSV3,Latency";
ss << "CSV3," << _id << ",Latency";
for (int ii=0; ii<nbins; ++ii)
ss << "," << _histmin + binwidth * ii;
ss << ",END\n";
// Dump the bins as percentages
ss << "CSV4,Frequency%";
ss << "CSV4," << _id << ",Frequency%";
for (const std::int64_t count : _histbins)
ss << "," << count / (double)_nsamples;
ss << ",END\n";
// Dump the cumulative counts.
ss << "CSV5,Cumulative%";
ss << "CSV5," << _id << ",Cumulative%";
std::int64_t summed = 0;
for (const std::int64_t count : _histbins) {
summed += count;
......@@ -227,7 +228,7 @@ FileWithPerformanceLogger::dumpLatency(bool clear)
ss << ",END\n";
// Dump the bins as raw counts
ss << "CSV6,Count";
ss << "CSV6," << _id << ",Count";
for (const std::int64_t count : _histbins)
ss << "," << count;
ss << ",END\n";
......@@ -249,6 +250,10 @@ FileWithPerformanceLogger::dumpThroughput(bool clear)
{
std::lock_guard<std::mutex> lk(_mutex);
std::stringstream ss;
if (_first) {
ss << "CSV0,ID,mbytes,time,speed,readcount,END\n";
_first = false;
}
if (_sumtimerbeg < _sumtimerend && _sumbytes > 0) {
const double bytecount = static_cast<double>(_sumbytes);
const double elapsed = _sumtimerend - _sumtimerbeg;
......@@ -256,8 +261,8 @@ FileWithPerformanceLogger::dumpThroughput(bool clear)
// regardless of whether there was any traffic then round
// begin and end to a multiple of interval. And add code
// to output a lot of zeros for intervals with no traffic.
ss << "CSV7,mbytes,time,speed,readcount,END\n";
ss << "CSV8"
<< "," << _id
<< "," << bytecount / (1024.0*1024.0)
<< "," << elapsed
<< "," << (bytecount / (1024.0*1024.0)) / elapsed
......@@ -273,6 +278,16 @@ FileWithPerformanceLogger::dumpThroughput(bool clear)
return ss.str();
}
void
FileWithPerformanceLogger::dumpToFile(const std::string& comment)
{
std::string str1 = dumpThroughput(true);
std::string str2 = dumpLatency(true);
std::string str3 = "CSV9," + std::to_string(_id) + "," + comment + ",END\n";
if (_outfile && (!str1.empty() || !str2.empty()))
*_outfile << str1 + str2 + str3 << std::flush;
}
/**
* Inject a telemetry module if enabled by environment variables.
* If not enabled the telemetry code has zero impact on the system.
......
......@@ -57,6 +57,9 @@ private:
double _sumtimerend;
const double _suminterval;
std::int64_t _sumbytes;
bool _first;
int _id;
static std::atomic<int> _last_id;
FileWithPerformanceLogger(const FileWithPerformanceLogger&) = delete;
FileWithPerformanceLogger& operator=(const FileWithPerformanceLogger&) = delete;
......@@ -73,8 +76,10 @@ public:
virtual bool xx_iscloud() const override;
public:
void add(const Timer& timer, std::int64_t blocksize);
// Nonvirtual; might be called from destructor.
std::string dumpLatency(bool clear);
std::string dumpThroughput(bool clear);
void dumpToFile(const std::string& comment);
static std::shared_ptr<FileADT> inject(std::shared_ptr<FileADT> file);
};
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment