Commit ac925be7 authored by Paal Kvamme's avatar Paal Kvamme
Browse files

Split the "threads" configuration option into "iothreads" and "cputhreads".

parent 4070f934
......@@ -71,7 +71,8 @@ SeismicStoreIOContext::SeismicStoreIOContext()
aligned(Environment::getNumericEnv("OPENZGY_ALIGNED_MB", 0));
segsize(Environment::getNumericEnv("OPENZGY_SEGSIZE_MB", 256));
segsplit(Environment::getNumericEnv("OPENZGY_SEGSPLIT", 8));
threads(Environment::getNumericEnv("OPENZGY_NUMTHREADS", 1));
iothreads(Environment::getNumericEnv("OPENZGY_NUMTHREADS_IO", 1));
cputhreads(Environment::getNumericEnv("OPENZGY_NUMTHREADS_CPU", 1));
legaltag(Environment::getStringEnv("OPENZGY_LEGALTAG"));
writeid(Environment::getStringEnv("OPENZGY_WRITEID"));
seismicmeta(Environment::getStringEnv("OPENZGY_SEISMICMETA"));
......@@ -90,7 +91,7 @@ SeismicStoreIOContext::toString() const
<< " aligned: " << _aligned / (1024*1024) << " MB\n"
<< " segsize: " << _segsize / (1024*1024) << " MB\n"
<< " segsplit: " << _segsplit << "\n"
<< " threads: " << _threads << "\n"
<< " threads: " << _iothreads << " I/O, " << _cputhreads << " CPU.\n"
<< " legaltag: \"" << _legaltag << "\"\n"
<< " writeid: \"" << _writeid << "\"\n"
<< " seismeta: \"" << _seismicmeta << "\"\n";
......
......@@ -103,7 +103,8 @@ private:
std::int64_t _aligned;
std::int64_t _segsize;
std::int64_t _segsplit;
std::int64_t _threads;
std::int64_t _iothreads;
std::int64_t _cputhreads;
std::string _legaltag;
std::string _writeid;
std::string _seismicmeta;
......@@ -319,10 +320,10 @@ public:
/**
* Use up to this many parallel requests to seismic store in order
* to speed up processing. Set between 1 and 1024, This applies to
* to speed up processing. Set between 1 and 1024. This applies to
* individual reads in the main API. So the reads must be for a
* large area (i.e. covering many bricks) for the setting to be
* of any use. Set to $OPENZGY_NUMTHREADS if not found, and 1
* of any use. Set to $OPENZGY_NUMTHREADS_IO if not set here, and 1
* (i.e. no threading) if the environment setting is also missing.
*
* Whether it is useful to set the variable depends on the
......@@ -330,12 +331,37 @@ public:
* multi threading, issuing multiple read requests to the high level
* API in parallel. In that case it might not be useful to also
* parallelize individual requests.
* It might even be a very bad idea if the total number of pending
* requests hits some service- or network limit.
*
* If the application normally asks for small regions or moderately
* sized regions that are contiguous on the file then this setting
* has no effect. Contrariwise, requesting an entire horizontal
* slice can use a lot of threads if it is allowed to. And if the
* application is single threaded it really should allow this.
*/
SeismicStoreIOContext& iothreads(int value)
{
if (value < 1 || value > 1024)
throw OpenZGY::Errors::ZgyUserError("iothreads must be between 1 and 1024");
this->_iothreads = value;
return *this;
}
/**
* Use up to this many parallel requests for cpu-intensive
* operations such as decompression. Set between 1 and 1024.
* Set to $OPENZGY_NUMTHREADS_CPU if not set here, and 1
* (i.e. no threading) if the environment setting is also missing.
*
* As with iothreads it might not be much use for Petrel but
* it is worth trying. There is less risk of hitting hard limits.
*/
SeismicStoreIOContext& threads(int value)
SeismicStoreIOContext& cputhreads(int value)
{
if (value < 1 || value > 1024)
throw OpenZGY::Errors::ZgyUserError("threads must be between 1 and 1024");
this->_threads = value;
throw OpenZGY::Errors::ZgyUserError("cputhreads must be between 1 and 1024");
this->_cputhreads = value;
return *this;
}
......
......@@ -68,7 +68,8 @@ void Test::TestIOContext::test_defaults()
TEST_CHECK(ctx._aligned == 0 * 1024*1024);
TEST_CHECK(ctx._segsize == 256 * 1024*1024 * ctx._segsplit);
TEST_CHECK(ctx._segsplit == 8);
TEST_CHECK(ctx._threads == 1);
TEST_CHECK(ctx._iothreads == 1);
TEST_CHECK(ctx._cputhreads == 1);
TEST_CHECK(ctx._legaltag == "");
TEST_CHECK(ctx._writeid == "");
TEST_CHECK(ctx._seismicmeta == "");
......@@ -85,7 +86,8 @@ void Test::TestIOContext::test_setters()
.aligned(1)
.segsize(15)
.segsplit(3)
.threads(8)
.iothreads(8)
.cputhreads(5)
.legaltag("illegal")
.writeid("WID")
.seismicmeta("{\"foo\": 42}");
......@@ -99,7 +101,8 @@ void Test::TestIOContext::test_setters()
TEST_CHECK(ctx._aligned == 1 * 1024*1024);
TEST_CHECK(ctx._segsize == 15 * 1024*1024 * ctx._segsplit);
TEST_CHECK(ctx._segsplit == 3);
TEST_CHECK(ctx._threads == 8);
TEST_CHECK(ctx._iothreads == 8);
TEST_CHECK(ctx._cputhreads== 5);
TEST_CHECK(ctx._legaltag == "illegal");
TEST_CHECK(ctx._writeid == "WID");
TEST_CHECK(ctx._seismicmeta == "{\"foo\": 42}");
......@@ -115,7 +118,8 @@ void Test::TestIOContext::test_errors()
must_throw("must be between", [&](){ctx.aligned(2000);});
must_throw("must be between", [&](){ctx.segsize(200000);});
must_throw("must be between", [&](){ctx.segsplit(0);});
must_throw("must be between", [&](){ctx.threads(0);});
must_throw("must be between", [&](){ctx.iothreads(0);});
must_throw("must be between", [&](){ctx.cputhreads(0);});
}
namespace {
......
......@@ -516,8 +516,10 @@ ZgyCommon_getIOContext(ZgyClass *self, const char *filename, PyObject* obj)
result->segsize((int)ivalue);
if ((ivalue = getLongValuedAttr(obj, "segsplit", -1)) != -1)
result->segsplit((int)ivalue);
if ((ivalue = getLongValuedAttr(obj, "threads", -1)) != -1)
result->threads((int)ivalue);
if ((ivalue = getLongValuedAttr(obj, "iothreads", -1)) != -1)
result->iothreads((int)ivalue);
if ((ivalue = getLongValuedAttr(obj, "cputhreads", -1)) != -1)
result->cputhreads((int)ivalue);
if (!(value = getStringValuedAttr(obj, "legaltag")).empty())
result->legaltag(value);
if (!(value = getStringValuedAttr(obj, "writeid")).empty())
......@@ -2086,7 +2088,7 @@ static PyMethodDef ZgyUtils_methods[] = {
static PyGetSetDef ZgyCommon_getseters[] = {
{const_cast<char*>("numthreads"),
(getter)ZgyReader_getnumthreads, (setter)ZgyReader_setnumthreads,
const_cast<char*>("How many threads to use when reading. Currently ignored when writing."),
const_cast<char*>("How many threads to use when reading. Currently ignored. Use iocontext instead."),
NULL},
{const_cast<char*>("verbose"),
(getter)ZgyReader_getverbose, (setter)ZgyReader_setverbose,
......@@ -2205,7 +2207,7 @@ static PyGetSetDef ZgyCommon_getseters[] = {
static PyGetSetDef ZgyUtils_getseters[] = {
{const_cast<char*>("numthreads"),
(getter)ZgyReader_getnumthreads, (setter)ZgyReader_setnumthreads,
const_cast<char*>("How many threads to use. Currently ignored."),
const_cast<char*>("How many threads to use. Currently ignored. Use iocontext instead."),
NULL},
{const_cast<char*>("verbose"),
(getter)ZgyReader_getverbose, (setter)ZgyReader_setverbose,
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment