From b3d9f31f2d47184b251660c94d505f7f9613f504 Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Thu, 31 Jul 2014 17:32:27 +1000 Subject: Start and stop eviction worker threads based on demand. Refs #1116 --- bench/wtperf/runners/medium-btree.wtperf | 4 +- dist/api_data.py | 6 ++- src/btree/bt_evict.c | 65 +++++++++++++++++++++----------- src/config/config_def.c | 8 ++-- src/conn/conn_cache.c | 4 +- src/include/cache.h | 15 ++++++++ src/include/wiredtiger.in | 12 ++++-- src/include/wt_internal.h | 2 + 8 files changed, 81 insertions(+), 35 deletions(-) diff --git a/bench/wtperf/runners/medium-btree.wtperf b/bench/wtperf/runners/medium-btree.wtperf index 92c63e73480..0e059f05095 100644 --- a/bench/wtperf/runners/medium-btree.wtperf +++ b/bench/wtperf/runners/medium-btree.wtperf @@ -1,7 +1,7 @@ # wtperf options file: medium btree configuration -conn_config="cache_size=1G" +conn_config="cache_size=100M,eviction_worker_max=6" table_config="type=file" -icount=50000000 +icount=5000000 report_interval=5 run_time=120 populate_threads=1 diff --git a/dist/api_data.py b/dist/api_data.py index 4c99471999c..f789b82bd60 100644 --- a/dist/api_data.py +++ b/dist/api_data.py @@ -340,8 +340,10 @@ connection_runtime_config = [ Config('eviction_trigger', '95', r''' trigger eviction when the cache is using this much memory, as a percentage of the total cache size''', min=10, max=99), - Config('eviction_workers', '0', r''' - additional threads to help evict pages from cache''', + Config('eviction_worker_max', '0', r''' + maximum number of additional threads WiredTiger will start to help + evict pages from cache. The number of threads currently running + will vary depending on the current eviction load''', min=0, max=20), Config('shared_cache', '', r''' shared cache configuration options. A database should configure diff --git a/src/btree/bt_evict.c b/src/btree/bt_evict.c index 7f51d586c45..8e9a8fd5ac9 100644 --- a/src/btree/bt_evict.c +++ b/src/btree/bt_evict.c @@ -17,13 +17,6 @@ static int __evict_walk(WT_SESSION_IMPL *, uint32_t *, uint32_t); static int __evict_walk_file(WT_SESSION_IMPL *, u_int *, uint32_t); static void *__evict_worker(void *); -typedef struct { - WT_CONNECTION_IMPL *conn; - u_int id; - - pthread_t tid; -} WT_EVICTION_WORKER; - /* * __evict_read_gen -- * Get the adjusted read generation for an eviction entry. @@ -157,7 +150,7 @@ __evict_server(void *arg) WT_CACHE *cache; WT_CONNECTION_IMPL *conn; WT_DECL_RET; - WT_EVICTION_WORKER *workers; + WT_EVICTION_WORKER *workers, *worker; WT_SESSION_IMPL *session; u_int i; @@ -166,17 +159,25 @@ __evict_server(void *arg) cache = conn->cache; workers = NULL; - if (cache->eviction_workers > 0) + if (cache->eviction_workers_max > 0) WT_ERR(__wt_calloc_def( - session, cache->eviction_workers, &workers)); + session, cache->eviction_workers_max, &workers)); - for (i = 0; i < cache->eviction_workers; i++) { - workers[i].conn = conn; - workers[i].id = i; + /* + * If eviction workers are configured, start with at most two - we'll + * launch new threads if there is work for them to do. + */ + for (i = 0; i < cache->eviction_workers_max && i < 2; i++) { + worker = &workers[cache->eviction_workers++]; + worker->conn = conn; + worker->id = i; + F_SET(worker, WT_EVICT_WORKER_RUN); WT_ERR(__wt_thread_create(session, - &workers[i].tid, __evict_worker, &workers[i])); + &worker->tid, __evict_worker, worker)); } + cache->workers = workers; + while (F_ISSET(conn, WT_CONN_EVICTION_RUN)) { /* Evict pages from the cache as needed. */ WT_ERR(__evict_pass(session)); @@ -184,6 +185,17 @@ __evict_server(void *arg) if (!F_ISSET(conn, WT_CONN_EVICTION_RUN)) break; + /* + * If we have caught up and there are more than the minimum + * number of eviction workers running, shut one down. + */ + if (cache->eviction_workers > 2) { + worker = &workers[cache->eviction_workers--]; + F_CLR(worker, WT_EVICT_WORKER_RUN); + WT_TRET(__wt_cond_signal( + session, cache->evict_waiter_cond)); + WT_TRET(__wt_thread_join(session, worker->tid)); + } F_CLR(cache, WT_EVICT_ACTIVE); WT_ERR(__wt_verbose(session, WT_VERB_EVICTSERVER, "sleeping")); /* Don't rely on signals: check periodically. */ @@ -303,15 +315,14 @@ __evict_worker(void *arg) session = conn->default_session; WT_ERR(__wt_open_session(conn, 1, NULL, NULL, &session)); - while (F_ISSET(conn, WT_CONN_EVICTION_RUN)) { + while (F_ISSET(conn, WT_CONN_EVICTION_RUN) && + F_ISSET(worker, WT_EVICT_WORKER_RUN)) { /* Don't spin in a busy loop if there is no work to do */ WT_ERR(__evict_has_work(session, &flags)); - if (flags == 0) { + if (flags == 0) WT_ERR(__wt_cond_wait( session, cache->evict_waiter_cond, 10000)); - if (!F_ISSET(conn, WT_CONN_EVICTION_RUN)) - break; - } else + else WT_ERR(__evict_lru_pages(session, 1)); } @@ -379,6 +390,7 @@ __evict_pass(WT_SESSION_IMPL *session) { WT_CACHE *cache; WT_CONNECTION_IMPL *conn; + WT_EVICTION_WORKER *worker; int loop; uint32_t flags; uint64_t bytes_inuse; @@ -412,11 +424,22 @@ __evict_pass(WT_SESSION_IMPL *session) * evicted. This will be cleared by the next thread to * successfully evict a page. */ - if (bytes_inuse > conn->cache_size) + if (bytes_inuse > conn->cache_size) { F_SET(cache, WT_EVICT_NO_PROGRESS); - else + } else F_CLR(cache, WT_EVICT_NO_PROGRESS); + /* Start a worker if we have capacity and the cache is full. */ + if (bytes_inuse > conn->cache_size && + cache->eviction_workers < cache->eviction_workers_max) { + worker = &cache->workers[cache->eviction_workers++]; + worker->conn = conn; + worker->id = cache->eviction_workers - 1; + F_SET(worker, WT_EVICT_WORKER_RUN); + WT_RET(__wt_thread_create(session, + &worker->tid, __evict_worker, worker)); + } + F_SET(cache, WT_EVICT_ACTIVE); WT_RET(__wt_verbose(session, WT_VERB_EVICTSERVER, "Eviction pass with: Max: %" PRIu64 diff --git a/src/config/config_def.c b/src/config/config_def.c index 9424c9d29c6..74bad4c6375 100644 --- a/src/config/config_def.c +++ b/src/config/config_def.c @@ -75,7 +75,7 @@ static const WT_CONFIG_CHECK confchk_connection_reconfigure[] = { { "eviction_dirty_target", "int", "min=10,max=99", NULL }, { "eviction_target", "int", "min=10,max=99", NULL }, { "eviction_trigger", "int", "min=10,max=99", NULL }, - { "eviction_workers", "int", "min=0,max=20", NULL }, + { "eviction_worker_max", "int", "min=0,max=20", NULL }, { "shared_cache", "category", NULL, confchk_shared_cache_subconfigs }, { "statistics", "list", @@ -299,7 +299,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { { "eviction_dirty_target", "int", "min=10,max=99", NULL }, { "eviction_target", "int", "min=10,max=99", NULL }, { "eviction_trigger", "int", "min=10,max=99", NULL }, - { "eviction_workers", "int", "min=0,max=20", NULL }, + { "eviction_worker_max", "int", "min=0,max=20", NULL }, { "exclusive", "boolean", NULL, NULL }, { "extensions", "list", NULL, NULL }, { "file_extend", "list", "choices=[\"data\",\"log\"]", NULL }, @@ -371,7 +371,7 @@ static const WT_CONFIG_ENTRY config_entries[] = { "async=(enabled=0,ops_max=1024,threads=2),cache_size=100MB," "checkpoint=(name=\"WiredTigerCheckpoint\",wait=0),error_prefix=," "eviction_dirty_target=80,eviction_target=80,eviction_trigger=95," - "eviction_workers=0,shared_cache=(chunk=10MB,name=,reserve=0," + "eviction_worker_max=0,shared_cache=(chunk=10MB,name=,reserve=0," "size=500MB),statistics=none,statistics_log=(on_close=0," "path=\"WiredTigerStat.%d.%H\",sources=," "timestamp=\"%b %d %H:%M:%S\",wait=0),verbose=", @@ -481,7 +481,7 @@ static const WT_CONFIG_ENTRY config_entries[] = { "cache_size=100MB,checkpoint=(name=\"WiredTigerCheckpoint\"," "wait=0),checkpoint_sync=,create=0,direct_io=,error_prefix=," "eviction_dirty_target=80,eviction_target=80,eviction_trigger=95," - "eviction_workers=0,exclusive=0,extensions=,file_extend=," + "eviction_worker_max=0,exclusive=0,extensions=,file_extend=," "hazard_max=1000,log=(archive=,enabled=0,file_max=100MB," "path=\"\"),lsm_merge=,mmap=,multiprocess=0,session_max=100," "shared_cache=(chunk=10MB,name=,reserve=0,size=500MB)," diff --git a/src/conn/conn_cache.c b/src/conn/conn_cache.c index 4eb6391334e..733b9f3adc7 100644 --- a/src/conn/conn_cache.c +++ b/src/conn/conn_cache.c @@ -55,8 +55,8 @@ __wt_cache_config(WT_CONNECTION_IMPL *conn, const char *cfg[]) WT_RET_NOTFOUND_OK(ret); if ((ret = - __wt_config_gets(session, cfg, "eviction_workers", &cval)) == 0) - cache->eviction_workers = (u_int)cval.val; + __wt_config_gets(session, cfg, "eviction_worker_max", &cval)) == 0) + cache->eviction_workers_max = (u_int)cval.val; WT_RET_NOTFOUND_OK(ret); return (0); diff --git a/src/include/cache.h b/src/include/cache.h index bbe4157aadf..6f117e67fb6 100644 --- a/src/include/cache.h +++ b/src/include/cache.h @@ -29,6 +29,19 @@ struct __wt_evict_entry { WT_REF *ref; /* Page to flush/evict */ }; +/* + * WT_EVICTION_WORKER -- + * Encapsulation of an eviction worker thread. + */ + +struct __wt_eviction_worker { + WT_CONNECTION_IMPL *conn; + u_int id; + pthread_t tid; +#define WT_EVICT_WORKER_RUN 0x01 + uint32_t flags; +}; + /* * WiredTiger cache structure. */ @@ -65,7 +78,9 @@ struct __wt_cache { u_int eviction_target; /* Percent to end eviction */ u_int eviction_dirty_target; /* Percent to allow dirty */ + u_int eviction_workers_max; /* Max additional eviction threads */ u_int eviction_workers; /* Additional eviction threads */ + WT_EVICTION_WORKER *workers; /* * LRU eviction list information. diff --git a/src/include/wiredtiger.in b/src/include/wiredtiger.in index c46b00b90e9..62a7b10c051 100644 --- a/src/include/wiredtiger.in +++ b/src/include/wiredtiger.in @@ -1595,8 +1595,10 @@ struct __wt_connection { * @config{eviction_trigger, trigger eviction when the cache is using * this much memory\, as a percentage of the total cache size., an * integer between 10 and 99; default \c 95.} - * @config{eviction_workers, additional threads to help evict pages from - * cache., an integer between 0 and 20; default \c 0.} + * @config{eviction_worker_max, maximum number of additional threads + * WiredTiger will start to help evict pages from cache. The number of + * threads currently running will vary depending on the current eviction + * load., an integer between 0 and 20; default \c 0.} * @config{shared_cache = (, shared cache configuration options. A * database should configure either a cache_size or a shared_cache not * both., a set of related configuration options defined below.} @@ -1909,8 +1911,10 @@ struct __wt_connection { * @config{eviction_trigger, trigger eviction when the cache is using this much * memory\, as a percentage of the total cache size., an integer between 10 and * 99; default \c 95.} - * @config{eviction_workers, additional threads to help evict pages from cache., - * an integer between 0 and 20; default \c 0.} + * @config{eviction_worker_max, maximum number of additional threads WiredTiger + * will start to help evict pages from cache. The number of threads currently + * running will vary depending on the current eviction load., an integer between + * 0 and 20; default \c 0.} * @config{exclusive, fail if the database already exists\, generally used with * the \c create option., a boolean flag; default \c false.} * @config{extensions, list of shared library extensions to load (using dlopen). diff --git a/src/include/wt_internal.h b/src/include/wt_internal.h index b60d7f8ce28..518068c47bd 100644 --- a/src/include/wt_internal.h +++ b/src/include/wt_internal.h @@ -147,6 +147,8 @@ struct __wt_dsrc_stats; typedef struct __wt_dsrc_stats WT_DSRC_STATS; struct __wt_evict_entry; typedef struct __wt_evict_entry WT_EVICT_ENTRY; +struct __wt_eviction_worker; + typedef struct __wt_eviction_worker WT_EVICTION_WORKER; struct __wt_ext; typedef struct __wt_ext WT_EXT; struct __wt_extlist; -- cgit v1.2.1 From 877020a240f6594a5c8ed51b320c6cf1f51238b2 Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Thu, 31 Jul 2014 17:59:19 +1000 Subject: Fix an off by one in dynamic eviction threads. --- src/btree/bt_evict.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/btree/bt_evict.c b/src/btree/bt_evict.c index 8e9a8fd5ac9..d1a8373382d 100644 --- a/src/btree/bt_evict.c +++ b/src/btree/bt_evict.c @@ -190,7 +190,7 @@ __evict_server(void *arg) * number of eviction workers running, shut one down. */ if (cache->eviction_workers > 2) { - worker = &workers[cache->eviction_workers--]; + worker = &workers[--cache->eviction_workers]; F_CLR(worker, WT_EVICT_WORKER_RUN); WT_TRET(__wt_cond_signal( session, cache->evict_waiter_cond)); -- cgit v1.2.1 From 306b791c59c0f8c6750d97579e84883af94b7838 Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Wed, 6 Aug 2014 09:31:44 +1000 Subject: Don't have eviction workers close sessions. --- src/btree/bt_evict.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/btree/bt_evict.c b/src/btree/bt_evict.c index 9edc26e0793..1cbd100df5a 100644 --- a/src/btree/bt_evict.c +++ b/src/btree/bt_evict.c @@ -336,9 +336,6 @@ err: __wt_err(session, ret, "cache eviction helper error"); WT_TRET(__wt_verbose(session, WT_VERB_EVICTSERVER, "helper exiting")); - if (session != conn->default_session) - (void)session->iface.close(&session->iface, NULL); - return (NULL); } -- cgit v1.2.1 From 2a4554b3c39a2ca7ec7f1115a3c0809082917669 Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Wed, 6 Aug 2014 09:50:12 +1000 Subject: Lint --- bench/wtperf/runners/medium-btree.wtperf | 4 ++-- src/include/connection.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bench/wtperf/runners/medium-btree.wtperf b/bench/wtperf/runners/medium-btree.wtperf index 0e059f05095..92c63e73480 100644 --- a/bench/wtperf/runners/medium-btree.wtperf +++ b/bench/wtperf/runners/medium-btree.wtperf @@ -1,7 +1,7 @@ # wtperf options file: medium btree configuration -conn_config="cache_size=100M,eviction_worker_max=6" +conn_config="cache_size=1G" table_config="type=file" -icount=5000000 +icount=50000000 report_interval=5 run_time=120 populate_threads=1 diff --git a/src/include/connection.h b/src/include/connection.h index ef2ffd5efce..96c26097045 100644 --- a/src/include/connection.h +++ b/src/include/connection.h @@ -203,7 +203,7 @@ struct __wt_connection_impl { pthread_t evict_tid; /* Eviction server thread ID */ int evict_tid_set; /* Eviction server thread ID set */ - u_int32_t evict_workers_max;/* Max eviction workers */ + uint32_t evict_workers_max;/* Max eviction workers */ uint32_t evict_workers; /* Number of eviction workers */ WT_EVICT_WORKER *evict_workctx; /* Eviction worker context */ -- cgit v1.2.1 From 7ccec49258ae36a6c0eee2fa850b416f88c025b4 Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Mon, 11 Aug 2014 02:34:00 +0000 Subject: Make dynamic eviction workers the default. #1143 --- dist/api_data.py | 15 ++++++++++----- src/btree/bt_evict.c | 21 ++++++++++++++++++--- src/config/config_def.c | 32 ++++++++++++++++++-------------- src/conn/conn_cache.c | 21 ++++++++++++++++++++- src/include/connection.h | 1 + src/include/wiredtiger.in | 24 ++++++++++++++++-------- 6 files changed, 83 insertions(+), 31 deletions(-) diff --git a/dist/api_data.py b/dist/api_data.py index 30c76d26a84..3f0fabdc3b9 100644 --- a/dist/api_data.py +++ b/dist/api_data.py @@ -340,11 +340,16 @@ connection_runtime_config = [ Config('eviction_trigger', '95', r''' trigger eviction when the cache is using this much memory, as a percentage of the total cache size''', min=10, max=99), - Config('eviction_worker_max', '0', r''' - maximum number of additional threads WiredTiger will start to help - evict pages from cache. The number of threads currently running - will vary depending on the current eviction load''', - min=0, max=20), + Config('eviction_thread_max', '3', r''' + maximum number of threads WiredTiger will start to help + evict pages from cache. The number of threads currently running + will vary depending on the current eviction load''', + min=1, max=20), + Config('eviction_thread_min', '1', r''' + minimum number of threads WiredTiger will start to help evict + pages from cache. The number of threads currently running will + vary depending on the current eviction load''', + min=1, max=20), Config('shared_cache', '', r''' shared cache configuration options. A database should configure either a cache_size or a shared_cache not both''', diff --git a/src/btree/bt_evict.c b/src/btree/bt_evict.c index 1cbd100df5a..e360981188b 100644 --- a/src/btree/bt_evict.c +++ b/src/btree/bt_evict.c @@ -168,12 +168,25 @@ __evict_server(void *arg) * If we have caught up and there are more than the minimum * number of eviction workers running, shut one down. */ - if (conn->evict_workers > 2) { + if (conn->evict_workers > conn->evict_workers_min) { + WT_TRET(__wt_verbose(session, WT_VERB_EVICTSERVER, + "Stopping evict worker: %"PRIu32"\n", + conn->evict_workers)); worker = &conn->evict_workctx[--conn->evict_workers]; F_CLR(worker, WT_EVICT_WORKER_RUN); WT_TRET(__wt_cond_signal( session, cache->evict_waiter_cond)); WT_TRET(__wt_thread_join(session, worker->tid)); + /* + * Flag errors here with a message, but don't shut down + * the eviction server - that's fatal. + */ + WT_ASSERT(session, ret == 0); + if (ret != 0) { + (void)__wt_msg(session, + "Error stopping eviction worker: %d", ret); + ret = 0; + } } F_CLR(cache, WT_EVICT_ACTIVE); WT_ERR(__wt_verbose(session, WT_VERB_EVICTSERVER, "sleeping")); @@ -238,8 +251,7 @@ __wt_evict_create(WT_CONNECTION_IMPL *conn) WT_RET(__wt_open_internal_session(conn, "eviction-worker", 0, 0, &workers[i].session)); workers[i].id = i; - /* Start at most two worker threads initially. */ - if (i < 2) { + if (i < conn->evict_workers_min) { ++conn->evict_workers; F_SET(&workers[i], WT_EVICT_WORKER_RUN); WT_RET(__wt_thread_create( @@ -433,6 +445,9 @@ __evict_pass(WT_SESSION_IMPL *session) /* Start a worker if we have capacity and the cache is full. */ if (bytes_inuse > conn->cache_size && conn->evict_workers < conn->evict_workers_max) { + WT_RET(__wt_verbose(session, WT_VERB_EVICTSERVER, + "Starting evict worker: %"PRIu32"\n", + conn->evict_workers)); worker = &conn->evict_workctx[conn->evict_workers++]; F_SET(worker, WT_EVICT_WORKER_RUN); WT_RET(__wt_thread_create(session, diff --git a/src/config/config_def.c b/src/config/config_def.c index 74bad4c6375..1b688384286 100644 --- a/src/config/config_def.c +++ b/src/config/config_def.c @@ -74,8 +74,9 @@ static const WT_CONFIG_CHECK confchk_connection_reconfigure[] = { { "error_prefix", "string", NULL, NULL }, { "eviction_dirty_target", "int", "min=10,max=99", NULL }, { "eviction_target", "int", "min=10,max=99", NULL }, + { "eviction_thread_max", "int", "min=1,max=20", NULL }, + { "eviction_thread_min", "int", "min=1,max=20", NULL }, { "eviction_trigger", "int", "min=10,max=99", NULL }, - { "eviction_worker_max", "int", "min=0,max=20", NULL }, { "shared_cache", "category", NULL, confchk_shared_cache_subconfigs }, { "statistics", "list", @@ -298,8 +299,9 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { { "error_prefix", "string", NULL, NULL }, { "eviction_dirty_target", "int", "min=10,max=99", NULL }, { "eviction_target", "int", "min=10,max=99", NULL }, + { "eviction_thread_max", "int", "min=1,max=20", NULL }, + { "eviction_thread_min", "int", "min=1,max=20", NULL }, { "eviction_trigger", "int", "min=10,max=99", NULL }, - { "eviction_worker_max", "int", "min=0,max=20", NULL }, { "exclusive", "boolean", NULL, NULL }, { "extensions", "list", NULL, NULL }, { "file_extend", "list", "choices=[\"data\",\"log\"]", NULL }, @@ -370,9 +372,10 @@ static const WT_CONFIG_ENTRY config_entries[] = { { "connection.reconfigure", "async=(enabled=0,ops_max=1024,threads=2),cache_size=100MB," "checkpoint=(name=\"WiredTigerCheckpoint\",wait=0),error_prefix=," - "eviction_dirty_target=80,eviction_target=80,eviction_trigger=95," - "eviction_worker_max=0,shared_cache=(chunk=10MB,name=,reserve=0," - "size=500MB),statistics=none,statistics_log=(on_close=0," + "eviction_dirty_target=80,eviction_target=80," + "eviction_thread_max=3,eviction_thread_min=1,eviction_trigger=95," + "shared_cache=(chunk=10MB,name=,reserve=0,size=500MB)," + "statistics=none,statistics_log=(on_close=0," "path=\"WiredTigerStat.%d.%H\",sources=," "timestamp=\"%b %d %H:%M:%S\",wait=0),verbose=", confchk_connection_reconfigure @@ -480,15 +483,16 @@ static const WT_CONFIG_ENTRY config_entries[] = { "async=(enabled=0,ops_max=1024,threads=2),buffer_alignment=-1," "cache_size=100MB,checkpoint=(name=\"WiredTigerCheckpoint\"," "wait=0),checkpoint_sync=,create=0,direct_io=,error_prefix=," - "eviction_dirty_target=80,eviction_target=80,eviction_trigger=95," - "eviction_worker_max=0,exclusive=0,extensions=,file_extend=," - "hazard_max=1000,log=(archive=,enabled=0,file_max=100MB," - "path=\"\"),lsm_merge=,mmap=,multiprocess=0,session_max=100," - "shared_cache=(chunk=10MB,name=,reserve=0,size=500MB)," - "statistics=none,statistics_log=(on_close=0," - "path=\"WiredTigerStat.%d.%H\",sources=," - "timestamp=\"%b %d %H:%M:%S\",wait=0),transaction_sync=(enabled=0" - ",method=fsync),use_environment_priv=0,verbose=", + "eviction_dirty_target=80,eviction_target=80," + "eviction_thread_max=3,eviction_thread_min=1,eviction_trigger=95," + "exclusive=0,extensions=,file_extend=,hazard_max=1000," + "log=(archive=,enabled=0,file_max=100MB,path=\"\"),lsm_merge=," + "mmap=,multiprocess=0,session_max=100,shared_cache=(chunk=10MB," + "name=,reserve=0,size=500MB),statistics=none," + "statistics_log=(on_close=0,path=\"WiredTigerStat.%d.%H\"," + "sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," + "transaction_sync=(enabled=0,method=fsync),use_environment_priv=0" + ",verbose=", confchk_wiredtiger_open }, { NULL, NULL, NULL } diff --git a/src/conn/conn_cache.c b/src/conn/conn_cache.c index d39a8f891d6..746df68758f 100644 --- a/src/conn/conn_cache.c +++ b/src/conn/conn_cache.c @@ -55,9 +55,28 @@ __wt_cache_config(WT_CONNECTION_IMPL *conn, const char *cfg[]) WT_RET_NOTFOUND_OK(ret); if ((ret = - __wt_config_gets(session, cfg, "eviction_worker_max", &cval)) == 0) + __wt_config_gets(session, cfg, "eviction_thread_max", &cval)) == 0) conn->evict_workers_max = (u_int)cval.val; WT_RET_NOTFOUND_OK(ret); + if ((ret = + __wt_config_gets(session, cfg, "eviction_thread_min", &cval)) == 0) + conn->evict_workers_min = (u_int)cval.val; + WT_RET_NOTFOUND_OK(ret); + + if (conn->evict_workers_min > conn->evict_workers_max) + WT_RET_MSG(session, EINVAL, + "eviction_thread_min cannot be greater than " + "eviction_thread_max"); + + /* + * The eviction thread configuration options include the main eviction + * thread and workers. Our implementation splits them out. Adjust + * for that here. + */ + WT_ASSERT(session, + conn->evict_workers_max > 0 && conn->evict_workers_min > 0); + conn->evict_workers_max--; + conn->evict_workers_min--; return (0); } diff --git a/src/include/connection.h b/src/include/connection.h index 96c26097045..b07ece5a9aa 100644 --- a/src/include/connection.h +++ b/src/include/connection.h @@ -204,6 +204,7 @@ struct __wt_connection_impl { int evict_tid_set; /* Eviction server thread ID set */ uint32_t evict_workers_max;/* Max eviction workers */ + uint32_t evict_workers_min;/* Min eviction workers */ uint32_t evict_workers; /* Number of eviction workers */ WT_EVICT_WORKER *evict_workctx; /* Eviction worker context */ diff --git a/src/include/wiredtiger.in b/src/include/wiredtiger.in index 0b93181e01c..fb04c4743d4 100644 --- a/src/include/wiredtiger.in +++ b/src/include/wiredtiger.in @@ -1526,13 +1526,17 @@ struct __wt_connection { * total memory than the value\, as a percentage of the total cache * size. Must be less than \c eviction_trigger., an integer between 10 * and 99; default \c 80.} + * @config{eviction_thread_max, maximum number of threads WiredTiger + * will start to help evict pages from cache. The number of threads + * currently running will vary depending on the current eviction load., + * an integer between 1 and 20; default \c 3.} + * @config{eviction_thread_min, minimum number of threads WiredTiger + * will start to help evict pages from cache. The number of threads + * currently running will vary depending on the current eviction load., + * an integer between 1 and 20; default \c 1.} * @config{eviction_trigger, trigger eviction when the cache is using * this much memory\, as a percentage of the total cache size., an * integer between 10 and 99; default \c 95.} - * @config{eviction_worker_max, maximum number of additional threads - * WiredTiger will start to help evict pages from cache. The number of - * threads currently running will vary depending on the current eviction - * load., an integer between 0 and 20; default \c 0.} * @config{shared_cache = (, shared cache configuration options. A * database should configure either a cache_size or a shared_cache not * both., a set of related configuration options defined below.} @@ -1842,13 +1846,17 @@ struct __wt_connection { * @config{eviction_target, continue evicting until the cache has less total * memory than the value\, as a percentage of the total cache size. Must be * less than \c eviction_trigger., an integer between 10 and 99; default \c 80.} + * @config{eviction_thread_max, maximum number of threads WiredTiger will start + * to help evict pages from cache. The number of threads currently running will + * vary depending on the current eviction load., an integer between 1 and 20; + * default \c 3.} + * @config{eviction_thread_min, minimum number of threads WiredTiger will start + * to help evict pages from cache. The number of threads currently running will + * vary depending on the current eviction load., an integer between 1 and 20; + * default \c 1.} * @config{eviction_trigger, trigger eviction when the cache is using this much * memory\, as a percentage of the total cache size., an integer between 10 and * 99; default \c 95.} - * @config{eviction_worker_max, maximum number of additional threads WiredTiger - * will start to help evict pages from cache. The number of threads currently - * running will vary depending on the current eviction load., an integer between - * 0 and 20; default \c 0.} * @config{exclusive, fail if the database already exists\, generally used with * the \c create option., a boolean flag; default \c false.} * @config{extensions, list of shared library extensions to load (using dlopen). -- cgit v1.2.1 From a62d2c704a9a4ba2423622583115e6c8f7df512e Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Mon, 11 Aug 2014 03:08:31 +0000 Subject: Fix a bug in new eviction thread parsing. Update example file and documentation as well. --- examples/c/ex_all.c | 4 ++-- src/btree/bt_evict.c | 16 ++++++++++------ src/conn/conn_cache.c | 33 ++++++++++++++++----------------- src/docs/tune-cache.dox | 7 +++++-- 4 files changed, 33 insertions(+), 27 deletions(-) diff --git a/examples/c/ex_all.c b/examples/c/ex_all.c index 91551fb1532..d3cd9a2425d 100644 --- a/examples/c/ex_all.c +++ b/examples/c/ex_all.c @@ -1019,9 +1019,9 @@ main(void) (void)conn->close(conn, NULL); /*! [Eviction worker configuration] */ - /* Configure an additional 4 eviction support threads. */ + /* Configure up to four eviction threads */ ret = wiredtiger_open(home, NULL, - "create,eviction_trigger=90,eviction_workers=4", &conn); + "create,eviction_trigger=90,eviction_thread_max=4", &conn); /*! [Eviction worker configuration] */ if (ret == 0) (void)conn->close(conn, NULL); diff --git a/src/btree/bt_evict.c b/src/btree/bt_evict.c index e360981188b..df38e2e2ff5 100644 --- a/src/btree/bt_evict.c +++ b/src/btree/bt_evict.c @@ -169,9 +169,9 @@ __evict_server(void *arg) * number of eviction workers running, shut one down. */ if (conn->evict_workers > conn->evict_workers_min) { - WT_TRET(__wt_verbose(session, WT_VERB_EVICTSERVER, - "Stopping evict worker: %"PRIu32"\n", - conn->evict_workers)); + WT_TRET(__wt_verbose(session, WT_VERB_EVICTSERVER, + "Stopping evict worker: %"PRIu32"\n", + conn->evict_workers)); worker = &conn->evict_workctx[--conn->evict_workers]; F_CLR(worker, WT_EVICT_WORKER_RUN); WT_TRET(__wt_cond_signal( @@ -294,6 +294,10 @@ __wt_evict_destroy(WT_CONNECTION_IMPL *conn) WT_TRET(__wt_cond_signal(session, cache->evict_waiter_cond)); WT_TRET(__wt_thread_join(session, workers[i].tid)); } + for (i = 0; i < conn->evict_workers_max; i++) { + wt_session = &conn->evict_workctx[i].session->iface; + WT_TRET(wt_session->close(wt_session, NULL)); + } __wt_free(session, conn->evict_workctx); if (conn->evict_tid_set) { @@ -445,9 +449,9 @@ __evict_pass(WT_SESSION_IMPL *session) /* Start a worker if we have capacity and the cache is full. */ if (bytes_inuse > conn->cache_size && conn->evict_workers < conn->evict_workers_max) { - WT_RET(__wt_verbose(session, WT_VERB_EVICTSERVER, - "Starting evict worker: %"PRIu32"\n", - conn->evict_workers)); + WT_RET(__wt_verbose(session, WT_VERB_EVICTSERVER, + "Starting evict worker: %"PRIu32"\n", + conn->evict_workers)); worker = &conn->evict_workctx[conn->evict_workers++]; F_SET(worker, WT_EVICT_WORKER_RUN); WT_RET(__wt_thread_create(session, diff --git a/src/conn/conn_cache.c b/src/conn/conn_cache.c index 746df68758f..2b7ccdfc79c 100644 --- a/src/conn/conn_cache.c +++ b/src/conn/conn_cache.c @@ -54,29 +54,28 @@ __wt_cache_config(WT_CONNECTION_IMPL *conn, const char *cfg[]) cache->eviction_dirty_target = (u_int)cval.val; WT_RET_NOTFOUND_OK(ret); - if ((ret = - __wt_config_gets(session, cfg, "eviction_thread_max", &cval)) == 0) - conn->evict_workers_max = (u_int)cval.val; + /* + * The eviction thread configuration options include the main eviction + * thread and workers. Our implementation splits them out. Adjust for + * the difference when parsing the configuration. + */ + if ((ret = __wt_config_gets( + session, cfg, "eviction_thread_max", &cval)) == 0) { + WT_ASSERT(session, cval.val > 0); + conn->evict_workers_max = (u_int)cval.val - 1; + } WT_RET_NOTFOUND_OK(ret); - if ((ret = - __wt_config_gets(session, cfg, "eviction_thread_min", &cval)) == 0) - conn->evict_workers_min = (u_int)cval.val; + if ((ret = __wt_config_gets( + session, cfg, "eviction_thread_min", &cval)) == 0) { + WT_ASSERT(session, cval.val > 0); + conn->evict_workers_min = (u_int)cval.val - 1; + } WT_RET_NOTFOUND_OK(ret); if (conn->evict_workers_min > conn->evict_workers_max) WT_RET_MSG(session, EINVAL, "eviction_thread_min cannot be greater than " - "eviction_thread_max"); - - /* - * The eviction thread configuration options include the main eviction - * thread and workers. Our implementation splits them out. Adjust - * for that here. - */ - WT_ASSERT(session, - conn->evict_workers_max > 0 && conn->evict_workers_min > 0); - conn->evict_workers_max--; - conn->evict_workers_min--; + "eviction_thread_max"); return (0); } diff --git a/src/docs/tune-cache.dox b/src/docs/tune-cache.dox index 7cd977dce70..e4ec15f2f16 100644 --- a/src/docs/tune-cache.dox +++ b/src/docs/tune-cache.dox @@ -60,8 +60,11 @@ thread. In a large, busy cache, a single thread will be insufficient (especially when the eviction thread must wait for I/O). Further, if the eviction thread falls behind, application threads will be taken and used for eviction, potentially resulting in latency spikes. The \c -eviction_workers configuration value can be used to configure additional -WiredTiger threads to support eviction. +eviction_thread_min and \c eviction_thread_max configuration values can +be used to configure the number of threads WiredTiger can use to keep +up with the application eviction load. If WiredTiger eviction threads +are unable to evict enough space in the cache to keep up with +application demand, application threads will be do eviction as well. @snippet ex_all.c Eviction worker configuration -- cgit v1.2.1 From 309486ead1af78ae6b84e128d83f05d5fa6c4bfe Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Mon, 11 Aug 2014 03:15:14 +0000 Subject: Add upgrading doc for eviction thread API change. --- src/docs/upgrading.dox | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/docs/upgrading.dox b/src/docs/upgrading.dox index de0b9199a31..d13ff9e9917 100644 --- a/src/docs/upgrading.dox +++ b/src/docs/upgrading.dox @@ -1,5 +1,21 @@ /*! @page upgrading Upgrading WiredTiger applications +@section version_231 Upgrading to Version 2.3.1 + +
+
::wiredtiger_open eviction_workers configuration changed
+
+The \c eviction_workers configuration setting has been replaced by +\c eviction_thread_min and \c eviction_thread_max settings. To +replicate the old default behavior set both \c eviction_thread_min and +\c eviction_thread_max to 1. + +There is also a semantic change \c eviction_workers used to configure +additional threads whereas the new settings configure the total number +of threads involved with eviction. +
+
+ @section version_230 Upgrading to Version 2.3.0 There are no special upgrade steps required. -- cgit v1.2.1 From 35ea7e98d4ddff266a790759d3d0db0829f68f52 Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Mon, 11 Aug 2014 03:24:05 +0000 Subject: Fix an issue cleaning up eviction after a failed open. --- src/btree/bt_evict.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/btree/bt_evict.c b/src/btree/bt_evict.c index df38e2e2ff5..aa585af0ec5 100644 --- a/src/btree/bt_evict.c +++ b/src/btree/bt_evict.c @@ -294,11 +294,14 @@ __wt_evict_destroy(WT_CONNECTION_IMPL *conn) WT_TRET(__wt_cond_signal(session, cache->evict_waiter_cond)); WT_TRET(__wt_thread_join(session, workers[i].tid)); } - for (i = 0; i < conn->evict_workers_max; i++) { - wt_session = &conn->evict_workctx[i].session->iface; - WT_TRET(wt_session->close(wt_session, NULL)); + /* Handle shutdown when cleaning up after a failed open */ + if (conn->evict_workctx != NULL) { + for (i = 0; i < conn->evict_workers_max; i++) { + wt_session = &conn->evict_workctx[i].session->iface; + WT_TRET(wt_session->close(wt_session, NULL)); + } + __wt_free(session, conn->evict_workctx); } - __wt_free(session, conn->evict_workctx); if (conn->evict_tid_set) { WT_TRET(__wt_evict_server_wake(session)); -- cgit v1.2.1 From b81279f3c1758ce78cadf86250b2b159ceecd37c Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Mon, 11 Aug 2014 13:56:05 +1000 Subject: whitespace --- src/docs/upgrading.dox | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/docs/upgrading.dox b/src/docs/upgrading.dox index d13ff9e9917..ba95cb48637 100644 --- a/src/docs/upgrading.dox +++ b/src/docs/upgrading.dox @@ -6,7 +6,7 @@
::wiredtiger_open eviction_workers configuration changed
The \c eviction_workers configuration setting has been replaced by -\c eviction_thread_min and \c eviction_thread_max settings. To +\c eviction_thread_min and \c eviction_thread_max settings. To replicate the old default behavior set both \c eviction_thread_min and \c eviction_thread_max to 1. -- cgit v1.2.1 From 23f3141eb5f561ac315b7474e7236b4124f66bb3 Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Mon, 11 Aug 2014 14:35:39 +1000 Subject: Move the new eviction settings into a group. We can transition older settings into the group in the next major release: that will break existing applications. --- api/leveldb/leveldb_wt.h | 7 ++++--- dist/api_data.py | 24 ++++++++++++++---------- src/config/config_def.c | 42 ++++++++++++++++++++++-------------------- src/conn/conn_cache.c | 8 ++++---- src/docs/tune-cache.dox | 12 ++++++------ src/docs/upgrading.dox | 6 +++--- src/include/wiredtiger.in | 38 ++++++++++++++++++++++---------------- 7 files changed, 75 insertions(+), 62 deletions(-) diff --git a/api/leveldb/leveldb_wt.h b/api/leveldb/leveldb_wt.h index 0d169f81782..5e18346ce3b 100644 --- a/api/leveldb/leveldb_wt.h +++ b/api/leveldb/leveldb_wt.h @@ -45,9 +45,10 @@ #include "wiredtiger.h" #define WT_URI "table:data" -#define WT_CONN_CONFIG "log=(enabled),checkpoint=(wait=180),checkpoint_sync=false," \ - "session_max=8192,mmap=false,eviction_workers=4," \ - "transaction_sync=(enabled=true,method=none)," +#define WT_CONN_CONFIG \ + "log=(enabled),checkpoint=(wait=180),checkpoint_sync=false," \ + "session_max=8192,mmap=false," \ + "transaction_sync=(enabled=true,method=none)," // Note: LSM doesn't split, build full pages from the start #define WT_TABLE_CONFIG "type=lsm,split_pct=100,leaf_item_max=1KB," \ "lsm=(chunk_size=100MB,bloom_config=(leaf_page_max=8MB))," diff --git a/dist/api_data.py b/dist/api_data.py index 3f0fabdc3b9..45b2655d8f0 100644 --- a/dist/api_data.py +++ b/dist/api_data.py @@ -340,16 +340,20 @@ connection_runtime_config = [ Config('eviction_trigger', '95', r''' trigger eviction when the cache is using this much memory, as a percentage of the total cache size''', min=10, max=99), - Config('eviction_thread_max', '3', r''' - maximum number of threads WiredTiger will start to help - evict pages from cache. The number of threads currently running - will vary depending on the current eviction load''', - min=1, max=20), - Config('eviction_thread_min', '1', r''' - minimum number of threads WiredTiger will start to help evict - pages from cache. The number of threads currently running will - vary depending on the current eviction load''', - min=1, max=20), + Config('eviction', '', r''' + eviction configuration options.''', + type='category', subconfig=[ + Config('threads_max', '3', r''' + maximum number of threads WiredTiger will start to help + evict pages from cache. The number of threads currently running + will vary depending on the current eviction load''', + min=1, max=20), + Config('threads_min', '1', r''' + minimum number of threads WiredTiger will start to help evict + pages from cache. The number of threads currently running will + vary depending on the current eviction load''', + min=1, max=20), + ]), Config('shared_cache', '', r''' shared cache configuration options. A database should configure either a cache_size or a shared_cache not both''', diff --git a/src/config/config_def.c b/src/config/config_def.c index 1b688384286..52b4199ee0a 100644 --- a/src/config/config_def.c +++ b/src/config/config_def.c @@ -49,6 +49,12 @@ static const WT_CONFIG_CHECK confchk_checkpoint_subconfigs[] = { { NULL, NULL, NULL, NULL } }; +static const WT_CONFIG_CHECK confchk_eviction_subconfigs[] = { + { "threads_max", "int", "min=1,max=20", NULL }, + { "threads_min", "int", "min=1,max=20", NULL }, + { NULL, NULL, NULL, NULL } +}; + static const WT_CONFIG_CHECK confchk_shared_cache_subconfigs[] = { { "chunk", "int", "min=1MB,max=10TB", NULL }, { "name", "string", NULL, NULL }, @@ -72,10 +78,9 @@ static const WT_CONFIG_CHECK confchk_connection_reconfigure[] = { { "checkpoint", "category", NULL, confchk_checkpoint_subconfigs }, { "error_prefix", "string", NULL, NULL }, + { "eviction", "category", NULL, confchk_eviction_subconfigs }, { "eviction_dirty_target", "int", "min=10,max=99", NULL }, { "eviction_target", "int", "min=10,max=99", NULL }, - { "eviction_thread_max", "int", "min=1,max=20", NULL }, - { "eviction_thread_min", "int", "min=1,max=20", NULL }, { "eviction_trigger", "int", "min=10,max=99", NULL }, { "shared_cache", "category", NULL, confchk_shared_cache_subconfigs }, @@ -297,10 +302,9 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { "choices=[\"checkpoint\",\"data\",\"log\"]", NULL }, { "error_prefix", "string", NULL, NULL }, + { "eviction", "category", NULL, confchk_eviction_subconfigs }, { "eviction_dirty_target", "int", "min=10,max=99", NULL }, { "eviction_target", "int", "min=10,max=99", NULL }, - { "eviction_thread_max", "int", "min=1,max=20", NULL }, - { "eviction_thread_min", "int", "min=1,max=20", NULL }, { "eviction_trigger", "int", "min=10,max=99", NULL }, { "exclusive", "boolean", NULL, NULL }, { "extensions", "list", NULL, NULL }, @@ -372,12 +376,11 @@ static const WT_CONFIG_ENTRY config_entries[] = { { "connection.reconfigure", "async=(enabled=0,ops_max=1024,threads=2),cache_size=100MB," "checkpoint=(name=\"WiredTigerCheckpoint\",wait=0),error_prefix=," - "eviction_dirty_target=80,eviction_target=80," - "eviction_thread_max=3,eviction_thread_min=1,eviction_trigger=95," - "shared_cache=(chunk=10MB,name=,reserve=0,size=500MB)," - "statistics=none,statistics_log=(on_close=0," - "path=\"WiredTigerStat.%d.%H\",sources=," - "timestamp=\"%b %d %H:%M:%S\",wait=0),verbose=", + "eviction=(threads_max=3,threads_min=1),eviction_dirty_target=80," + "eviction_target=80,eviction_trigger=95,shared_cache=(chunk=10MB," + "name=,reserve=0,size=500MB),statistics=none," + "statistics_log=(on_close=0,path=\"WiredTigerStat.%d.%H\"," + "sources=,timestamp=\"%b %d %H:%M:%S\",wait=0),verbose=", confchk_connection_reconfigure }, { "cursor.close", @@ -483,16 +486,15 @@ static const WT_CONFIG_ENTRY config_entries[] = { "async=(enabled=0,ops_max=1024,threads=2),buffer_alignment=-1," "cache_size=100MB,checkpoint=(name=\"WiredTigerCheckpoint\"," "wait=0),checkpoint_sync=,create=0,direct_io=,error_prefix=," - "eviction_dirty_target=80,eviction_target=80," - "eviction_thread_max=3,eviction_thread_min=1,eviction_trigger=95," - "exclusive=0,extensions=,file_extend=,hazard_max=1000," - "log=(archive=,enabled=0,file_max=100MB,path=\"\"),lsm_merge=," - "mmap=,multiprocess=0,session_max=100,shared_cache=(chunk=10MB," - "name=,reserve=0,size=500MB),statistics=none," - "statistics_log=(on_close=0,path=\"WiredTigerStat.%d.%H\"," - "sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," - "transaction_sync=(enabled=0,method=fsync),use_environment_priv=0" - ",verbose=", + "eviction=(threads_max=3,threads_min=1),eviction_dirty_target=80," + "eviction_target=80,eviction_trigger=95,exclusive=0,extensions=," + "file_extend=,hazard_max=1000,log=(archive=,enabled=0," + "file_max=100MB,path=\"\"),lsm_merge=,mmap=,multiprocess=0," + "session_max=100,shared_cache=(chunk=10MB,name=,reserve=0," + "size=500MB),statistics=none,statistics_log=(on_close=0," + "path=\"WiredTigerStat.%d.%H\",sources=," + "timestamp=\"%b %d %H:%M:%S\",wait=0),transaction_sync=(enabled=0" + ",method=fsync),use_environment_priv=0,verbose=", confchk_wiredtiger_open }, { NULL, NULL, NULL } diff --git a/src/conn/conn_cache.c b/src/conn/conn_cache.c index 2b7ccdfc79c..ac594d3aa88 100644 --- a/src/conn/conn_cache.c +++ b/src/conn/conn_cache.c @@ -60,13 +60,13 @@ __wt_cache_config(WT_CONNECTION_IMPL *conn, const char *cfg[]) * the difference when parsing the configuration. */ if ((ret = __wt_config_gets( - session, cfg, "eviction_thread_max", &cval)) == 0) { + session, cfg, "eviction.threads_max", &cval)) == 0) { WT_ASSERT(session, cval.val > 0); conn->evict_workers_max = (u_int)cval.val - 1; } WT_RET_NOTFOUND_OK(ret); if ((ret = __wt_config_gets( - session, cfg, "eviction_thread_min", &cval)) == 0) { + session, cfg, "eviction.threads_min", &cval)) == 0) { WT_ASSERT(session, cval.val > 0); conn->evict_workers_min = (u_int)cval.val - 1; } @@ -74,8 +74,8 @@ __wt_cache_config(WT_CONNECTION_IMPL *conn, const char *cfg[]) if (conn->evict_workers_min > conn->evict_workers_max) WT_RET_MSG(session, EINVAL, - "eviction_thread_min cannot be greater than " - "eviction_thread_max"); + "eviction(threads_min) cannot be greater than " + "eviction(threads_max)"); return (0); } diff --git a/src/docs/tune-cache.dox b/src/docs/tune-cache.dox index e4ec15f2f16..5f1e2c060c3 100644 --- a/src/docs/tune-cache.dox +++ b/src/docs/tune-cache.dox @@ -59,12 +59,12 @@ By default, WiredTiger cache eviction is handled by a single, separate thread. In a large, busy cache, a single thread will be insufficient (especially when the eviction thread must wait for I/O). Further, if the eviction thread falls behind, application threads will be taken and -used for eviction, potentially resulting in latency spikes. The \c -eviction_thread_min and \c eviction_thread_max configuration values can -be used to configure the number of threads WiredTiger can use to keep -up with the application eviction load. If WiredTiger eviction threads -are unable to evict enough space in the cache to keep up with -application demand, application threads will be do eviction as well. +used for eviction, potentially resulting in latency spikes. The +\c eviction(threads_min) and \c eviction(threads_max) configuration values +can be used to configure the number of threads WiredTiger can use to keep +up with the application eviction load. If WiredTiger eviction threads are +unable to evict enough space in the cache to keep up with application +demand, application threads will be do eviction as well. @snippet ex_all.c Eviction worker configuration diff --git a/src/docs/upgrading.dox b/src/docs/upgrading.dox index ba95cb48637..1a210d386b3 100644 --- a/src/docs/upgrading.dox +++ b/src/docs/upgrading.dox @@ -6,9 +6,9 @@
::wiredtiger_open eviction_workers configuration changed
The \c eviction_workers configuration setting has been replaced by -\c eviction_thread_min and \c eviction_thread_max settings. To -replicate the old default behavior set both \c eviction_thread_min and -\c eviction_thread_max to 1. +\c eviction(threads_min) and \c eviction(threads_max) settings. To +replicate the old default behavior set +eviction(threads_min=1,threads_max=1). There is also a semantic change \c eviction_workers used to configure additional threads whereas the new settings configure the total number diff --git a/src/include/wiredtiger.in b/src/include/wiredtiger.in index fb04c4743d4..d5da1249457 100644 --- a/src/include/wiredtiger.in +++ b/src/include/wiredtiger.in @@ -1518,6 +1518,17 @@ struct __wt_connection { * @config{ ),,} * @config{error_prefix, prefix string for error messages., a string; * default empty.} + * @config{eviction = (, eviction configuration options., a set of + * related configuration options defined below.} + * @config{    threads_max, maximum number of + * threads WiredTiger will start to help evict pages from cache. The + * number of threads currently running will vary depending on the + * current eviction load., an integer between 1 and 20; default \c 3.} + * @config{    threads_min, minimum number of + * threads WiredTiger will start to help evict pages from cache. The + * number of threads currently running will vary depending on the + * current eviction load., an integer between 1 and 20; default \c 1.} + * @config{ ),,} * @config{eviction_dirty_target, continue evicting until the cache has * less dirty memory than the value\, as a percentage of the total cache * size. Dirty pages will only be evicted if the cache is full enough @@ -1526,14 +1537,6 @@ struct __wt_connection { * total memory than the value\, as a percentage of the total cache * size. Must be less than \c eviction_trigger., an integer between 10 * and 99; default \c 80.} - * @config{eviction_thread_max, maximum number of threads WiredTiger - * will start to help evict pages from cache. The number of threads - * currently running will vary depending on the current eviction load., - * an integer between 1 and 20; default \c 3.} - * @config{eviction_thread_min, minimum number of threads WiredTiger - * will start to help evict pages from cache. The number of threads - * currently running will vary depending on the current eviction load., - * an integer between 1 and 20; default \c 1.} * @config{eviction_trigger, trigger eviction when the cache is using * this much memory\, as a percentage of the total cache size., an * integer between 10 and 99; default \c 95.} @@ -1839,6 +1842,17 @@ struct __wt_connection { * "log"; default empty.} * @config{error_prefix, prefix string for error messages., a string; default * empty.} + * @config{eviction = (, eviction configuration options., a set of related + * configuration options defined below.} + * @config{    threads_max, maximum number of threads + * WiredTiger will start to help evict pages from cache. The number of threads + * currently running will vary depending on the current eviction load., an + * integer between 1 and 20; default \c 3.} + * @config{    threads_min, minimum number of threads + * WiredTiger will start to help evict pages from cache. The number of threads + * currently running will vary depending on the current eviction load., an + * integer between 1 and 20; default \c 1.} + * @config{ ),,} * @config{eviction_dirty_target, continue evicting until the cache has less * dirty memory than the value\, as a percentage of the total cache size. Dirty * pages will only be evicted if the cache is full enough to trigger eviction., @@ -1846,14 +1860,6 @@ struct __wt_connection { * @config{eviction_target, continue evicting until the cache has less total * memory than the value\, as a percentage of the total cache size. Must be * less than \c eviction_trigger., an integer between 10 and 99; default \c 80.} - * @config{eviction_thread_max, maximum number of threads WiredTiger will start - * to help evict pages from cache. The number of threads currently running will - * vary depending on the current eviction load., an integer between 1 and 20; - * default \c 3.} - * @config{eviction_thread_min, minimum number of threads WiredTiger will start - * to help evict pages from cache. The number of threads currently running will - * vary depending on the current eviction load., an integer between 1 and 20; - * default \c 1.} * @config{eviction_trigger, trigger eviction when the cache is using this much * memory\, as a percentage of the total cache size., an integer between 10 and * 99; default \c 95.} -- cgit v1.2.1 From 212631c1e5fd9fb01938ceed16fc9ad7402014b3 Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Mon, 11 Aug 2014 14:38:40 +1000 Subject: Change ex_all.c to match grouping of eviction settings. --- examples/c/ex_all.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/c/ex_all.c b/examples/c/ex_all.c index d3cd9a2425d..04ef32a8edb 100644 --- a/examples/c/ex_all.c +++ b/examples/c/ex_all.c @@ -1021,7 +1021,7 @@ main(void) /*! [Eviction worker configuration] */ /* Configure up to four eviction threads */ ret = wiredtiger_open(home, NULL, - "create,eviction_trigger=90,eviction_thread_max=4", &conn); + "create,eviction_trigger=90,eviction(threads_max=4)", &conn); /*! [Eviction worker configuration] */ if (ret == 0) (void)conn->close(conn, NULL); -- cgit v1.2.1 From e1cba5463d0e9e78d059f6a19f50ea04d9536c63 Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Mon, 11 Aug 2014 14:46:37 +1000 Subject: Engage brain: applications have to use =() syntax for nested configs. --- examples/c/ex_all.c | 2 +- src/conn/conn_cache.c | 4 ++-- src/docs/tune-cache.dox | 2 +- src/docs/upgrading.dox | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/c/ex_all.c b/examples/c/ex_all.c index 04ef32a8edb..278c8bb8750 100644 --- a/examples/c/ex_all.c +++ b/examples/c/ex_all.c @@ -1021,7 +1021,7 @@ main(void) /*! [Eviction worker configuration] */ /* Configure up to four eviction threads */ ret = wiredtiger_open(home, NULL, - "create,eviction_trigger=90,eviction(threads_max=4)", &conn); + "create,eviction_trigger=90,eviction=(threads_max=4)", &conn); /*! [Eviction worker configuration] */ if (ret == 0) (void)conn->close(conn, NULL); diff --git a/src/conn/conn_cache.c b/src/conn/conn_cache.c index ac594d3aa88..42e45a9c58b 100644 --- a/src/conn/conn_cache.c +++ b/src/conn/conn_cache.c @@ -74,8 +74,8 @@ __wt_cache_config(WT_CONNECTION_IMPL *conn, const char *cfg[]) if (conn->evict_workers_min > conn->evict_workers_max) WT_RET_MSG(session, EINVAL, - "eviction(threads_min) cannot be greater than " - "eviction(threads_max)"); + "eviction=(threads_min) cannot be greater than " + "eviction=(threads_max)"); return (0); } diff --git a/src/docs/tune-cache.dox b/src/docs/tune-cache.dox index 5f1e2c060c3..eb3cf3f18f7 100644 --- a/src/docs/tune-cache.dox +++ b/src/docs/tune-cache.dox @@ -60,7 +60,7 @@ thread. In a large, busy cache, a single thread will be insufficient (especially when the eviction thread must wait for I/O). Further, if the eviction thread falls behind, application threads will be taken and used for eviction, potentially resulting in latency spikes. The -\c eviction(threads_min) and \c eviction(threads_max) configuration values +\c eviction=(threads_min) and \c eviction=(threads_max) configuration values can be used to configure the number of threads WiredTiger can use to keep up with the application eviction load. If WiredTiger eviction threads are unable to evict enough space in the cache to keep up with application diff --git a/src/docs/upgrading.dox b/src/docs/upgrading.dox index 1a210d386b3..eaecc3e6b42 100644 --- a/src/docs/upgrading.dox +++ b/src/docs/upgrading.dox @@ -6,9 +6,9 @@
::wiredtiger_open eviction_workers configuration changed
The \c eviction_workers configuration setting has been replaced by -\c eviction(threads_min) and \c eviction(threads_max) settings. To +\c eviction=(threads_min) and \c eviction=(threads_max) settings. To replicate the old default behavior set -eviction(threads_min=1,threads_max=1). +eviction=(threads_min=1,threads_max=1). There is also a semantic change \c eviction_workers used to configure additional threads whereas the new settings configure the total number -- cgit v1.2.1