diff options
| -rw-r--r-- | memcached.c | 4 | ||||
| -rw-r--r-- | memcached.h | 1 | ||||
| -rw-r--r-- | proto_text.c | 3 | ||||
| -rw-r--r-- | storage.c | 29 | ||||
| -rwxr-xr-x | t/binary-extstore.t | 2 | ||||
| -rw-r--r-- | t/chunked-extstore.t | 2 | ||||
| -rw-r--r-- | t/error-extstore.t | 2 | ||||
| -rw-r--r-- | t/extstore-jbod.t | 2 | ||||
| -rw-r--r-- | t/extstore.t | 2 |
9 files changed, 32 insertions, 15 deletions
diff --git a/memcached.c b/memcached.c index f77294b..584bff1 100644 --- a/memcached.c +++ b/memcached.c @@ -1961,6 +1961,7 @@ void process_stat_settings(ADD_STAT add_stats, void *c) { APPEND_STAT("ext_wbuf_size", "%u", settings.ext_wbuf_size); APPEND_STAT("ext_compact_under", "%u", settings.ext_compact_under); APPEND_STAT("ext_drop_under", "%u", settings.ext_drop_under); + APPEND_STAT("ext_max_sleep", "%u", settings.ext_max_sleep); APPEND_STAT("ext_max_frag", "%.2f", settings.ext_max_frag); APPEND_STAT("slab_automove_freeratio", "%.3f", settings.slab_automove_freeratio); APPEND_STAT("ext_drop_unread", "%s", settings.ext_drop_unread ? "yes" : "no"); @@ -4040,12 +4041,13 @@ static void usage(void) { " - ext_drop_under: drop COLD items when fewer than this many free pages\n" " (default: 1/4th of the assigned storage)\n" " - ext_max_frag: max page fragmentation to tolerate (default: %.2f)\n" + " - ext_max_sleep: max sleep time of background threads in us (default: %u)\n" " - slab_automove_freeratio: ratio of memory to hold free as buffer.\n" " (see doc/storage.txt for more info, default: %.3f)\n", settings.ext_page_size / (1 << 20), settings.ext_wbuf_size / (1 << 20), settings.ext_io_threadcount, settings.ext_item_size, settings.ext_low_ttl, flag_enabled_disabled(settings.ext_drop_unread), settings.ext_recache_rate, - settings.ext_max_frag, settings.slab_automove_freeratio); + settings.ext_max_frag, settings.ext_max_sleep, settings.slab_automove_freeratio); verify_default("ext_item_age", settings.ext_item_age == UINT_MAX); #endif #ifdef PROXY diff --git a/memcached.h b/memcached.h index 0650515..3435677 100644 --- a/memcached.h +++ b/memcached.h @@ -495,6 +495,7 @@ struct settings { unsigned int ext_wbuf_size; /* read only note for the engine */ unsigned int ext_compact_under; /* when fewer than this many pages, compact */ unsigned int ext_drop_under; /* when fewer than this many pages, drop COLD items */ + unsigned int ext_max_sleep; /* maximum sleep time for extstore bg threads, in us */ double ext_max_frag; /* ideal maximum page fragmentation */ double slab_automove_freeratio; /* % of memory to hold free as buffer */ bool ext_drop_unread; /* skip unread items during compaction */ diff --git a/proto_text.c b/proto_text.c index 1024090..26e8eec 100644 --- a/proto_text.c +++ b/proto_text.c @@ -2441,6 +2441,9 @@ static void process_extstore_command(conn *c, token_t *tokens, const size_t ntok } else if (strcmp(tokens[1].value, "drop_under") == 0) { if (!safe_strtoul(tokens[2].value, &settings.ext_drop_under)) ok = false; + } else if (strcmp(tokens[1].value, "max_sleep") == 0) { + if (!safe_strtoul(tokens[2].value, &settings.ext_max_sleep)) + ok = false; } else if (strcmp(tokens[1].value, "max_frag") == 0) { if (!safe_strtod(tokens[2].value, &settings.ext_max_frag)) ok = false; @@ -571,7 +571,6 @@ static int storage_write(void *storage, const int clsid, const int item_age) { static pthread_t storage_write_tid; static pthread_mutex_t storage_write_plock; -#define WRITE_SLEEP_MAX 1000000 #define WRITE_SLEEP_MIN 500 static void *storage_write_thread(void *arg) { @@ -594,8 +593,8 @@ static void *storage_write_thread(void *arg) { int min_class = slabs_clsid(settings.ext_item_size); bool do_sleep = true; counter++; - if (to_sleep > WRITE_SLEEP_MAX) - to_sleep = WRITE_SLEEP_MAX; + if (to_sleep > settings.ext_max_sleep) + to_sleep = settings.ext_max_sleep; for (int x = 0; x < MAX_NUMBER_OF_SLAB_CLASSES; x++) { bool did_move = false; @@ -765,7 +764,6 @@ static int storage_compact_check(void *storage, logger *l, static pthread_t storage_compact_tid; static pthread_mutex_t storage_compact_plock; #define MIN_STORAGE_COMPACT_SLEEP 10000 -#define MAX_STORAGE_COMPACT_SLEEP 2000000 struct storage_compact_wrap { obj_io io; @@ -888,7 +886,7 @@ static void _storage_compact_cb(void *e, obj_io *io, int ret) { // I guess it's only COLD. that's probably fine. static void *storage_compact_thread(void *arg) { void *storage = arg; - useconds_t to_sleep = MAX_STORAGE_COMPACT_SLEEP; + useconds_t to_sleep = settings.ext_max_sleep; bool compacting = false; uint64_t page_version = 0; uint64_t page_size = 0; @@ -974,11 +972,11 @@ static void *storage_compact_thread(void *arg) { } pthread_mutex_unlock(&wrap.lock); - if (to_sleep > MIN_STORAGE_COMPACT_SLEEP) - to_sleep /= 2; + // finish actual compaction quickly. + to_sleep = MIN_STORAGE_COMPACT_SLEEP; } else { - if (to_sleep < MAX_STORAGE_COMPACT_SLEEP) - to_sleep += MIN_STORAGE_COMPACT_SLEEP; + if (to_sleep < settings.ext_max_sleep) + to_sleep += settings.ext_max_sleep; } } free(readback_buf); @@ -1124,6 +1122,7 @@ void *storage_init_config(struct settings *s) { s->ext_wbuf_size = 1024 * 1024 * 4; s->ext_compact_under = 0; s->ext_drop_under = 0; + s->ext_max_sleep = 1000000; s->slab_automove_freeratio = 0.01; s->ext_page_size = 1024 * 1024 * 64; s->ext_io_threadcount = 1; @@ -1155,6 +1154,7 @@ int storage_read_config(void *conf, char **subopt) { EXT_RECACHE_RATE, EXT_COMPACT_UNDER, EXT_DROP_UNDER, + EXT_MAX_SLEEP, EXT_MAX_FRAG, EXT_DROP_UNREAD, SLAB_AUTOMOVE_FREERATIO, // FIXME: move this back? @@ -1172,6 +1172,7 @@ int storage_read_config(void *conf, char **subopt) { [EXT_RECACHE_RATE] = "ext_recache_rate", [EXT_COMPACT_UNDER] = "ext_compact_under", [EXT_DROP_UNDER] = "ext_drop_under", + [EXT_MAX_SLEEP] = "ext_max_sleep", [EXT_MAX_FRAG] = "ext_max_frag", [EXT_DROP_UNREAD] = "ext_drop_unread", [SLAB_AUTOMOVE_FREERATIO] = "slab_automove_freeratio", @@ -1286,6 +1287,16 @@ int storage_read_config(void *conf, char **subopt) { return 1; } break; + case EXT_MAX_SLEEP: + if (subopts_value == NULL) { + fprintf(stderr, "Missing ext_max_sleep argument\n"); + return 1; + } + if (!safe_strtoul(subopts_value, &settings.ext_max_sleep)) { + fprintf(stderr, "could not parse argument to ext_max_sleep\n"); + return 1; + } + break; case EXT_MAX_FRAG: if (subopts_value == NULL) { fprintf(stderr, "Missing ext_max_frag argument\n"); diff --git a/t/binary-extstore.t b/t/binary-extstore.t index 746fe52..39de347 100755 --- a/t/binary-extstore.t +++ b/t/binary-extstore.t @@ -17,7 +17,7 @@ if (!supports_extstore()) { $ext_path = "/tmp/extstore.$$"; -my $server = new_memcached("-m 64 -U 0 -o ext_page_size=8,ext_wbuf_size=2,ext_threads=1,ext_io_depth=2,ext_item_size=512,ext_item_age=2,ext_recache_rate=10000,ext_max_frag=0.9,ext_path=$ext_path:64m,no_lru_crawler,slab_automove=0"); +my $server = new_memcached("-m 64 -U 0 -o ext_page_size=8,ext_wbuf_size=2,ext_threads=1,ext_io_depth=2,ext_item_size=512,ext_item_age=2,ext_recache_rate=10000,ext_max_frag=0.9,ext_path=$ext_path:64m,no_lru_crawler,slab_automove=0,ext_max_sleep=100000"); ok($server, "started the server"); # Based almost 100% off testClient.py which is: diff --git a/t/chunked-extstore.t b/t/chunked-extstore.t index 0e3a72c..2ffd612 100644 --- a/t/chunked-extstore.t +++ b/t/chunked-extstore.t @@ -18,7 +18,7 @@ if (!supports_extstore()) { $ext_path = "/tmp/extstore.$$"; -my $server = new_memcached("-m 64 -U 0 -o ext_page_size=8,ext_wbuf_size=2,ext_threads=1,ext_io_depth=2,ext_item_size=512,ext_item_age=2,ext_recache_rate=0,ext_max_frag=0,ext_path=$ext_path:64m,slab_chunk_max=16384,slab_automove=0"); +my $server = new_memcached("-m 64 -U 0 -o ext_page_size=8,ext_wbuf_size=2,ext_threads=1,ext_io_depth=2,ext_item_size=512,ext_item_age=2,ext_recache_rate=0,ext_max_frag=0,ext_path=$ext_path:64m,slab_chunk_max=16384,slab_automove=0,ext_max_sleep=100000"); my $sock = $server->sock; # Only able to disable compaction at runtime. diff --git a/t/error-extstore.t b/t/error-extstore.t index ed196bc..865360c 100644 --- a/t/error-extstore.t +++ b/t/error-extstore.t @@ -20,7 +20,7 @@ if (!supports_extstore()) { $ext_path = "/tmp/extstore.$$"; -my $server = new_memcached("-m 64 -I 4m -U 0 -o ext_page_size=8,ext_wbuf_size=8,ext_threads=1,ext_io_depth=2,ext_item_size=512,ext_item_age=2,ext_recache_rate=10000,ext_max_frag=0.9,ext_path=$ext_path:64m,slab_automove=0,ext_compact_under=1"); +my $server = new_memcached("-m 64 -I 4m -U 0 -o ext_page_size=8,ext_wbuf_size=8,ext_threads=1,ext_io_depth=2,ext_item_size=512,ext_item_age=2,ext_recache_rate=10000,ext_max_frag=0.9,ext_path=$ext_path:64m,slab_automove=0,ext_compact_under=1,ext_max_sleep=100000"); my $sock = $server->sock; # Wait until all items have flushed diff --git a/t/extstore-jbod.t b/t/extstore-jbod.t index 1618803..de8e5e8 100644 --- a/t/extstore-jbod.t +++ b/t/extstore-jbod.t @@ -19,7 +19,7 @@ if (!supports_extstore()) { $ext_path = "/tmp/extstore1.$$"; $ext_path2 = "/tmp/extstore2.$$"; -my $server = new_memcached("-m 256 -U 0 -o ext_page_size=8,ext_wbuf_size=2,ext_threads=1,ext_io_depth=2,ext_item_size=512,ext_item_age=2,ext_recache_rate=10000,ext_max_frag=0.9,ext_path=$ext_path:64m,ext_path=$ext_path2:96m,slab_automove=1"); +my $server = new_memcached("-m 256 -U 0 -o ext_page_size=8,ext_wbuf_size=2,ext_threads=1,ext_io_depth=2,ext_item_size=512,ext_item_age=2,ext_recache_rate=10000,ext_max_frag=0.9,ext_path=$ext_path:64m,ext_path=$ext_path2:96m,slab_automove=1,ext_max_sleep=100000"); my $sock = $server->sock; my $value; diff --git a/t/extstore.t b/t/extstore.t index 116a7f4..0304662 100644 --- a/t/extstore.t +++ b/t/extstore.t @@ -17,7 +17,7 @@ if (!supports_extstore()) { $ext_path = "/tmp/extstore.$$"; -my $server = new_memcached("-m 64 -U 0 -o ext_page_size=8,ext_wbuf_size=2,ext_threads=1,ext_io_depth=2,ext_item_size=512,ext_item_age=2,ext_recache_rate=10000,ext_max_frag=0.9,ext_path=$ext_path:64m,slab_automove=0,ext_compact_under=1"); +my $server = new_memcached("-m 64 -U 0 -o ext_page_size=8,ext_wbuf_size=2,ext_threads=1,ext_io_depth=2,ext_item_size=512,ext_item_age=2,ext_recache_rate=10000,ext_max_frag=0.9,ext_path=$ext_path:64m,slab_automove=0,ext_compact_under=1,ext_max_sleep=100000"); my $sock = $server->sock; eval { |