diff options
62 files changed, 1252 insertions, 777 deletions
diff --git a/bench/wtperf/wtperf.c b/bench/wtperf/wtperf.c index b9d72e45184..8780d270664 100644 --- a/bench/wtperf/wtperf.c +++ b/bench/wtperf/wtperf.c @@ -833,7 +833,17 @@ populate_thread(void *arg) if (cfg->random_value) randomize_value(thread, value_buf); cursor->set_value(cursor, value_buf); - if ((ret = cursor->insert(cursor)) != 0) { + if ((ret = cursor->insert(cursor)) == WT_ROLLBACK) { + lprintf(cfg, ret, 0, "insert retrying"); + if ((ret = session->rollback_transaction( + session, NULL)) != 0) { + lprintf(cfg, ret, 0, + "Failed rollback_transaction"); + goto err; + } + intxn = 0; + continue; + } else if (ret != 0) { lprintf(cfg, ret, 0, "Failed inserting"); goto err; } diff --git a/build_win/wiredtiger.def b/build_win/wiredtiger.def index 02884e4fd65..86096fb778d 100644 --- a/build_win/wiredtiger.def +++ b/build_win/wiredtiger.def @@ -9,7 +9,6 @@ EXPORTS wiredtiger_pack_str wiredtiger_pack_uint wiredtiger_strerror - wiredtiger_strerror_r wiredtiger_struct_pack wiredtiger_struct_size wiredtiger_struct_unpack diff --git a/dist/api_data.py b/dist/api_data.py index 82335e3f831..feb51011309 100644 --- a/dist/api_data.py +++ b/dist/api_data.py @@ -325,13 +325,13 @@ connection_runtime_config = [ min='1MB', max='10TB'), Config('cache_overhead', '8', r''' assume the heap allocator overhead is the specified percentage, and - adjust the cache size by that amount (for example, if the cache size is - 100GB, a percentage of 10 means WiredTiger limits itself to allocating - 90GB of memory). This value is configurable because different heap - allocators have different overhead and different workloads will have - different heap allocation sizes and patterns, therefore applications - may need to adjust this value based on allocator choice and behavior - in measured workloads''', + adjust the cache usage by that amount (for example, if there is 10GB + of data in cache, a percentage of 10 means WiredTiger treats this as + 11GB). This value is configurable because different heap allocators + have different overhead and different workloads will have different + heap allocation sizes and patterns, therefore applications may need to + adjust this value based on allocator choice and behavior in measured + workloads''', min='0', max='30'), Config('checkpoint', '', r''' periodically checkpoint the database''', @@ -550,6 +550,10 @@ common_wiredtiger_open = [ Config('prealloc', 'true', r''' pre-allocate log files.''', type='boolean'), + Config('recover', 'on', r''' + run recovery or error if recovery needs to run after an + unclean shutdown.''', + choices=['error','on']), ]), Config('mmap', 'true', r''' Use memory mapping to access files when possible''', @@ -719,24 +723,29 @@ methods = { files''', type='boolean'), ]), +'session.strerror' : Method([]), 'session.truncate' : Method([]), 'session.upgrade' : Method([]), 'session.verify' : Method([ Config('dump_address', 'false', r''' - Display addresses and page types as pages are verified, using - the application's message handler, intended for debugging''', + Display addresses and page types as pages are verified, + using the application's message handler, intended for debugging''', type='boolean'), Config('dump_blocks', 'false', r''' - Display the contents of on-disk blocks as they are verified, using - the application's message handler, intended for debugging''', + Display the contents of on-disk blocks as they are verified, + using the application's message handler, intended for debugging''', type='boolean'), Config('dump_offsets', '', r''' - Display the contents of specific on-disk blocks, using - the application's message handler, intended for debugging''', + Display the contents of specific on-disk blocks, + using the application's message handler, intended for debugging''', type='list'), Config('dump_pages', 'false', r''' - Display the contents of in-memory pages as they are verified, using - the application's message handler, intended for debugging''', + Display the contents of in-memory pages as they are verified, + using the application's message handler, intended for debugging''', + type='boolean'), + Config('dump_shape', 'false', r''' + Display the shape of the tree after verification, + using the application's message handler, intended for debugging''', type='boolean') ]), diff --git a/dist/api_err.py b/dist/api_err.py index 6c893c9af82..d39f076656f 100644 --- a/dist/api_err.py +++ b/dist/api_err.py @@ -47,6 +47,10 @@ errors = [ interface, no further WiredTiger calls are required.'''), Error('WT_RESTART', -31805, 'restart the operation (internal)', undoc=True), + Error('WT_RUN_RECOVERY', -31806, + 'recovery must be run to continue', ''' + This error is generated when wiredtiger_open is configured + to return an error if recovery is required to use the database.'''), ] # Update the #defines in the wiredtiger.in file. @@ -88,28 +92,41 @@ tfile.write('''/* DO NOT EDIT: automatically built by dist/api_err.py. */ /* * Historically, there was only the wiredtiger_strerror call because the POSIX * port didn't need anything more complex; Windows requires memory allocation - * of error strings, so we added the wiredtiger_strerror_r call. Because we + * of error strings, so we added the WT_SESSION.strerror method. Because we * want wiredtiger_strerror to continue to be as thread-safe as possible, errors - * are split into three categories: WiredTiger constant strings, system constant - * strings and Everything Else, and we check constant strings before Everything - * Else. + * are split into two categories: WiredTiger's or the system's constant strings + * and Everything Else, and we check constant strings before Everything Else. */ /* - * __wiredtiger_error -- - *\tReturn a constant string for the WiredTiger errors. + * __wt_wiredtiger_error -- + *\tReturn a constant string for WiredTiger POSIX-standard and errors. */ -static const char * -__wiredtiger_error(int error) +const char * +__wt_wiredtiger_error(int error) { +\tconst char *p; + +\t/* +\t * Check for WiredTiger specific errors. +\t */ \tswitch (error) { ''') for err in errors: tfile.write('\tcase ' + err.name + ':\n') tfile.write('\t\treturn ("' + err.name + ': ' + err.desc + '");\n') - tfile.write('''\t} + +\t/* +\t * POSIX errors are non-negative integers; check for 0 explicitly +\t * in-case the underlying strerror doesn't handle 0, some don't. +\t */ +\tif (error == 0) +\t\treturn ("Successful return: 0"); +\tif (error > 0 && (p = strerror(error)) != NULL) +\t\treturn (p); + \treturn (NULL); } @@ -121,39 +138,8 @@ const char * wiredtiger_strerror(int error) { \tstatic char buf[128]; -\tconst char *p; - -\t/* Check for a constant string. */ -\tif ((p = __wiredtiger_error(error)) != NULL || -\t (p = __wt_strerror(error)) != NULL) -\t\treturn (p); - -\t/* Else, fill in the non-thread-safe static buffer. */ -\tif (wiredtiger_strerror_r(error, buf, sizeof(buf)) != 0) -\t\t(void)snprintf(buf, sizeof(buf), "error return: %d", error); - -\treturn (buf); -} - -/* - * wiredtiger_strerror_r -- - *\tReturn a string for any error value, thread-safe version. - */ -int -wiredtiger_strerror_r(int error, char *buf, size_t buflen) -{ -\tconst char *p; - -\t/* Require at least 2 bytes, printable character and trailing nul. */ -\tif (buflen < 2) -\t\treturn (ENOMEM); - -\t/* Check for a constant string. */ -\tif ((p = __wiredtiger_error(error)) != NULL || -\t (p = __wt_strerror(error)) != NULL) -\t\treturn (snprintf(buf, buflen, "%s", p) > 0 ? 0 : ENOMEM); -\treturn (__wt_strerror_r(error, buf, buflen)); +\treturn (__wt_strerror(NULL, error, buf, sizeof(buf))); } ''') tfile.close() diff --git a/dist/s_export.list b/dist/s_export.list index 8f469e94433..d3803bc3afa 100644 --- a/dist/s_export.list +++ b/dist/s_export.list @@ -8,7 +8,6 @@ wiredtiger_pack_start wiredtiger_pack_str wiredtiger_pack_uint wiredtiger_strerror -wiredtiger_strerror_r wiredtiger_struct_pack wiredtiger_struct_size wiredtiger_struct_unpack diff --git a/dist/s_string.ok b/dist/s_string.ok index 1658684313c..66439faf161 100644 --- a/dist/s_string.ok +++ b/dist/s_string.ok @@ -283,6 +283,7 @@ RNG ROCKSDB RPC RUNDIR +RVv Radu Recno Recurse @@ -1073,6 +1074,7 @@ treplacement trk trk's troot +trun trunc trylock trywrlock diff --git a/examples/c/ex_all.c b/examples/c/ex_all.c index 51b4bb42040..51c63e307c6 100644 --- a/examples/c/ex_all.c +++ b/examples/c/ex_all.c @@ -357,12 +357,8 @@ cursor_ops(WT_SESSION *session) const char *key = "non-existent key"; cursor->set_key(cursor, key); if ((ret = cursor->remove(cursor)) != 0) { - char buf[128]; - - if (wiredtiger_strerror_r(ret, buf, sizeof(buf)) != 0) - (void)snprintf( - buf, sizeof(buf), "error value: %d\n", ret); - fprintf(stderr, "cursor.remove: %s\n", buf); + fprintf(stderr, + "cursor.remove: %s\n", session->strerror(session, ret)); return (ret); } /*! [Display an error thread safe] */ diff --git a/examples/c/ex_backup.c b/examples/c/ex_backup.c index 6088c4d6473..f58f28dbd93 100644 --- a/examples/c/ex_backup.c +++ b/examples/c/ex_backup.c @@ -72,18 +72,18 @@ compare_backups(int i) */ if (i == 0) (void)snprintf(buf, sizeof(buf), - "../../wt -h %s dump logtest > %s.%d", + "../../wt -R -h %s dump logtest > %s.%d", home, full_out, i); else (void)snprintf(buf, sizeof(buf), - "../../wt -h %s.%d dump logtest > %s.%d", + "../../wt -R -h %s.%d dump logtest > %s.%d", home_full, i, full_out, i); ret = system(buf); /* * Now run dump on the incremental directory. */ (void)snprintf(buf, sizeof(buf), - "../../wt -h %s.%d dump logtest > %s.%d", + "../../wt -R -h %s.%d dump logtest > %s.%d", home_incr, i, incr_out, i); ret = system(buf); diff --git a/examples/c/ex_data_source.c b/examples/c/ex_data_source.c index 5043fa1b67d..7fb400b5922 100644 --- a/examples/c/ex_data_source.c +++ b/examples/c/ex_data_source.c @@ -75,8 +75,9 @@ my_create(WT_DATA_SOURCE *dsrc, WT_SESSION *session, { int ret = 0; /*! [WT_EXTENSION_API strerror] */ - (void)wt_api->err_printf(wt_api, - session, "WiredTiger error return: %s", wt_api->strerror(ret)); + (void)wt_api->err_printf(wt_api, session, + "WiredTiger error return: %s", + wt_api->strerror(wt_api, session, ret)); /*! [WT_EXTENSION_API strerror] */ } diff --git a/ext/datasources/helium/helium.c b/ext/datasources/helium/helium.c index d62ecb846e9..3fc521d93b2 100644 --- a/ext/datasources/helium/helium.c +++ b/ext/datasources/helium/helium.c @@ -1913,7 +1913,7 @@ bad_name: ERET(wtext, session, EINVAL, "%s: illegal name format", uri); if (ret != 0 && ret != WT_NOTFOUND) EMSG_ERR(wtext, session, ret, "helium_o_truncate configuration: %s", - wtext->strerror(ret)); + wtext->strerror(wtext, session, ret)); if ((ret = ws_source_open_object( wtds, session, hs, uri, NULL, oflags, &ws->he)) != 0) @@ -2041,7 +2041,8 @@ master_uri_set(WT_DATA_SOURCE *wtds, exclusive = a.val != 0; else if (ret != WT_NOTFOUND) ERET(wtext, session, ret, - "exclusive configuration: %s", wtext->strerror(ret)); + "exclusive configuration: %s", + wtext->strerror(wtext, session, ret)); /* Get the key/value format strings. */ if ((ret = wtext->config_get( @@ -2052,7 +2053,7 @@ master_uri_set(WT_DATA_SOURCE *wtds, } else ERET(wtext, session, ret, "key_format configuration: %s", - wtext->strerror(ret)); + wtext->strerror(wtext, session, ret)); } if ((ret = wtext->config_get( wtext, session, config, "value_format", &b)) != 0) { @@ -2062,7 +2063,7 @@ master_uri_set(WT_DATA_SOURCE *wtds, } else ERET(wtext, session, ret, "value_format configuration: %s", - wtext->strerror(ret)); + wtext->strerror(wtext, session, ret)); } /* Get the compression configuration. */ @@ -2073,7 +2074,7 @@ master_uri_set(WT_DATA_SOURCE *wtds, else ERET(wtext, session, ret, "helium_o_compress configuration: %s", - wtext->strerror(ret)); + wtext->strerror(wtext, session, ret)); } /* @@ -2090,7 +2091,8 @@ master_uri_set(WT_DATA_SOURCE *wtds, return (0); if (ret == WT_DUPLICATE_KEY) return (exclusive ? EEXIST : 0); - ERET(wtext, session, ret, "%s: %s", uri, wtext->strerror(ret)); + ERET(wtext, + session, ret, "%s: %s", uri, wtext->strerror(wtext, session, ret)); } /* @@ -2129,19 +2131,22 @@ helium_session_open_cursor(WT_DATA_SOURCE *wtds, WT_SESSION *session, if ((ret = wtext->config_get( /* Parse configuration */ wtext, session, config, "append", &v)) != 0) EMSG_ERR(wtext, session, ret, - "append configuration: %s", wtext->strerror(ret)); + "append configuration: %s", + wtext->strerror(wtext, session, ret)); cursor->config_append = v.val != 0; if ((ret = wtext->config_get( wtext, session, config, "overwrite", &v)) != 0) EMSG_ERR(wtext, session, ret, - "overwrite configuration: %s", wtext->strerror(ret)); + "overwrite configuration: %s", + wtext->strerror(wtext, session, ret)); cursor->config_overwrite = v.val != 0; if ((ret = wtext->collator_config( wtext, session, uri, config, NULL, &own)) != 0) EMSG_ERR(wtext, session, ret, - "collator configuration: %s", wtext->strerror(ret)); + "collator configuration: %s", + wtext->strerror(wtext, session, ret)); /* Finish initializing the cursor. */ cursor->wtcursor.close = helium_cursor_close; @@ -2178,19 +2183,19 @@ helium_session_open_cursor(WT_DATA_SOURCE *wtds, WT_SESSION *session, session, value, strlen(value), &config_parser)) != 0) EMSG_ERR(wtext, session, ret, "Configuration string parser: %s", - wtext->strerror(ret)); + wtext->strerror(wtext, session, ret)); if ((ret = config_parser->get( config_parser, "key_format", &v)) != 0) EMSG_ERR(wtext, session, ret, "key_format configuration: %s", - wtext->strerror(ret)); + wtext->strerror(wtext, session, ret)); ws->config_recno = v.len == 1 && v.str[0] == 'r'; if ((ret = config_parser->get( config_parser, "value_format", &v)) != 0) EMSG_ERR(wtext, session, ret, "value_format configuration: %s", - wtext->strerror(ret)); + wtext->strerror(wtext, session, ret)); ws->config_bitfield = v.len == 2 && isdigit(v.str[0]) && v.str[1] == 't'; @@ -2198,7 +2203,7 @@ helium_session_open_cursor(WT_DATA_SOURCE *wtds, WT_SESSION *session, config_parser, "helium_o_compress", &v)) != 0) EMSG_ERR(wtext, session, ret, "helium_o_compress configuration: %s", - wtext->strerror(ret)); + wtext->strerror(wtext, session, ret)); ws->config_compress = v.val ? 1 : 0; /* @@ -2237,7 +2242,8 @@ err: if (ws != NULL && locked) if (config_parser != NULL && (tret = config_parser->close(config_parser)) != 0) EMSG(wtext, session, tret, - "WT_CONFIG_PARSER.close: %s", wtext->strerror(tret)); + "WT_CONFIG_PARSER.close: %s", + wtext->strerror(wtext, session, tret)); free((void *)value); return (ret); @@ -2913,7 +2919,7 @@ helium_config_read(WT_EXTENSION_API *wtext, WT_CONFIG_ITEM *config, wtext, NULL, config->str, config->len, &config_parser)) != 0) ERET(wtext, NULL, ret, "WT_EXTENSION_API.config_parser_open: %s", - wtext->strerror(ret)); + wtext->strerror(wtext, NULL, ret)); while ((ret = config_parser->next(config_parser, &k, &v)) == 0) { if (string_match("helium_devices", k.str, k.len)) { if ((*devicep = calloc(1, v.len + 1)) == NULL) @@ -2944,11 +2950,13 @@ helium_config_read(WT_EXTENSION_API *wtext, WT_CONFIG_ITEM *config, ret = 0; if (ret != 0) EMSG_ERR(wtext, NULL, ret, - "WT_CONFIG_PARSER.next: %s", wtext->strerror(ret)); + "WT_CONFIG_PARSER.next: %s", + wtext->strerror(wtext, NULL, ret)); err: if ((tret = config_parser->close(config_parser)) != 0) EMSG(wtext, NULL, tret, - "WT_CONFIG_PARSER.close: %s", wtext->strerror(tret)); + "WT_CONFIG_PARSER.close: %s", + wtext->strerror(wtext, NULL, tret)); return (ret); } @@ -3373,14 +3381,14 @@ wiredtiger_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) if ((ret = wtext->config_get(wtext, NULL, config, "config", &v)) != 0) EMSG_ERR(wtext, NULL, ret, "WT_EXTENSION_API.config_get: config: %s", - wtext->strerror(ret)); + wtext->strerror(wtext, NULL, ret)); /* Step through the list of Helium sources, opening each one. */ if ((ret = wtext->config_parser_open( wtext, NULL, v.str, v.len, &config_parser)) != 0) EMSG_ERR(wtext, NULL, ret, "WT_EXTENSION_API.config_parser_open: config: %s", - wtext->strerror(ret)); + wtext->strerror(wtext, NULL, ret)); while ((ret = config_parser->next(config_parser, &k, &v)) == 0) { if (string_match("helium_verbose", k.str, k.len)) { verbose = v.val == 0 ? 0 : 1; @@ -3392,11 +3400,11 @@ wiredtiger_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) if (ret != WT_NOTFOUND) EMSG_ERR(wtext, NULL, ret, "WT_CONFIG_PARSER.next: config: %s", - wtext->strerror(ret)); + wtext->strerror(wtext, NULL, ret)); if ((ret = config_parser->close(config_parser)) != 0) EMSG_ERR(wtext, NULL, ret, "WT_CONFIG_PARSER.close: config: %s", - wtext->strerror(ret)); + wtext->strerror(wtext, NULL, ret)); config_parser = NULL; /* Find and open the database transaction store. */ @@ -3423,13 +3431,14 @@ wiredtiger_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) EMSG_ERR(wtext, NULL, ret, "WT_CONNECTION.configure_method: session.create: " "%s: %s", - *p, wtext->strerror(ret)); + *p, wtext->strerror(wtext, NULL, ret)); /* Add the data source */ if ((ret = connection->add_data_source( connection, "helium:", (WT_DATA_SOURCE *)ds, NULL)) != 0) EMSG_ERR(wtext, NULL, ret, - "WT_CONNECTION.add_data_source: %s", wtext->strerror(ret)); + "WT_CONNECTION.add_data_source: %s", + wtext->strerror(wtext, NULL, ret)); return (0); err: if (ds != NULL) diff --git a/ext/test/kvs_bdb/kvs_bdb.c b/ext/test/kvs_bdb/kvs_bdb.c index 48305937236..76dccb89bf2 100644 --- a/ext/test/kvs_bdb/kvs_bdb.c +++ b/ext/test/kvs_bdb/kvs_bdb.c @@ -722,7 +722,8 @@ kvs_session_create(WT_DATA_SOURCE *wtds, if ((ret = wtext->config_get(wtext, session, config, "key_format", &v)) != 0) ERET(wtext, session, ret, - "key_format configuration: %s", wtext->strerror(ret)); + "key_format configuration: %s", + wtext->strerror(wtext, session, ret)); type = v.len == 1 && v.str[0] == 'r' ? DB_RECNO : DB_BTREE; /* Create the Berkeley DB table */ @@ -798,7 +799,8 @@ kvs_session_open_cursor(WT_DATA_SOURCE *wtds, WT_SESSION *session, if ((ret = wtext->config_get( wtext, session, config, "append", &v)) != 0) { ESET(wtext, session, ret, - "append configuration: %s", wtext->strerror(ret)); + "append configuration: %s", + wtext->strerror(wtext, session, ret)); goto err; } cursor->config_append = v.val != 0; @@ -806,7 +808,8 @@ kvs_session_open_cursor(WT_DATA_SOURCE *wtds, WT_SESSION *session, if ((ret = wtext->config_get( wtext, session, config, "overwrite", &v)) != 0) { ESET(wtext, session, ret, - "overwrite configuration: %s", wtext->strerror(ret)); + "overwrite configuration: %s", + wtext->strerror(wtext, session, ret)); goto err; } cursor->config_overwrite = v.val != 0; @@ -814,7 +817,8 @@ kvs_session_open_cursor(WT_DATA_SOURCE *wtds, WT_SESSION *session, if ((ret = wtext->config_get( wtext, session, config, "key_format", &v)) != 0) { ESET(wtext, session, ret, - "key_format configuration: %s", wtext->strerror(ret)); + "key_format configuration: %s", + wtext->strerror(wtext, session, ret)); goto err; } cursor->config_recno = v.len == 1 && v.str[0] == 'r'; @@ -822,7 +826,8 @@ kvs_session_open_cursor(WT_DATA_SOURCE *wtds, WT_SESSION *session, if ((ret = wtext->config_get( wtext, session, config, "value_format", &v)) != 0) { ESET(wtext, session, ret, - "value_format configuration: %s", wtext->strerror(ret)); + "value_format configuration: %s", + wtext->strerror(wtext, session, ret)); goto err; } cursor->config_bitfield = diff --git a/src/btree/bt_compact.c b/src/btree/bt_compact.c index 405410c6a1c..d8b3a638de3 100644 --- a/src/btree/bt_compact.c +++ b/src/btree/bt_compact.c @@ -105,8 +105,8 @@ __wt_compact(WT_SESSION_IMPL *session, const char *cfg[]) * writing the page modify information. * * There are three ways we call reconciliation: checkpoints, threads - * writing leaf pages (usually in preparation for a checkpoint), and - * eviction. + * writing leaf pages (usually in preparation for a checkpoint or if + * closing a file), and eviction. * * We're holding the schema lock which serializes with checkpoints. */ @@ -149,7 +149,7 @@ __wt_compact(WT_SESSION_IMPL *session, const char *cfg[]) * read, set its generation to a low value so it is evicted * quickly. */ - WT_ERR(__wt_tree_walk(session, &ref, + WT_ERR(__wt_tree_walk(session, &ref, NULL, WT_READ_COMPACT | WT_READ_NO_GEN | WT_READ_WONT_NEED)); if (ref == NULL) break; @@ -171,10 +171,12 @@ err: if (ref != NULL) if (block_manager_begin) WT_TRET(bm->compact_end(bm, session)); - __wt_spin_unlock(session, &btree->flush_lock); - + /* + * Unlock will be a release barrier, use it to update the compaction + * status for reconciliation. + */ conn->compact_in_memory_pass = 0; - WT_FULL_BARRIER(); + __wt_spin_unlock(session, &btree->flush_lock); return (ret); } diff --git a/src/btree/bt_curnext.c b/src/btree/bt_curnext.c index 6140dca1fad..d80a5f4740d 100644 --- a/src/btree/bt_curnext.c +++ b/src/btree/bt_curnext.c @@ -487,7 +487,7 @@ __wt_btcur_next(WT_CURSOR_BTREE *cbt, int truncating) __wt_page_evict_soon(page); cbt->page_deleted_count = 0; - WT_ERR(__wt_tree_walk(session, &cbt->ref, flags)); + WT_ERR(__wt_tree_walk(session, &cbt->ref, NULL, flags)); WT_ERR_TEST(cbt->ref == NULL, WT_NOTFOUND); } diff --git a/src/btree/bt_curprev.c b/src/btree/bt_curprev.c index 880cb777954..f1ca81ee145 100644 --- a/src/btree/bt_curprev.c +++ b/src/btree/bt_curprev.c @@ -574,7 +574,7 @@ __wt_btcur_prev(WT_CURSOR_BTREE *cbt, int truncating) __wt_page_evict_soon(page); cbt->page_deleted_count = 0; - WT_ERR(__wt_tree_walk(session, &cbt->ref, flags)); + WT_ERR(__wt_tree_walk(session, &cbt->ref, NULL, flags)); WT_ERR_TEST(cbt->ref == NULL, WT_NOTFOUND); } diff --git a/src/btree/bt_handle.c b/src/btree/bt_handle.c index 6a2789c909b..299849ad365 100644 --- a/src/btree/bt_handle.c +++ b/src/btree/bt_handle.c @@ -566,7 +566,7 @@ __btree_get_last_recno(WT_SESSION_IMPL *session) btree = S2BT(session); next_walk = NULL; - WT_RET(__wt_tree_walk(session, &next_walk, WT_READ_PREV)); + WT_RET(__wt_tree_walk(session, &next_walk, NULL, WT_READ_PREV)); if (next_walk == NULL) return (WT_NOTFOUND); diff --git a/src/btree/bt_page.c b/src/btree/bt_page.c index d1da615dafe..2f2ce4cf4f7 100644 --- a/src/btree/bt_page.c +++ b/src/btree/bt_page.c @@ -32,17 +32,11 @@ __evict_force_check(WT_SESSION_IMPL *session, WT_PAGE *page, uint32_t flags) return (0); /* Leaf pages only. */ - if (page->type != WT_PAGE_COL_FIX && - page->type != WT_PAGE_COL_VAR && - page->type != WT_PAGE_ROW_LEAF) + if (WT_PAGE_IS_INTERNAL(page)) return (0); - /* - * Eviction may be turned off (although that's rare), or we may be in - * the middle of a checkpoint. - */ - if (LF_ISSET(WT_READ_NO_EVICT) || - F_ISSET(btree, WT_BTREE_NO_EVICTION) || btree->checkpointing) + /* Eviction may be turned off. */ + if (LF_ISSET(WT_READ_NO_EVICT) || F_ISSET(btree, WT_BTREE_NO_EVICTION)) return (0); /* @@ -52,17 +46,11 @@ __evict_force_check(WT_SESSION_IMPL *session, WT_PAGE *page, uint32_t flags) if (page->modify == NULL) return (0); - /* - * If the page was recently split in-memory, don't force it out: we - * hope eviction will find it first. - */ - if (!__wt_txn_visible_all(session, page->modify->first_dirty_txn)) - return (0); - /* Trigger eviction on the next page release. */ __wt_page_evict_soon(page); - return (1); + /* If eviction cannot succeed, don't try. */ + return (__wt_page_can_evict(session, page, 1)); } /* diff --git a/src/btree/bt_stat.c b/src/btree/bt_stat.c index b7108b52395..2e34a925f84 100644 --- a/src/btree/bt_stat.c +++ b/src/btree/bt_stat.c @@ -43,9 +43,21 @@ __wt_btree_stat_init(WT_SESSION_IMPL *session, WT_CURSOR_STAT *cst) if (!F_ISSET(cst, WT_CONN_STAT_ALL)) return (0); + /* + * Clear the statistics we're about to count. + */ + WT_STAT_SET(stats, btree_column_deleted, 0); + WT_STAT_SET(stats, btree_column_fix, 0); + WT_STAT_SET(stats, btree_column_internal, 0); + WT_STAT_SET(stats, btree_column_variable, 0); + WT_STAT_SET(stats, btree_entries, 0); + WT_STAT_SET(stats, btree_overflow, 0); + WT_STAT_SET(stats, btree_row_internal, 0); + WT_STAT_SET(stats, btree_row_leaf, 0); + next_walk = NULL; - while ((ret = - __wt_tree_walk(session, &next_walk, 0)) == 0 && next_walk != NULL) { + while ((ret = __wt_tree_walk(session, &next_walk, NULL, 0)) == 0 && + next_walk != NULL) { WT_WITH_PAGE_INDEX(session, ret = __stat_page(session, next_walk->page, stats)); WT_RET(ret); diff --git a/src/btree/bt_sync.c b/src/btree/bt_sync.c index f038ebf3ecc..a75af03d8c8 100644 --- a/src/btree/bt_sync.c +++ b/src/btree/bt_sync.c @@ -56,13 +56,19 @@ __sync_file(WT_SESSION_IMPL *session, int syncop) flags |= WT_READ_NO_WAIT | WT_READ_SKIP_INTL; for (walk = NULL;;) { - WT_ERR(__wt_tree_walk(session, &walk, flags)); + WT_ERR(__wt_tree_walk(session, &walk, NULL, flags)); if (walk == NULL) break; - /* Write dirty pages if nobody beat us to it. */ + /* + * Write dirty pages if nobody beat us to it. Don't + * try to write the hottest pages: checkpoint will have + * to visit them anyway. + */ page = walk->page; - if (__wt_page_is_modified(page)) { + if (__wt_page_is_modified(page) && + __wt_txn_visible_all( + session, page->modify->update_txn)) { if (txn->isolation == TXN_ISO_READ_COMMITTED) __wt_txn_refresh(session, 1); leaf_bytes += page->memory_footprint; @@ -102,7 +108,7 @@ __sync_file(WT_SESSION_IMPL *session, int syncop) /* Write all dirty in-cache pages. */ flags |= WT_READ_NO_EVICT; for (walk = NULL;;) { - WT_ERR(__wt_tree_walk(session, &walk, flags)); + WT_ERR(__wt_tree_walk(session, &walk, NULL, flags)); if (walk == NULL) break; @@ -137,7 +143,6 @@ __sync_file(WT_SESSION_IMPL *session, int syncop) } } break; - WT_ILLEGAL_VALUE_ERR(session); } if (WT_VERBOSE_ISSET(session, WT_VERB_CHECKPOINT)) { @@ -170,6 +175,12 @@ err: /* On error, clear any left-over tree walk. */ WT_FULL_BARRIER(); /* + * If this tree was being skipped by the eviction server during + * the checkpoint, clear the wait. + */ + btree->evict_walk_period = 0; + + /* * Wake the eviction server, in case application threads have * stalled while the eviction server decided it couldn't make * progress. Without this, application threads will be stalled diff --git a/src/btree/bt_vrfy.c b/src/btree/bt_vrfy.c index 2957eda3a49..45c2029f6ed 100644 --- a/src/btree/bt_vrfy.c +++ b/src/btree/bt_vrfy.c @@ -20,17 +20,21 @@ typedef struct { uint64_t fcnt; /* Progress counter */ +#define WT_VRFY_DUMP(vs) \ + ((vs)->dump_address || \ + (vs)->dump_blocks || (vs)->dump_pages || (vs)->dump_shape) int dump_address; /* Debugging hooks */ - int dump_pages; int dump_blocks; + int dump_pages; + int dump_shape; + + u_int depth, depth_internal[100], depth_leaf[100]; WT_ITEM *tmp1; /* Temporary buffer */ WT_ITEM *tmp2; /* Temporary buffer */ } WT_VSTUFF; static void __verify_checkpoint_reset(WT_VSTUFF *); -static int __verify_config(WT_SESSION_IMPL *, const char *[], WT_VSTUFF *); -static int __verify_config_offsets(WT_SESSION_IMPL *, const char *[], int *); static int __verify_overflow( WT_SESSION_IMPL *, const uint8_t *, size_t, WT_VSTUFF *); static int __verify_overflow_cell( @@ -42,6 +46,96 @@ static int __verify_row_leaf_key_order( static int __verify_tree(WT_SESSION_IMPL *, WT_REF *, WT_VSTUFF *); /* + * __verify_config -- + * Debugging: verification supports dumping pages in various formats. + */ +static int +__verify_config(WT_SESSION_IMPL *session, const char *cfg[], WT_VSTUFF *vs) +{ + WT_CONFIG_ITEM cval; + + WT_RET(__wt_config_gets(session, cfg, "dump_address", &cval)); + vs->dump_address = cval.val != 0; + + WT_RET(__wt_config_gets(session, cfg, "dump_blocks", &cval)); + vs->dump_blocks = cval.val != 0; + + WT_RET(__wt_config_gets(session, cfg, "dump_pages", &cval)); + vs->dump_pages = cval.val != 0; + + WT_RET(__wt_config_gets(session, cfg, "dump_shape", &cval)); + vs->dump_shape = cval.val != 0; + +#if !defined(HAVE_DIAGNOSTIC) + if (vs->dump_blocks || vs->dump_pages) + WT_RET_MSG(session, ENOTSUP, + "the WiredTiger library was not built in diagnostic mode"); +#endif + return (0); +} + +/* + * __verify_config_offsets -- + * Debugging: optionally dump specific blocks from the file. + */ +static int +__verify_config_offsets(WT_SESSION_IMPL *session, const char *cfg[], int *quitp) +{ + WT_CONFIG list; + WT_CONFIG_ITEM cval, k, v; + WT_DECL_RET; + u_long offset; + + *quitp = 0; + + WT_RET(__wt_config_gets(session, cfg, "dump_offsets", &cval)); + WT_RET(__wt_config_subinit(session, &list, &cval)); + while ((ret = __wt_config_next(&list, &k, &v)) == 0) { + /* + * Quit after dumping the requested blocks. (That's hopefully + * what the user wanted, all of this stuff is just hooked into + * verify because that's where we "dump blocks" for debugging.) + */ + *quitp = 1; + if (v.len != 0 || sscanf(k.str, "%lu", &offset) != 1) + WT_RET_MSG(session, EINVAL, + "unexpected dump offset format"); +#if !defined(HAVE_DIAGNOSTIC) + WT_RET_MSG(session, ENOTSUP, + "the WiredTiger library was not built in diagnostic mode"); +#else + WT_TRET( + __wt_debug_offset_blind(session, (wt_off_t)offset, NULL)); +#endif + } + return (ret == WT_NOTFOUND ? 0 : ret); +} + +/* + * __verify_tree_shape -- + * Dump the tree shape. + */ +static int +__verify_tree_shape(WT_SESSION_IMPL *session, WT_VSTUFF *vs) +{ + size_t i; + + WT_RET(__wt_msg(session, "Internal page tree-depth:")); + for (i = 0; i < WT_ELEMENTS(vs->depth_internal); ++i) + if (vs->depth_internal[i] != 0) + WT_RET(__wt_msg(session, + "\t%03zu: %u", i, vs->depth_internal[i])); + + WT_RET(__wt_msg(session, "Leaf page tree-depth:")); + for (i = 0; i < WT_ELEMENTS(vs->depth_leaf); ++i) + if (vs->depth_leaf[i] != 0) + WT_RET(__wt_msg(session, + "\t%03zu: %u", i, vs->depth_leaf[i])); + + return (0); +} + +/* * __wt_verify -- * Verify a file. */ @@ -97,11 +191,10 @@ __wt_verify(WT_SESSION_IMPL *session, const char *cfg[]) /* House-keeping between checkpoints. */ __verify_checkpoint_reset(vs); -#ifdef HAVE_DIAGNOSTIC - if (vs->dump_address || vs->dump_blocks || vs->dump_pages) + if (WT_VRFY_DUMP(vs)) WT_ERR(__wt_msg(session, "%s: checkpoint %s", btree->dhandle->name, ckpt->name)); -#endif + /* Load the checkpoint. */ WT_ERR(bm->checkpoint_load(bm, session, ckpt->raw.data, ckpt->raw.size, @@ -114,15 +207,13 @@ __wt_verify(WT_SESSION_IMPL *session, const char *cfg[]) if (root_addr_size != 0 && (ret = __wt_btree_tree_open( session, root_addr, root_addr_size)) == 0) { -#ifdef HAVE_DIAGNOSTIC - if (vs->dump_address || - vs->dump_blocks || vs->dump_pages) + if (WT_VRFY_DUMP(vs)) WT_ERR(__wt_msg(session, "Root: %s %s", __wt_addr_string(session, root_addr, root_addr_size, vs->tmp1), __wt_page_type_string( btree->root.page->type))); -#endif + WT_WITH_PAGE_INDEX(session, ret = __verify_tree(session, &btree->root, vs)); @@ -132,6 +223,10 @@ __wt_verify(WT_SESSION_IMPL *session, const char *cfg[]) /* Unload the checkpoint. */ WT_TRET(bm->checkpoint_unload(bm, session)); WT_ERR(ret); + + /* Display the tree shape. */ + if (vs->dump_shape) + WT_ERR(__verify_tree_shape(session, vs)); } done: @@ -156,69 +251,6 @@ err: /* Inform the underlying block manager we're done. */ } /* - * __verify_config -- - * Debugging: verification supports dumping pages in various formats. - */ -static int -__verify_config(WT_SESSION_IMPL *session, const char *cfg[], WT_VSTUFF *vs) -{ - WT_CONFIG_ITEM cval; - - WT_RET(__wt_config_gets(session, cfg, "dump_address", &cval)); - vs->dump_address = cval.val != 0; - - WT_RET(__wt_config_gets(session, cfg, "dump_blocks", &cval)); - vs->dump_blocks = cval.val != 0; - - WT_RET(__wt_config_gets(session, cfg, "dump_pages", &cval)); - vs->dump_pages = cval.val != 0; - -#if !defined(HAVE_DIAGNOSTIC) - if (vs->dump_address || vs->dump_blocks || vs->dump_pages) - WT_RET_MSG(session, ENOTSUP, - "the WiredTiger library was not built in diagnostic mode"); -#endif - return (0); -} - -/* - * __verify_config_offsets -- - * Debugging: optionally dump specific blocks from the file. - */ -static int -__verify_config_offsets(WT_SESSION_IMPL *session, const char *cfg[], int *quitp) -{ - WT_CONFIG list; - WT_CONFIG_ITEM cval, k, v; - WT_DECL_RET; - u_long offset; - - *quitp = 0; - - WT_RET(__wt_config_gets(session, cfg, "dump_offsets", &cval)); - WT_RET(__wt_config_subinit(session, &list, &cval)); - while ((ret = __wt_config_next(&list, &k, &v)) == 0) { - /* - * Quit after dumping the requested blocks. (That's hopefully - * what the user wanted, all of this stuff is just hooked into - * verify because that's where we "dump blocks" for debugging.) - */ - *quitp = 1; - if (v.len != 0 || sscanf(k.str, "%lu", &offset) != 1) - WT_RET_MSG(session, EINVAL, - "unexpected dump offset format"); -#if !defined(HAVE_DIAGNOSTIC) - WT_RET_MSG(session, ENOTSUP, - "the WiredTiger library was not built in diagnostic mode"); -#else - WT_TRET( - __wt_debug_offset_blind(session, (wt_off_t)offset, NULL)); -#endif - } - return (ret == WT_NOTFOUND ? 0 : ret); -} - -/* * __verify_checkpoint_reset -- * Reset anything needing to be reset for each new checkpoint verification. */ @@ -233,6 +265,9 @@ __verify_checkpoint_reset(WT_VSTUFF *vs) /* Record total is per checkpoint, reset the record count. */ vs->record_total = 0; + + /* Tree depth. */ + vs->depth = 1; } /* @@ -265,12 +300,20 @@ __verify_tree(WT_SESSION_IMPL *session, WT_REF *ref, WT_VSTUFF *vs) WT_RET(__wt_verbose(session, WT_VERB_VERIFY, "%s %s", __wt_page_addr_string(session, ref, vs->tmp1), __wt_page_type_string(page->type))); -#ifdef HAVE_DIAGNOSTIC + + /* Optionally dump the address. */ if (vs->dump_address) WT_RET(__wt_msg(session, "%s %s", __wt_page_addr_string(session, ref, vs->tmp1), __wt_page_type_string(page->type))); -#endif + + /* Track the shape of the tree. */ + if (WT_PAGE_IS_INTERNAL(page)) + ++vs->depth_internal[ + WT_MIN(vs->depth, WT_ELEMENTS(vs->depth_internal) - 1)]; + else + ++vs->depth_leaf[ + WT_MIN(vs->depth, WT_ELEMENTS(vs->depth_internal) - 1)]; /* * The page's physical structure was verified when it was read into @@ -447,9 +490,11 @@ celltype_err: WT_RET_MSG(session, WT_ERROR, } /* Verify the subtree. */ + ++vs->depth; WT_RET(__wt_page_in(session, child_ref, 0)); ret = __verify_tree(session, child_ref, vs); WT_TRET(__wt_page_release(session, child_ref, 0)); + --vs->depth; WT_RET(ret); __wt_cell_unpack(child_ref->addr, unpack); @@ -475,9 +520,11 @@ celltype_err: WT_RET_MSG(session, WT_ERROR, session, page, child_ref, entry, vs)); /* Verify the subtree. */ + ++vs->depth; WT_RET(__wt_page_in(session, child_ref, 0)); ret = __verify_tree(session, child_ref, vs); WT_TRET(__wt_page_release(session, child_ref, 0)); + --vs->depth; WT_RET(ret); __wt_cell_unpack(child_ref->addr, unpack); diff --git a/src/btree/bt_walk.c b/src/btree/bt_walk.c index a2b2a6bb7c8..10dd5b12936 100644 --- a/src/btree/bt_walk.c +++ b/src/btree/bt_walk.c @@ -13,14 +13,14 @@ * Move to the next/previous page in the tree. */ int -__wt_tree_walk(WT_SESSION_IMPL *session, WT_REF **refp, uint32_t flags) +__wt_tree_walk(WT_SESSION_IMPL *session, + WT_REF **refp, uint64_t *walkcntp, uint32_t flags) { WT_BTREE *btree; WT_DECL_RET; WT_PAGE *page; WT_PAGE_INDEX *pindex; WT_REF *couple, *ref; - WT_TXN_STATE *txn_state; int descending, prev, skip; uint32_t slot; @@ -44,16 +44,6 @@ __wt_tree_walk(WT_SESSION_IMPL *session, WT_REF **refp, uint32_t flags) prev = LF_ISSET(WT_READ_PREV) ? 1 : 0; /* - * Pin a transaction ID, required to safely look at page index - * structures, if our caller has not already done so. - */ - txn_state = WT_SESSION_TXN_STATE(session); - if (txn_state->snap_min == WT_TXN_NONE) - txn_state->snap_min = S2C(session)->txn_global.last_running; - else - txn_state = NULL; - - /* * There are multiple reasons and approaches to walking the in-memory * tree: * @@ -95,11 +85,8 @@ __wt_tree_walk(WT_SESSION_IMPL *session, WT_REF **refp, uint32_t flags) /* If no page is active, begin a walk from the start of the tree. */ if (ref == NULL) { ref = &btree->root; - if (ref->page == NULL) { - if (txn_state != NULL) - txn_state->snap_min = WT_TXN_NONE; + if (ref->page == NULL) goto done; - } goto descend; } @@ -129,11 +116,8 @@ restart: /* ref = couple; if (ref == &btree->root) { ref = &btree->root; - if (ref->page == NULL) { - if (txn_state != NULL) - txn_state->snap_min = WT_TXN_NONE; + if (ref->page == NULL) goto done; - } goto descend; } __wt_page_refp(session, ref, &pindex, &slot); @@ -195,6 +179,9 @@ restart: /* else ++slot; + if (walkcntp != NULL) + ++*walkcntp; + for (descending = 0;;) { ref = pindex->index[slot]; @@ -283,9 +270,6 @@ descend: couple = ref; } done: -err: if (txn_state != NULL) - txn_state->snap_min = WT_TXN_NONE; - - WT_LEAVE_PAGE_INDEX(session); +err: WT_LEAVE_PAGE_INDEX(session); return (ret); } diff --git a/src/config/config_def.c b/src/config/config_def.c index 052fbf3d0b4..43d87c518e4 100644 --- a/src/config/config_def.c +++ b/src/config/config_def.c @@ -299,6 +299,7 @@ static const WT_CONFIG_CHECK confchk_session_verify[] = { { "dump_blocks", "boolean", NULL, NULL }, { "dump_offsets", "list", NULL, NULL }, { "dump_pages", "boolean", NULL, NULL }, + { "dump_shape", "boolean", NULL, NULL }, { NULL, NULL, NULL, NULL } }; @@ -318,6 +319,7 @@ static const WT_CONFIG_CHECK confchk_log_subconfigs[] = { { "file_max", "int", "min=100KB,max=2GB", NULL }, { "path", "string", NULL, NULL }, { "prealloc", "boolean", NULL, NULL }, + { "recover", "string", "choices=[\"error\",\"on\"]", NULL }, { NULL, NULL, NULL, NULL } }; @@ -666,6 +668,10 @@ static const WT_CONFIG_ENTRY config_entries[] = { "force=0", confchk_session_salvage }, + { "session.strerror", + "", + NULL + }, { "session.truncate", "", NULL @@ -675,7 +681,8 @@ static const WT_CONFIG_ENTRY config_entries[] = { NULL }, { "session.verify", - "dump_address=0,dump_blocks=0,dump_offsets=,dump_pages=0", + "dump_address=0,dump_blocks=0,dump_offsets=,dump_pages=0," + "dump_shape=0", confchk_session_verify }, { "table.meta", @@ -690,7 +697,7 @@ static const WT_CONFIG_ENTRY config_entries[] = { "eviction=(threads_max=1,threads_min=1),eviction_dirty_target=80," "eviction_target=80,eviction_trigger=95,exclusive=0,extensions=," "file_extend=,hazard_max=1000,log=(archive=,compressor=,enabled=0" - ",file_max=100MB,path=,prealloc=),lsm_manager=(merge=," + ",file_max=100MB,path=,prealloc=,recover=on),lsm_manager=(merge=," "worker_thread_max=4),lsm_merge=,mmap=,multiprocess=0," "session_max=100,session_scratch_max=2MB,shared_cache=(chunk=10MB" ",name=,reserve=0,size=500MB),statistics=none," @@ -708,7 +715,7 @@ static const WT_CONFIG_ENTRY config_entries[] = { "eviction=(threads_max=1,threads_min=1),eviction_dirty_target=80," "eviction_target=80,eviction_trigger=95,exclusive=0,extensions=," "file_extend=,hazard_max=1000,log=(archive=,compressor=,enabled=0" - ",file_max=100MB,path=,prealloc=),lsm_manager=(merge=," + ",file_max=100MB,path=,prealloc=,recover=on),lsm_manager=(merge=," "worker_thread_max=4),lsm_merge=,mmap=,multiprocess=0," "session_max=100,session_scratch_max=2MB,shared_cache=(chunk=10MB" ",name=,reserve=0,size=500MB),statistics=none," @@ -725,8 +732,8 @@ static const WT_CONFIG_ENTRY config_entries[] = { "direct_io=,error_prefix=,eviction=(threads_max=1,threads_min=1)," "eviction_dirty_target=80,eviction_target=80,eviction_trigger=95," "extensions=,file_extend=,hazard_max=1000,log=(archive=," - "compressor=,enabled=0,file_max=100MB,path=,prealloc=)," - "lsm_manager=(merge=,worker_thread_max=4),lsm_merge=,mmap=," + "compressor=,enabled=0,file_max=100MB,path=,prealloc=,recover=on)" + ",lsm_manager=(merge=,worker_thread_max=4),lsm_merge=,mmap=," "multiprocess=0,session_max=100,session_scratch_max=2MB," "shared_cache=(chunk=10MB,name=,reserve=0,size=500MB)," "statistics=none,statistics_log=(on_close=0," @@ -742,8 +749,8 @@ static const WT_CONFIG_ENTRY config_entries[] = { "direct_io=,error_prefix=,eviction=(threads_max=1,threads_min=1)," "eviction_dirty_target=80,eviction_target=80,eviction_trigger=95," "extensions=,file_extend=,hazard_max=1000,log=(archive=," - "compressor=,enabled=0,file_max=100MB,path=,prealloc=)," - "lsm_manager=(merge=,worker_thread_max=4),lsm_merge=,mmap=," + "compressor=,enabled=0,file_max=100MB,path=,prealloc=,recover=on)" + ",lsm_manager=(merge=,worker_thread_max=4),lsm_merge=,mmap=," "multiprocess=0,session_max=100,session_scratch_max=2MB," "shared_cache=(chunk=10MB,name=,reserve=0,size=500MB)," "statistics=none,statistics_log=(on_close=0," diff --git a/src/conn/api_strerror.c b/src/conn/api_strerror.c index 396ae7a3e0f..e41e402a1fd 100644 --- a/src/conn/api_strerror.c +++ b/src/conn/api_strerror.c @@ -5,20 +5,24 @@ /* * Historically, there was only the wiredtiger_strerror call because the POSIX * port didn't need anything more complex; Windows requires memory allocation - * of error strings, so we added the wiredtiger_strerror_r call. Because we + * of error strings, so we added the WT_SESSION.strerror method. Because we * want wiredtiger_strerror to continue to be as thread-safe as possible, errors - * are split into three categories: WiredTiger constant strings, system constant - * strings and Everything Else, and we check constant strings before Everything - * Else. + * are split into two categories: WiredTiger's or the system's constant strings + * and Everything Else, and we check constant strings before Everything Else. */ /* - * __wiredtiger_error -- - * Return a constant string for the WiredTiger errors. + * __wt_wiredtiger_error -- + * Return a constant string for WiredTiger POSIX-standard and errors. */ -static const char * -__wiredtiger_error(int error) +const char * +__wt_wiredtiger_error(int error) { + const char *p; + + /* + * Check for WiredTiger specific errors. + */ switch (error) { case WT_ROLLBACK: return ("WT_ROLLBACK: conflict between concurrent operations"); @@ -32,7 +36,19 @@ __wiredtiger_error(int error) return ("WT_PANIC: WiredTiger library panic"); case WT_RESTART: return ("WT_RESTART: restart the operation (internal)"); + case WT_RUN_RECOVERY: + return ("WT_RUN_RECOVERY: recovery must be run to continue"); } + + /* + * POSIX errors are non-negative integers; check for 0 explicitly + * in-case the underlying strerror doesn't handle 0, some don't. + */ + if (error == 0) + return ("Successful return: 0"); + if (error > 0 && (p = strerror(error)) != NULL) + return (p); + return (NULL); } @@ -44,37 +60,6 @@ const char * wiredtiger_strerror(int error) { static char buf[128]; - const char *p; - - /* Check for a constant string. */ - if ((p = __wiredtiger_error(error)) != NULL || - (p = __wt_strerror(error)) != NULL) - return (p); - - /* Else, fill in the non-thread-safe static buffer. */ - if (wiredtiger_strerror_r(error, buf, sizeof(buf)) != 0) - (void)snprintf(buf, sizeof(buf), "error return: %d", error); - - return (buf); -} - -/* - * wiredtiger_strerror_r -- - * Return a string for any error value, thread-safe version. - */ -int -wiredtiger_strerror_r(int error, char *buf, size_t buflen) -{ - const char *p; - - /* Require at least 2 bytes, printable character and trailing nul. */ - if (buflen < 2) - return (ENOMEM); - - /* Check for a constant string. */ - if ((p = __wiredtiger_error(error)) != NULL || - (p = __wt_strerror(error)) != NULL) - return (snprintf(buf, buflen, "%s", p) > 0 ? 0 : ENOMEM); - return (__wt_strerror_r(error, buf, buflen)); + return (__wt_strerror(NULL, error, buf, sizeof(buf))); } diff --git a/src/conn/conn_api.c b/src/conn/conn_api.c index 5be55a77f24..0562f9cfc34 100644 --- a/src/conn/conn_api.c +++ b/src/conn/conn_api.c @@ -117,7 +117,7 @@ __conn_get_extension_api(WT_CONNECTION *wt_conn) conn->extension_api.conn = wt_conn; conn->extension_api.err_printf = __wt_ext_err_printf; conn->extension_api.msg_printf = __wt_ext_msg_printf; - conn->extension_api.strerror = wiredtiger_strerror; + conn->extension_api.strerror = __wt_ext_strerror; conn->extension_api.scr_alloc = __wt_ext_scr_alloc; conn->extension_api.scr_free = __wt_ext_scr_free; conn->extension_api.collator_config = ext_collator_config; diff --git a/src/conn/conn_cache.c b/src/conn/conn_cache.c index b278d7a6b8a..c513d46137c 100644 --- a/src/conn/conn_cache.c +++ b/src/conn/conn_cache.c @@ -29,8 +29,6 @@ __wt_cache_config(WT_SESSION_IMPL *session, const char *cfg[]) if (!F_ISSET(conn, WT_CONN_CACHE_POOL)) { WT_RET(__wt_config_gets(session, cfg, "cache_size", &cval)); conn->cache_size = (uint64_t)cval.val; - WT_RET(__wt_config_gets(session, cfg, "cache_overhead", &cval)); - conn->cache_overhead = (int)cval.val; } else { WT_RET(__wt_config_gets( session, cfg, "shared_cache.reserve", &cval)); @@ -40,6 +38,9 @@ __wt_cache_config(WT_SESSION_IMPL *session, const char *cfg[]) cache->cp_reserved = (uint64_t)cval.val; } + WT_RET(__wt_config_gets(session, cfg, "cache_overhead", &cval)); + cache->overhead_pct = (u_int)cval.val; + WT_RET(__wt_config_gets(session, cfg, "eviction_target", &cval)); cache->eviction_target = (u_int)cval.val; @@ -145,9 +146,9 @@ __wt_cache_stats_update(WT_SESSION_IMPL *session) WT_STAT_SET(stats, cache_bytes_max, conn->cache_size); WT_STAT_SET(stats, cache_bytes_inuse, __wt_cache_bytes_inuse(cache)); - WT_STAT_SET(stats, cache_overhead, conn->cache_overhead); + WT_STAT_SET(stats, cache_overhead, cache->overhead_pct); WT_STAT_SET(stats, cache_pages_inuse, __wt_cache_pages_inuse(cache)); - WT_STAT_SET(stats, cache_bytes_dirty, cache->bytes_dirty); + WT_STAT_SET(stats, cache_bytes_dirty, __wt_cache_dirty_inuse(cache)); WT_STAT_SET(stats, cache_eviction_maximum_page_size, cache->evict_max_page_size); WT_STAT_SET(stats, cache_pages_dirty, cache->pages_dirty); diff --git a/src/conn/conn_log.c b/src/conn/conn_log.c index 11abc7c1e2b..36d4d539d92 100644 --- a/src/conn/conn_log.c +++ b/src/conn/conn_log.c @@ -98,6 +98,9 @@ __logmgr_config(WT_SESSION_IMPL *session, const char **cfg, int *runp) FLD_SET(conn->log_flags, WT_CONN_LOG_PREALLOC); conn->log_prealloc = 1; } + WT_RET(__wt_config_gets_def(session, cfg, "log.recover", 0, &cval)); + if (cval.len != 0 && WT_STRING_MATCH("error", cval.str, cval.len)) + FLD_SET(conn->log_flags, WT_CONN_LOG_RECOVER_ERR); WT_RET(__logmgr_sync_cfg(session, cfg)); return (0); diff --git a/src/docs/Doxyfile b/src/docs/Doxyfile index 8e051aaff3b..d203826dcae 100644 --- a/src/docs/Doxyfile +++ b/src/docs/Doxyfile @@ -779,8 +779,7 @@ EXCLUDE_SYMBOLS = __F \ EXAMPLE_PATH = ../../examples/c \ ../../examples/java/com/wiredtiger/examples/ \ - ../../ext/compressors/nop \ - ./examples + ../../ext/compressors/nop # If the value of the EXAMPLE_PATH tag contains directories, you can use the # EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp diff --git a/src/docs/command-line.dox b/src/docs/command-line.dox index ea83b3f58be..84279250066 100644 --- a/src/docs/command-line.dox +++ b/src/docs/command-line.dox @@ -3,7 +3,7 @@ WiredTiger includes a command line utility, \c wt. @section util_global_synopsis Synopsis -<code>wt [-Vv] [-C config] [-h directory] command [command-specific arguments]</code> +<code>wt [-rVv] [-C config] [-h directory] command [command-specific arguments]</code> @section util_global_description Description The \c wt tool is a command-line utility that provides access to @@ -16,6 +16,8 @@ There are four global options: Specify configuration strings for the ::wiredtiger_open function. @par <code>-h directory</code> Specify a database home directory. +@par <code>-r</code> +Run recovery if the underlying database is configured to do so. @par <code>-V</code> Display WiredTiger version and exit. @par <code>-v</code> @@ -24,7 +26,11 @@ Set verbose output. Unless otherwise described by a \c wt command, the \c wt tool exits zero on success and non-zero on error. -The \c wt tool supports several commands. +The \c wt tool supports several commands. If configured in the underlying +database, some commands will run recovery when opening the database. If +the user wants to force recovery on any command, use the \c -r option. +In general, commands that modify the database or tables will run recovery +by default and commands that only read data will not run recovery. <hr> @section util_backup wt backup @@ -36,7 +42,7 @@ opened as a WiredTiger database. See @ref backup for more information, and @ref file_permissions for specifics on the copied file permissions. @subsection util_backup_synopsis Synopsis -<code>wt [-Vv] [-C config] [-h directory] backup [-t uri] directory</code> +<code>wt [-rVv] [-C config] [-h directory] backup [-t uri] directory</code> @subsection util_backup_options Options The following are command-specific options for the \c backup command: @@ -54,7 +60,7 @@ The \c compact command attempts to rewrite the specified table or file to consume less disk space. @subsection util_compact_synopsis Synopsis -<code>wt [-Vv] [-C config] [-h directory] compact uri</code> +<code>wt [-rVv] [-C config] [-h directory] compact uri</code> @subsection util_compact_options Options The \c compact command has no command-specific options. @@ -68,7 +74,7 @@ configuration. It is equivalent to a call to WT_SESSION::create with the specified string arguments. @subsection util_create_synopsis Synopsis -<code>wt [-Vv] [-C config] [-h directory] create [-c config] uri</code> +<code>wt [-rVv] [-C config] [-h directory] create [-c config] uri</code> @subsection util_create_options Options The following are command-specific options for the \c create command: @@ -84,7 +90,7 @@ The \c drop command drops the specified \c uri. It is equivalent to a call to WT_SESSION::drop with the "force" configuration argument. @subsection util_drop_synopsis Synopsis -<code>wt [-Vv] [-C config] [-h directory] drop uri</code> +<code>wt [-rVv] [-C config] [-h directory] drop uri</code> @subsection util_drop_options Options The \c drop command has no command-specific options. @@ -99,7 +105,7 @@ which can be re-loaded into a new table using the \c load command. See @subpage dump_formats for details of the dump file formats. @subsection util_dump_synopsis Synopsis -<code>wt [-Vv] [-C config] [-h directory] dump [-jrx] [-c checkpoint] [-f output] uri</code> +<code>wt [-rVv] [-C config] [-h directory] dump [-jrx] [-c checkpoint] [-f output] uri</code> @subsection util_dump_options Options The following are command-specific options for the \c dump command: @@ -133,7 +139,7 @@ the database. If a URI is specified as an argument, only information about that data source is printed. @subsection util_list_synopsis Synopsis -<code>wt [-Vv] [-C config] [-h directory] list [-cv] [uri]</code> +<code>wt [-rVv] [-C config] [-h directory] list [-cv] [uri]</code> @subsection util_list_options Options The following are command-specific options for the \c list command: @@ -160,7 +166,7 @@ table will be overwritten by the new data (use the \c -n option to make an attempt to overwrite existing data return an error). @subsection util_load_synopsis Synopsis -<code>wt [-Vv] [-C config] [-h directory] load [-ajn] [-f input] [-r name] [uri configuration ...]</code> +<code>wt [-rVv] [-C config] [-h directory] load [-ajn] [-f input] [-r name] [uri configuration ...]</code> @subsection util_load_options Options The following are command-specific options for the \c load command: @@ -212,7 +218,7 @@ row-store table or file already exists, data in the table or file will be overwritten by the new data. @subsection util_loadtext_synopsis Synopsis -<code>wt [-Vv] [-C config] [-h directory] loadtext [-f input]</code> +<code>wt [-rVv] [-C config] [-h directory] loadtext [-f input]</code> @subsection util_loadtext_options Options The following are command-specific options for the \c loadtext command: @@ -228,7 +234,7 @@ Display the database log. The \c printlog command outputs the database log. @subsection util_printlog_synopsis Synopsis -<code>wt [-Vv] [-C config] [-h directory] printlog [-p] [-f output]</code> +<code>wt [-rVv] [-C config] [-h directory] printlog [-p] [-f output]</code> @subsection util_printlog_options Options The following are command-specific options for the \c printlog command: @@ -251,7 +257,7 @@ with string or record number keys and string values. The \c read command exits non-zero if a specified record is not found. @subsection util_read_synopsis Synopsis -<code>wt [-Vv] [-C config] [-h directory] read uri key ...</code> +<code>wt [-rVv] [-C config] [-h directory] read uri key ...</code> @subsection util_read_options Options The \c read command has no command-specific options. @@ -263,7 +269,7 @@ Rename a table or file. The \c rename command renames the specified table or file. @subsection util_rename_synopsis Synopsis -<code>wt [-Vv] [-C config] [-h directory] rename uri name</code> +<code>wt [-rVv] [-C config] [-h directory] rename uri name</code> @subsection util_rename_options Options The \c rename command has no command-specific options. @@ -277,7 +283,7 @@ data that cannot be recovered. Underlying files are re-written in place, overwriting the original file contents. @subsection util_salvage_synopsis Synopsis -<code>wt [-Vv] [-C config] [-h directory] salvage [-F force] uri</code> +<code>wt [-rVv] [-C config] [-h directory] salvage [-F force] uri</code> @subsection util_salvage_options Options The following are command-specific options for the \c salvage command: @@ -295,7 +301,7 @@ The \c stat command outputs run-time statistics for the WiredTiger engine, or, if specified, for the URI on the command-line. @subsection util_stat_synopsis Synopsis -<code>wt [-Vv] [-C config] [-h directory] stat [-f] [uri]</code> +<code>wt [-rVv] [-C config] [-h directory] stat [-f] [uri]</code> @subsection util_stat_options Options The following are command-specific options for the \c stat command: @@ -313,7 +319,7 @@ success if the data source is up-to-date, and failure if the data source cannot be upgraded. @subsection util_upgrade_synopsis Synopsis -<code>wt [-Vv] [-C config] [-h directory] upgrade uri</code> +<code>wt [-rVv] [-C config] [-h directory] upgrade uri</code> @subsection util_upgrade_options Options The \c upgrade command has no command-specific options. @@ -327,7 +333,7 @@ success if the data source is correct, and failure if the data source is corrupted. @subsection util_verify_synopsis Synopsis -<code>wt [-Vv] [-C config] [-h directory] verify uri</code> +<code>wt [-rVv] [-C config] [-h directory] verify uri</code> @subsection util_verify_options Options The \c verify command has no command-specific options. @@ -349,9 +355,9 @@ Attempting to overwrite an already existing record will fail. @subsection util_write_synopsis Synopsis <code> -wt [-Vv] [-C config] [-h directory] write -a uri value ... +wt [-rVv] [-C config] [-h directory] write -a uri value ... <br> -wt [-Vv] [-C config] [-h directory] write [-o] uri key value ... +wt [-rVv] [-C config] [-h directory] write [-o] uri key value ... </code> @subsection util_write_options Options diff --git a/src/docs/error-handling.dox b/src/docs/error-handling.dox index cf268f80500..d1291e38ff0 100644 --- a/src/docs/error-handling.dox +++ b/src/docs/error-handling.dox @@ -49,6 +49,9 @@ This error indicates an operation did not find a value to return. This includes @par <code>WT_PANIC</code> This error indicates an underlying problem that requires the application exit and restart. The application can exit immediately when \c WT_PANIC is returned from a WiredTiger interface, no further WiredTiger calls are required. +@par <code>WT_RUN_RECOVERY</code> +This error is generated when wiredtiger_open is configured to return an error if recovery is required to use the database. + @if IGNORE_BUILT_BY_API_ERR_END @endif diff --git a/src/docs/examples/wtperf-sample.html b/src/docs/examples/wtperf-sample.html deleted file mode 100644 index b58e3847159..00000000000 --- a/src/docs/examples/wtperf-sample.html +++ /dev/null @@ -1,37 +0,0 @@ - -<!DOCTYPE html> -<html lang="en"> -<head> -<link href="http://source.wiredtiger.com/graphs/nvd3/src/nv.d3.css" rel="stylesheet"> -<script src="http://source.wiredtiger.com/graphs/d3/d3.min.js"></script> -<script src="http://source.wiredtiger.com/graphs/nvd3/nv.d3.min.js"></script> - -</head> -<body> - -<div id="wtperf"><svg style="height:540px;"></svg></div> - - -<script> - nv.addGraph(function() { - var chart = nv.models.multiChart(); - chart.margin({top: 30, right: 60, bottom: 20, left: 60}) - chart.yAxis1 - .tickFormat(d3.format(',.02f')); - chart.yAxis2 - .tickFormat(d3.format(',.02f')); - chart.xAxis - .tickFormat(d3.format(',r')); - chart.showLegend(true); - d3.select('#wtperf svg') - .datum(data_wtperf) - .transition().duration(500) - .attr('height', 540) - .call(chart); - - nv.utils.windowResize(chart.update); - return chart; -});data_wtperf=[{"type": "line", "values": [{"y": 0.001, "x": 0}, {"y": 0.002, "x": 10}, {"y": 0.002, "x": 20}, {"y": 0.002, "x": 30}, {"y": 0.002, "x": 40}, {"y": 0.002, "x": 50}, {"y": 0.002, "x": 60}, {"y": 0.002, "x": 70}, {"y": 0.002, "x": 80}, {"y": 0.002, "x": 90}, {"y": 0.002, "x": 100}, {"y": 0.002, "x": 110}, {"y": 0.002, "x": 120}, {"y": 0.0030000000000000001, "x": 130}, {"y": 0.002, "x": 140}, {"y": 0.0030000000000000001, "x": 150}, {"y": 0.0030000000000000001, "x": 160}, {"y": 0.002, "x": 170}, {"y": 0.0030000000000000001, "x": 180}, {"y": 0.002, "x": 190}, {"y": 0.002, "x": 200}, {"y": 0.002, "x": 210}, {"y": 0.002, "x": 220}, {"y": 0.002, "x": 230}, {"y": 0.002, "x": 240}, {"y": 0.002, "x": 250}, {"y": 0.002, "x": 260}, {"y": 0.002, "x": 270}, {"y": 0.002, "x": 280}, {"y": 0.002, "x": 290}, {"y": 0.002, "x": 300}, {"y": 0.002, "x": 310}, {"y": 0.002, "x": 320}, {"y": 0.002, "x": 330}, {"y": 0.002, "x": 340}, {"y": 0.002, "x": 350}, {"y": 0.002, "x": 360}, {"y": 0.002, "x": 370}, {"y": 0.002, "x": 380}, {"y": 0.002, "x": 390}, {"y": 0.002, "x": 400}, {"y": 0.002, "x": 410}, {"y": 0.002, "x": 420}, {"y": 0.002, "x": 430}, {"y": 0.002, "x": 440}, {"y": 0.0040000000000000001, "x": 450}, {"y": 0.002, "x": 460}, {"y": 0.002, "x": 470}, {"y": 0.002, "x": 480}, {"y": 0.002, "x": 490}, {"y": 0.002, "x": 500}, {"y": 0.002, "x": 510}, {"y": 0.002, "x": 520}, {"y": 0.002, "x": 530}, {"y": 0.002, "x": 540}, {"y": 0.002, "x": 550}, {"y": 0.002, "x": 560}, {"y": 0.002, "x": 570}, {"y": 0.002, "x": 580}, {"y": 0.002, "x": 590}, {"y": 0.002, "x": 600}, {"y": 0.002, "x": 610}, {"y": 0.002, "x": 620}, {"y": 0.002, "x": 630}, {"y": 0.002, "x": 640}, {"y": 0.002, "x": 650}, {"y": 0.002, "x": 660}, {"y": 0.002, "x": 670}, {"y": 0.002, "x": 680}, {"y": 0.002, "x": 690}, {"y": 0.002, "x": 700}, {"y": 0.002, "x": 710}, {"y": 0.002, "x": 720}, {"y": 0.002, "x": 730}, {"y": 0.002, "x": 741}, {"y": 0.002, "x": 751}, {"y": 0.002, "x": 761}, {"y": 0.002, "x": 771}, {"y": 0.002, "x": 781}, {"y": 0.002, "x": 791}, {"y": 0.002, "x": 801}, {"y": 0.002, "x": 811}, {"y": 0.002, "x": 821}, {"y": 0.002, "x": 831}, {"y": 0.002, "x": 841}, {"y": 0.002, "x": 851}, {"y": 0.002, "x": 861}, {"y": 0.002, "x": 871}, {"y": 0.002, "x": 881}, {"y": 0.002, "x": 891}, {"y": 0.002, "x": 901}, {"y": 0.002, "x": 911}, {"y": 0.002, "x": 921}, {"y": 0.002, "x": 931}, {"y": 0.002, "x": 941}, {"y": 0.002, "x": 951}, {"y": 0.002, "x": 961}, {"y": 0.002, "x": 971}, {"y": 0.002, "x": 981}, {"y": 0.002, "x": 991}, {"y": 0.002, "x": 1001}, {"y": 0.002, "x": 1011}, {"y": 0.002, "x": 1021}, {"y": 0.002, "x": 1031}, {"y": 0.002, "x": 1041}, {"y": 0.002, "x": 1051}, {"y": 0.002, "x": 1061}, {"y": 0.002, "x": 1071}, {"y": 0.002, "x": 1081}, {"y": 0.002, "x": 1091}, {"y": 0.002, "x": 1101}, {"y": 0.002, "x": 1111}, {"y": 0.002, "x": 1121}, {"y": 0.002, "x": 1131}, {"y": 0.002, "x": 1141}, {"y": 0.002, "x": 1151}, {"y": 0.002, "x": 1161}, {"y": 0.002, "x": 1171}, {"y": 0.002, "x": 1181}, {"y": 0.002, "x": 1191}, {"y": 0.002, "x": 1201}, {"y": 0.002, "x": 1211}, {"y": 0.002, "x": 1221}, {"y": 0.002, "x": 1231}, {"y": 0.002, "x": 1241}, {"y": 0.002, "x": 1251}, {"y": 0.002, "x": 1261}, {"y": 0.002, "x": 1271}, {"y": 0.002, "x": 1281}, {"y": 0.002, "x": 1291}, {"y": 0.002, "x": 1301}, {"y": 0.002, "x": 1311}, {"y": 0.002, "x": 1321}, {"y": 0.002, "x": 1331}, {"y": 0.002, "x": 1341}, {"y": 0.002, "x": 1351}, {"y": 0.002, "x": 1361}, {"y": 0.002, "x": 1371}, {"y": 0.002, "x": 1381}, {"y": 0.002, "x": 1391}, {"y": 0.002, "x": 1401}, {"y": 0.002, "x": 1411}, {"y": 0.002, "x": 1421}, {"y": 0.002, "x": 1431}, {"y": 0.002, "x": 1441}, {"y": 0.002, "x": 1451}, {"y": 0.002, "x": 1461}, {"y": 0.002, "x": 1471}, {"y": 0.002, "x": 1481}, {"y": 0.002, "x": 1491}, {"y": 0.002, "x": 1501}, {"y": 0.002, "x": 1511}, {"y": 0.002, "x": 1521}, {"y": 0.002, "x": 1531}, {"y": 0.002, "x": 1541}, {"y": 0.002, "x": 1551}, {"y": 0.002, "x": 1561}, {"y": 0.002, "x": 1571}, {"y": 0.002, "x": 1581}, {"y": 0.002, "x": 1591}, {"y": 0.002, "x": 1601}, {"y": 0.002, "x": 1611}, {"y": 0.002, "x": 1621}, {"y": 0.002, "x": 1631}, {"y": 0.002, "x": 1641}, {"y": 0.002, "x": 1651}, {"y": 0.002, "x": 1661}, {"y": 0.002, "x": 1671}, {"y": 0.002, "x": 1681}, {"y": 0.002, "x": 1691}, {"y": 0.002, "x": 1701}, {"y": 0.002, "x": 1711}, {"y": 0.002, "x": 1721}, {"y": 0.002, "x": 1731}, {"y": 0.002, "x": 1741}, {"y": 0.002, "x": 1751}, {"y": 0.002, "x": 1761}, {"y": 0.002, "x": 1771}, {"y": 0.002, "x": 1781}, {"y": 0.002, "x": 1791}, {"y": 0.002, "x": 1801}, {"y": 0.002, "x": 1811}, {"y": 0.002, "x": 1821}, {"y": 0.002, "x": 1831}, {"y": 0.002, "x": 1841}, {"y": 0.002, "x": 1851}, {"y": 0.002, "x": 1861}, {"y": 0.002, "x": 1871}, {"y": 0.002, "x": 1881}, {"y": 0.002, "x": 1891}, {"y": 0.002, "x": 1901}, {"y": 0.002, "x": 1911}, {"y": 0.002, "x": 1921}, {"y": 0.002, "x": 1931}, {"y": 0.002, "x": 1941}, {"y": 0.002, "x": 1951}, {"y": 0.002, "x": 1961}, {"y": 0.002, "x": 1971}, {"y": 0.002, "x": 1981}, {"y": 0.002, "x": 1991}, {"y": 0.002, "x": 2001}, {"y": 0.002, "x": 2011}, {"y": 0.002, "x": 2021}, {"y": 0.002, "x": 2031}, {"y": 0.002, "x": 2041}, {"y": 0.002, "x": 2051}, {"y": 0.002, "x": 2061}, {"y": 0.002, "x": 2071}], "key": "insert average latency(ms)", "yAxis": "1"}, {"type": "line", "values": [{"y": 0.051999999999999998, "x": 0}, {"y": 1.514, "x": 10}, {"y": 0.031, "x": 20}, {"y": 0.041000000000000002, "x": 30}, {"y": 0.024, "x": 40}, {"y": 0.028000000000000001, "x": 50}, {"y": 0.044999999999999998, "x": 60}, {"y": 0.021000000000000001, "x": 70}, {"y": 3.552, "x": 80}, {"y": 3.8639999999999999, "x": 90}, {"y": 3.6589999999999998, "x": 100}, {"y": 3.7370000000000001, "x": 110}, {"y": 0.027, "x": 120}, {"y": 117.611, "x": 130}, {"y": 0.045999999999999999, "x": 140}, {"y": 81.257000000000005, "x": 150}, {"y": 131.268, "x": 160}, {"y": 4.024, "x": 170}, {"y": 0.045999999999999999, "x": 180}, {"y": 0.025999999999999999, "x": 190}, {"y": 3.7330000000000001, "x": 200}, {"y": 0.028000000000000001, "x": 210}, {"y": 0.069000000000000006, "x": 220}, {"y": 4.1710000000000003, "x": 230}, {"y": 3.7879999999999998, "x": 240}, {"y": 1.5580000000000001, "x": 250}, {"y": 52.651000000000003, "x": 260}, {"y": 4.2409999999999997, "x": 270}, {"y": 0.021000000000000001, "x": 280}, {"y": 0.055, "x": 290}, {"y": 19.741, "x": 300}, {"y": 24.530999999999999, "x": 310}, {"y": 0.23000000000000001, "x": 320}, {"y": 1.337, "x": 330}, {"y": 0.024, "x": 340}, {"y": 0.156, "x": 350}, {"y": 1.121, "x": 360}, {"y": 0.83199999999999996, "x": 370}, {"y": 0.034000000000000002, "x": 380}, {"y": 1.6160000000000001, "x": 390}, {"y": 0.028000000000000001, "x": 400}, {"y": 1.891, "x": 410}, {"y": 0.076999999999999999, "x": 420}, {"y": 0.021999999999999999, "x": 430}, {"y": 4.8390000000000004, "x": 440}, {"y": 175.25899999999999, "x": 450}, {"y": 0.13700000000000001, "x": 460}, {"y": 0.037999999999999999, "x": 470}, {"y": 4.8150000000000004, "x": 480}, {"y": 4.8150000000000004, "x": 490}, {"y": 4.8150000000000004, "x": 500}, {"y": 4.8150000000000004, "x": 510}, {"y": 4.8150000000000004, "x": 520}, {"y": 4.8150000000000004, "x": 530}, {"y": 4.8150000000000004, "x": 540}, {"y": 4.8150000000000004, "x": 550}, {"y": 4.8150000000000004, "x": 560}, {"y": 4.8150000000000004, "x": 570}, {"y": 4.8150000000000004, "x": 580}, {"y": 4.8150000000000004, "x": 590}, {"y": 4.8150000000000004, "x": 600}, {"y": 4.8150000000000004, "x": 610}, {"y": 4.8150000000000004, "x": 620}, {"y": 4.8150000000000004, "x": 630}, {"y": 4.8150000000000004, "x": 640}, {"y": 4.8150000000000004, "x": 650}, {"y": 4.8150000000000004, "x": 660}, {"y": 4.8150000000000004, "x": 670}, {"y": 4.8150000000000004, "x": 680}, {"y": 4.8150000000000004, "x": 690}, {"y": 4.8150000000000004, "x": 700}, {"y": 4.8150000000000004, "x": 710}, {"y": 4.8150000000000004, "x": 720}, {"y": 4.8150000000000004, "x": 730}, {"y": 4.8150000000000004, "x": 741}, {"y": 4.8150000000000004, "x": 751}, {"y": 4.8150000000000004, "x": 761}, {"y": 4.8150000000000004, "x": 771}, {"y": 4.8150000000000004, "x": 781}, {"y": 4.8150000000000004, "x": 791}, {"y": 4.8150000000000004, "x": 801}, {"y": 4.8150000000000004, "x": 811}, {"y": 4.8150000000000004, "x": 821}, {"y": 4.8150000000000004, "x": 831}, {"y": 4.8150000000000004, "x": 841}, {"y": 4.8150000000000004, "x": 851}, {"y": 4.8150000000000004, "x": 861}, {"y": 4.8150000000000004, "x": 871}, {"y": 4.8150000000000004, "x": 881}, {"y": 4.8150000000000004, "x": 891}, {"y": 4.8150000000000004, "x": 901}, {"y": 4.8150000000000004, "x": 911}, {"y": 4.8150000000000004, "x": 921}, {"y": 4.8150000000000004, "x": 931}, {"y": 4.8150000000000004, "x": 941}, {"y": 4.8150000000000004, "x": 951}, {"y": 4.8150000000000004, "x": 961}, {"y": 4.8150000000000004, "x": 971}, {"y": 4.8150000000000004, "x": 981}, {"y": 4.8150000000000004, "x": 991}, {"y": 4.8150000000000004, "x": 1001}, {"y": 4.8150000000000004, "x": 1011}, {"y": 4.8150000000000004, "x": 1021}, {"y": 4.8150000000000004, "x": 1031}, {"y": 4.8150000000000004, "x": 1041}, {"y": 4.8150000000000004, "x": 1051}, {"y": 4.8150000000000004, "x": 1061}, {"y": 4.8150000000000004, "x": 1071}, {"y": 4.8150000000000004, "x": 1081}, {"y": 4.8150000000000004, "x": 1091}, {"y": 4.8150000000000004, "x": 1101}, {"y": 4.8150000000000004, "x": 1111}, {"y": 4.8150000000000004, "x": 1121}, {"y": 4.8150000000000004, "x": 1131}, {"y": 4.8150000000000004, "x": 1141}, {"y": 4.8150000000000004, "x": 1151}, {"y": 4.8150000000000004, "x": 1161}, {"y": 4.8150000000000004, "x": 1171}, {"y": 4.8150000000000004, "x": 1181}, {"y": 4.8150000000000004, "x": 1191}, {"y": 4.8150000000000004, "x": 1201}, {"y": 4.8150000000000004, "x": 1211}, {"y": 4.8150000000000004, "x": 1221}, {"y": 4.8150000000000004, "x": 1231}, {"y": 4.8150000000000004, "x": 1241}, {"y": 4.8150000000000004, "x": 1251}, {"y": 4.8150000000000004, "x": 1261}, {"y": 4.8150000000000004, "x": 1271}, {"y": 4.8150000000000004, "x": 1281}, {"y": 4.8150000000000004, "x": 1291}, {"y": 4.8150000000000004, "x": 1301}, {"y": 4.8150000000000004, "x": 1311}, {"y": 4.8150000000000004, "x": 1321}, {"y": 4.8150000000000004, "x": 1331}, {"y": 4.8150000000000004, "x": 1341}, {"y": 4.8150000000000004, "x": 1351}, {"y": 4.8150000000000004, "x": 1361}, {"y": 4.8150000000000004, "x": 1371}, {"y": 4.8150000000000004, "x": 1381}, {"y": 4.8150000000000004, "x": 1391}, {"y": 4.8150000000000004, "x": 1401}, {"y": 4.8150000000000004, "x": 1411}, {"y": 4.8150000000000004, "x": 1421}, {"y": 4.8150000000000004, "x": 1431}, {"y": 4.8150000000000004, "x": 1441}, {"y": 4.8150000000000004, "x": 1451}, {"y": 4.8150000000000004, "x": 1461}, {"y": 4.8150000000000004, "x": 1471}, {"y": 4.8150000000000004, "x": 1481}, {"y": 4.8150000000000004, "x": 1491}, {"y": 4.8150000000000004, "x": 1501}, {"y": 4.8150000000000004, "x": 1511}, {"y": 4.8150000000000004, "x": 1521}, {"y": 4.8150000000000004, "x": 1531}, {"y": 4.8150000000000004, "x": 1541}, {"y": 4.8150000000000004, "x": 1551}, {"y": 4.8150000000000004, "x": 1561}, {"y": 4.8150000000000004, "x": 1571}, {"y": 4.8150000000000004, "x": 1581}, {"y": 4.8150000000000004, "x": 1591}, {"y": 4.8150000000000004, "x": 1601}, {"y": 4.8150000000000004, "x": 1611}, {"y": 4.8150000000000004, "x": 1621}, {"y": 4.8150000000000004, "x": 1631}, {"y": 4.8150000000000004, "x": 1641}, {"y": 4.8150000000000004, "x": 1651}, {"y": 4.8150000000000004, "x": 1661}, {"y": 4.8150000000000004, "x": 1671}, {"y": 4.8150000000000004, "x": 1681}, {"y": 4.8150000000000004, "x": 1691}, {"y": 4.8150000000000004, "x": 1701}, {"y": 4.8150000000000004, "x": 1711}, {"y": 4.8150000000000004, "x": 1721}, {"y": 4.8150000000000004, "x": 1731}, {"y": 4.8150000000000004, "x": 1741}, {"y": 4.8150000000000004, "x": 1751}, {"y": 4.8150000000000004, "x": 1761}, {"y": 4.8150000000000004, "x": 1771}, {"y": 4.8150000000000004, "x": 1781}, {"y": 4.8150000000000004, "x": 1791}, {"y": 4.8150000000000004, "x": 1801}, {"y": 4.8150000000000004, "x": 1811}, {"y": 4.8150000000000004, "x": 1821}, {"y": 4.8150000000000004, "x": 1831}, {"y": 4.8150000000000004, "x": 1841}, {"y": 4.8150000000000004, "x": 1851}, {"y": 4.8150000000000004, "x": 1861}, {"y": 4.8150000000000004, "x": 1871}, {"y": 4.8150000000000004, "x": 1881}, {"y": 4.8150000000000004, "x": 1891}, {"y": 4.8150000000000004, "x": 1901}, {"y": 4.8150000000000004, "x": 1911}, {"y": 4.8150000000000004, "x": 1921}, {"y": 4.8150000000000004, "x": 1931}, {"y": 4.8150000000000004, "x": 1941}, {"y": 4.8150000000000004, "x": 1951}, {"y": 4.8150000000000004, "x": 1961}, {"y": 4.8150000000000004, "x": 1971}, {"y": 4.8150000000000004, "x": 1981}, {"y": 4.8150000000000004, "x": 1991}, {"y": 4.8150000000000004, "x": 2001}, {"y": 4.8150000000000004, "x": 2011}, {"y": 4.8150000000000004, "x": 2021}, {"y": 4.8150000000000004, "x": 2031}, {"y": 4.8150000000000004, "x": 2041}, {"y": 4.8150000000000004, "x": 2051}, {"y": 4.8150000000000004, "x": 2061}, {"y": 4.8150000000000004, "x": 2071}], "key": "insert maximum latency(ms)", "yAxis": "1"}, {"type": "line", "values": [{"y": 0.001, "x": 0}, {"y": 0.001, "x": 10}, {"y": 0.001, "x": 20}, {"y": 0.001, "x": 30}, {"y": 0.001, "x": 40}, {"y": 0.001, "x": 50}, {"y": 0.001, "x": 60}, {"y": 0.001, "x": 70}, {"y": 0.001, "x": 80}, {"y": 0.002, "x": 90}, {"y": 0.001, "x": 100}, {"y": 0.001, "x": 110}, {"y": 0.001, "x": 120}, {"y": 0.001, "x": 130}, {"y": 0.001, "x": 140}, {"y": 0.001, "x": 150}, {"y": 0.001, "x": 160}, {"y": 0.001, "x": 170}, {"y": 0.001, "x": 180}, {"y": 0.001, "x": 190}, {"y": 0.001, "x": 200}, {"y": 0.001, "x": 210}, {"y": 0.001, "x": 220}, {"y": 0.001, "x": 230}, {"y": 0.001, "x": 240}, {"y": 0.001, "x": 250}, {"y": 0.001, "x": 260}, {"y": 0.001, "x": 270}, {"y": 0.001, "x": 280}, {"y": 0.001, "x": 290}, {"y": 0.001, "x": 300}, {"y": 0.001, "x": 310}, {"y": 0.001, "x": 320}, {"y": 0.001, "x": 330}, {"y": 0.001, "x": 340}, {"y": 0.001, "x": 350}, {"y": 0.001, "x": 360}, {"y": 0.001, "x": 370}, {"y": 0.001, "x": 380}, {"y": 0.001, "x": 390}, {"y": 0.001, "x": 400}, {"y": 0.001, "x": 410}, {"y": 0.001, "x": 420}, {"y": 0.001, "x": 430}, {"y": 0.001, "x": 440}, {"y": 0.001, "x": 450}, {"y": 0.001, "x": 460}, {"y": 0.001, "x": 470}, {"y": 0.001, "x": 480}, {"y": 0.001, "x": 490}, {"y": 0.001, "x": 500}, {"y": 0.001, "x": 510}, {"y": 0.001, "x": 520}, {"y": 0.001, "x": 530}, {"y": 0.001, "x": 540}, {"y": 0.001, "x": 550}, {"y": 0.001, "x": 560}, {"y": 0.001, "x": 570}, {"y": 0.001, "x": 580}, {"y": 0.001, "x": 590}, {"y": 0.001, "x": 600}, {"y": 0.001, "x": 610}, {"y": 0.001, "x": 620}, {"y": 0.001, "x": 630}, {"y": 0.001, "x": 640}, {"y": 0.001, "x": 650}, {"y": 0.001, "x": 660}, {"y": 0.001, "x": 670}, {"y": 0.001, "x": 680}, {"y": 0.001, "x": 690}, {"y": 0.001, "x": 700}, {"y": 0.001, "x": 710}, {"y": 0.001, "x": 720}, {"y": 0.001, "x": 730}, {"y": 0.001, "x": 741}, {"y": 0.001, "x": 751}, {"y": 0.001, "x": 761}, {"y": 0.001, "x": 771}, {"y": 0.001, "x": 781}, {"y": 0.001, "x": 791}, {"y": 0.001, "x": 801}, {"y": 0.001, "x": 811}, {"y": 0.001, "x": 821}, {"y": 0.001, "x": 831}, {"y": 0.001, "x": 841}, {"y": 0.001, "x": 851}, {"y": 0.001, "x": 861}, {"y": 0.001, "x": 871}, {"y": 0.001, "x": 881}, {"y": 0.001, "x": 891}, {"y": 0.001, "x": 901}, {"y": 0.001, "x": 911}, {"y": 0.001, "x": 921}, {"y": 0.001, "x": 931}, {"y": 0.001, "x": 941}, {"y": 0.001, "x": 951}, {"y": 0.001, "x": 961}, {"y": 0.001, "x": 971}, {"y": 0.001, "x": 981}, {"y": 0.001, "x": 991}, {"y": 0.001, "x": 1001}, {"y": 0.001, "x": 1011}, {"y": 0.001, "x": 1021}, {"y": 0.001, "x": 1031}, {"y": 0.001, "x": 1041}, {"y": 0.001, "x": 1051}, {"y": 0.001, "x": 1061}, {"y": 0.001, "x": 1071}, {"y": 0.001, "x": 1081}, {"y": 0.001, "x": 1091}, {"y": 0.001, "x": 1101}, {"y": 0.001, "x": 1111}, {"y": 0.001, "x": 1121}, {"y": 0.001, "x": 1131}, {"y": 0.001, "x": 1141}, {"y": 0.001, "x": 1151}, {"y": 0.001, "x": 1161}, {"y": 0.001, "x": 1171}, {"y": 0.001, "x": 1181}, {"y": 0.001, "x": 1191}, {"y": 0.001, "x": 1201}, {"y": 0.001, "x": 1211}, {"y": 0.001, "x": 1221}, {"y": 0.001, "x": 1231}, {"y": 0.001, "x": 1241}, {"y": 0.001, "x": 1251}, {"y": 0.001, "x": 1261}, {"y": 0.001, "x": 1271}, {"y": 0.001, "x": 1281}, {"y": 0.001, "x": 1291}, {"y": 0.001, "x": 1301}, {"y": 0.001, "x": 1311}, {"y": 0.001, "x": 1321}, {"y": 0.001, "x": 1331}, {"y": 0.001, "x": 1341}, {"y": 0.001, "x": 1351}, {"y": 0.001, "x": 1361}, {"y": 0.001, "x": 1371}, {"y": 0.001, "x": 1381}, {"y": 0.001, "x": 1391}, {"y": 0.001, "x": 1401}, {"y": 0.001, "x": 1411}, {"y": 0.001, "x": 1421}, {"y": 0.001, "x": 1431}, {"y": 0.001, "x": 1441}, {"y": 0.001, "x": 1451}, {"y": 0.001, "x": 1461}, {"y": 0.001, "x": 1471}, {"y": 0.001, "x": 1481}, {"y": 0.001, "x": 1491}, {"y": 0.001, "x": 1501}, {"y": 0.001, "x": 1511}, {"y": 0.001, "x": 1521}, {"y": 0.001, "x": 1531}, {"y": 0.001, "x": 1541}, {"y": 0.001, "x": 1551}, {"y": 0.001, "x": 1561}, {"y": 0.001, "x": 1571}, {"y": 0.001, "x": 1581}, {"y": 0.001, "x": 1591}, {"y": 0.001, "x": 1601}, {"y": 0.001, "x": 1611}, {"y": 0.001, "x": 1621}, {"y": 0.001, "x": 1631}, {"y": 0.001, "x": 1641}, {"y": 0.001, "x": 1651}, {"y": 0.001, "x": 1661}, {"y": 0.001, "x": 1671}, {"y": 0.001, "x": 1681}, {"y": 0.001, "x": 1691}, {"y": 0.001, "x": 1701}, {"y": 0.001, "x": 1711}, {"y": 0.001, "x": 1721}, {"y": 0.001, "x": 1731}, {"y": 0.001, "x": 1741}, {"y": 0.001, "x": 1751}, {"y": 0.001, "x": 1761}, {"y": 0.001, "x": 1771}, {"y": 0.001, "x": 1781}, {"y": 0.001, "x": 1791}, {"y": 0.001, "x": 1801}, {"y": 0.001, "x": 1811}, {"y": 0.001, "x": 1821}, {"y": 0.001, "x": 1831}, {"y": 0.001, "x": 1841}, {"y": 0.001, "x": 1851}, {"y": 0.001, "x": 1861}, {"y": 0.001, "x": 1871}, {"y": 0.001, "x": 1881}, {"y": 0.001, "x": 1891}, {"y": 0.001, "x": 1901}, {"y": 0.001, "x": 1911}, {"y": 0.001, "x": 1921}, {"y": 0.001, "x": 1931}, {"y": 0.001, "x": 1941}, {"y": 0.001, "x": 1951}, {"y": 0.001, "x": 1961}, {"y": 0.001, "x": 1971}, {"y": 0.001, "x": 1981}, {"y": 0.001, "x": 1991}, {"y": 0.001, "x": 2001}, {"y": 0.001, "x": 2011}, {"y": 0.001, "x": 2021}, {"y": 0.001, "x": 2031}, {"y": 0.001, "x": 2041}, {"y": 0.001, "x": 2051}, {"y": 0.001, "x": 2061}, {"y": 0.001, "x": 2071}], "key": "insert min latency(ms)", "yAxis": "1"}, {"type": "line", "values": [{"y": 673.32399999999996, "x": 0}, {"y": 613.65099999999995, "x": 10}, {"y": 569.447, "x": 20}, {"y": 549.23299999999995, "x": 30}, {"y": 533.52800000000002, "x": 40}, {"y": 531.38099999999997, "x": 50}, {"y": 541.70399999999995, "x": 60}, {"y": 523.24000000000001, "x": 70}, {"y": 509.34399999999999, "x": 80}, {"y": 518.51300000000003, "x": 90}, {"y": 504.92500000000001, "x": 100}, {"y": 533.303, "x": 110}, {"y": 521.779, "x": 120}, {"y": 461.125, "x": 130}, {"y": 463.322, "x": 140}, {"y": 496.60300000000001, "x": 150}, {"y": 538.97500000000002, "x": 160}, {"y": 529.49000000000001, "x": 170}, {"y": 425.33699999999999, "x": 180}, {"y": 449.089, "x": 190}, {"y": 474.11700000000002, "x": 200}, {"y": 551.91700000000003, "x": 210}, {"y": 505.95699999999999, "x": 220}, {"y": 511.49000000000001, "x": 230}, {"y": 467.42000000000002, "x": 240}, {"y": 438.48500000000001, "x": 250}, {"y": 509.20600000000002, "x": 260}, {"y": 516.97500000000002, "x": 270}, {"y": 501.05399999999997, "x": 280}, {"y": 532.24300000000005, "x": 290}, {"y": 479.39499999999998, "x": 300}, {"y": 513.63300000000004, "x": 310}, {"y": 492.87700000000001, "x": 320}, {"y": 468.93000000000001, "x": 330}, {"y": 478.51499999999999, "x": 340}, {"y": 491.68799999999999, "x": 350}, {"y": 476.03300000000002, "x": 360}, {"y": 455.291, "x": 370}, {"y": 434.09100000000001, "x": 380}, {"y": 519.94200000000001, "x": 390}, {"y": 461.71600000000001, "x": 400}, {"y": 467.95299999999997, "x": 410}, {"y": 457.50200000000001, "x": 420}, {"y": 486.61500000000001, "x": 430}, {"y": 516.79700000000003, "x": 440}, {"y": 483.339, "x": 450}, {"y": 518.43899999999996, "x": 460}, {"y": 457.46300000000002, "x": 470}, {"y": 456.03800000000001, "x": 480}, {"y": 0.0, "x": 490}, {"y": 0.0, "x": 500}, {"y": 0.0, "x": 510}, {"y": 0.0, "x": 520}, {"y": 0.0, "x": 530}, {"y": 0.0, "x": 540}, {"y": 0.0, "x": 550}, {"y": 0.0, "x": 560}, {"y": 0.0, "x": 570}, {"y": 0.0, "x": 580}, {"y": 0.0, "x": 590}, {"y": 0.0, "x": 600}, {"y": 0.0, "x": 610}, {"y": 0.0, "x": 620}, {"y": 0.0, "x": 630}, {"y": 0.0, "x": 640}, {"y": 0.0, "x": 650}, {"y": 0.0, "x": 660}, {"y": 0.0, "x": 670}, {"y": 0.0, "x": 680}, {"y": 0.0, "x": 690}, {"y": 0.0, "x": 700}, {"y": 0.0, "x": 710}, {"y": 0.0, "x": 720}, {"y": 0.0, "x": 730}, {"y": 0.0, "x": 741}, {"y": 0.0, "x": 751}, {"y": 0.0, "x": 761}, {"y": 0.0, "x": 771}, {"y": 0.0, "x": 781}, {"y": 0.0, "x": 791}, {"y": 0.0, "x": 801}, {"y": 0.0, "x": 811}, {"y": 0.0, "x": 821}, {"y": 0.0, "x": 831}, {"y": 0.0, "x": 841}, {"y": 0.0, "x": 851}, {"y": 0.0, "x": 861}, {"y": 0.0, "x": 871}, {"y": 0.0, "x": 881}, {"y": 0.0, "x": 891}, {"y": 0.0, "x": 901}, {"y": 0.0, "x": 911}, {"y": 0.0, "x": 921}, {"y": 0.0, "x": 931}, {"y": 0.0, "x": 941}, {"y": 0.0, "x": 951}, {"y": 0.0, "x": 961}, {"y": 0.0, "x": 971}, {"y": 0.0, "x": 981}, {"y": 0.0, "x": 991}, {"y": 0.0, "x": 1001}, {"y": 0.0, "x": 1011}, {"y": 0.0, "x": 1021}, {"y": 0.0, "x": 1031}, {"y": 0.0, "x": 1041}, {"y": 0.0, "x": 1051}, {"y": 0.0, "x": 1061}, {"y": 0.0, "x": 1071}, {"y": 0.0, "x": 1081}, {"y": 0.0, "x": 1091}, {"y": 0.0, "x": 1101}, {"y": 0.0, "x": 1111}, {"y": 0.0, "x": 1121}, {"y": 0.0, "x": 1131}, {"y": 0.0, "x": 1141}, {"y": 0.0, "x": 1151}, {"y": 0.0, "x": 1161}, {"y": 0.0, "x": 1171}, {"y": 0.0, "x": 1181}, {"y": 0.0, "x": 1191}, {"y": 0.0, "x": 1201}, {"y": 0.0, "x": 1211}, {"y": 0.0, "x": 1221}, {"y": 0.0, "x": 1231}, {"y": 0.0, "x": 1241}, {"y": 0.0, "x": 1251}, {"y": 0.0, "x": 1261}, {"y": 0.0, "x": 1271}, {"y": 0.0, "x": 1281}, {"y": 0.0, "x": 1291}, {"y": 0.0, "x": 1301}, {"y": 0.0, "x": 1311}, {"y": 0.0, "x": 1321}, {"y": 0.0, "x": 1331}, {"y": 0.0, "x": 1341}, {"y": 0.0, "x": 1351}, {"y": 0.0, "x": 1361}, {"y": 0.0, "x": 1371}, {"y": 0.0, "x": 1381}, {"y": 0.0, "x": 1391}, {"y": 0.0, "x": 1401}, {"y": 0.0, "x": 1411}, {"y": 0.0, "x": 1421}, {"y": 0.0, "x": 1431}, {"y": 0.0, "x": 1441}, {"y": 0.0, "x": 1451}, {"y": 0.0, "x": 1461}, {"y": 0.0, "x": 1471}, {"y": 0.0, "x": 1481}, {"y": 0.0, "x": 1491}, {"y": 0.0, "x": 1501}, {"y": 0.0, "x": 1511}, {"y": 0.0, "x": 1521}, {"y": 0.0, "x": 1531}, {"y": 0.0, "x": 1541}, {"y": 0.0, "x": 1551}, {"y": 0.0, "x": 1561}, {"y": 0.0, "x": 1571}, {"y": 0.0, "x": 1581}, {"y": 0.0, "x": 1591}, {"y": 0.0, "x": 1601}, {"y": 0.0, "x": 1611}, {"y": 0.0, "x": 1621}, {"y": 0.0, "x": 1631}, {"y": 0.0, "x": 1641}, {"y": 0.0, "x": 1651}, {"y": 0.0, "x": 1661}, {"y": 0.0, "x": 1671}, {"y": 0.0, "x": 1681}, {"y": 0.0, "x": 1691}, {"y": 0.0, "x": 1701}, {"y": 0.0, "x": 1711}, {"y": 0.0, "x": 1721}, {"y": 0.0, "x": 1731}, {"y": 0.0, "x": 1741}, {"y": 0.0, "x": 1751}, {"y": 0.0, "x": 1761}, {"y": 0.0, "x": 1771}, {"y": 0.0, "x": 1781}, {"y": 0.0, "x": 1791}, {"y": 0.0, "x": 1801}, {"y": 0.0, "x": 1811}, {"y": 0.0, "x": 1821}, {"y": 0.0, "x": 1831}, {"y": 0.0, "x": 1841}, {"y": 0.0, "x": 1851}, {"y": 0.0, "x": 1861}, {"y": 0.0, "x": 1871}, {"y": 0.0, "x": 1881}, {"y": 0.0, "x": 1891}, {"y": 0.0, "x": 1901}, {"y": 0.0, "x": 1911}, {"y": 0.0, "x": 1921}, {"y": 0.0, "x": 1931}, {"y": 0.0, "x": 1941}, {"y": 0.0, "x": 1951}, {"y": 0.0, "x": 1961}, {"y": 0.0, "x": 1971}, {"y": 0.0, "x": 1981}, {"y": 0.0, "x": 1991}, {"y": 0.0, "x": 2001}, {"y": 0.0, "x": 2011}, {"y": 0.0, "x": 2021}, {"y": 0.0, "x": 2031}, {"y": 0.0, "x": 2041}, {"y": 0.0, "x": 2051}, {"y": 0.0, "x": 2061}, {"y": 0.0, "x": 2071}], "key": "insert ops per second (thousands)", "yAxis": "2"}, {"type": "line", "values": [{"y": 0.0, "x": 0}, {"y": 0.0, "x": 10}, {"y": 0.0, "x": 20}, {"y": 0.0, "x": 30}, {"y": 0.0, "x": 40}, {"y": 0.0, "x": 50}, {"y": 0.0, "x": 60}, {"y": 0.0, "x": 70}, {"y": 0.0, "x": 80}, {"y": 0.0, "x": 90}, {"y": 0.0, "x": 100}, {"y": 0.0, "x": 110}, {"y": 0.0, "x": 120}, {"y": 0.0, "x": 130}, {"y": 0.0, "x": 140}, {"y": 0.0, "x": 150}, {"y": 0.0, "x": 160}, {"y": 0.0, "x": 170}, {"y": 0.0, "x": 180}, {"y": 0.0, "x": 190}, {"y": 0.0, "x": 200}, {"y": 0.0, "x": 210}, {"y": 0.0, "x": 220}, {"y": 0.0, "x": 230}, {"y": 0.0, "x": 240}, {"y": 0.0, "x": 250}, {"y": 0.0, "x": 260}, {"y": 0.0, "x": 270}, {"y": 0.0, "x": 280}, {"y": 0.0, "x": 290}, {"y": 0.0, "x": 300}, {"y": 0.0, "x": 310}, {"y": 0.0, "x": 320}, {"y": 0.0, "x": 330}, {"y": 0.0, "x": 340}, {"y": 0.0, "x": 350}, {"y": 0.0, "x": 360}, {"y": 0.0, "x": 370}, {"y": 0.0, "x": 380}, {"y": 0.0, "x": 390}, {"y": 0.0, "x": 400}, {"y": 0.0, "x": 410}, {"y": 0.0, "x": 420}, {"y": 0.0, "x": 430}, {"y": 0.0, "x": 440}, {"y": 0.0, "x": 450}, {"y": 0.0, "x": 460}, {"y": 0.0, "x": 470}, {"y": 0.0, "x": 480}, {"y": 0.0, "x": 490}, {"y": 0.0, "x": 500}, {"y": 0.0, "x": 510}, {"y": 0.0, "x": 520}, {"y": 0.0, "x": 530}, {"y": 0.0, "x": 540}, {"y": 0.0, "x": 550}, {"y": 0.0, "x": 560}, {"y": 0.0, "x": 570}, {"y": 0.0, "x": 580}, {"y": 0.0, "x": 590}, {"y": 0.0, "x": 600}, {"y": 0.0, "x": 610}, {"y": 0.0, "x": 620}, {"y": 0.0, "x": 630}, {"y": 0.0, "x": 640}, {"y": 0.0, "x": 650}, {"y": 0.0, "x": 660}, {"y": 0.0, "x": 670}, {"y": 0.0, "x": 680}, {"y": 0.0, "x": 690}, {"y": 0.0, "x": 700}, {"y": 0.0, "x": 710}, {"y": 0.0, "x": 720}, {"y": 0.0, "x": 730}, {"y": 0.0, "x": 741}, {"y": 0.0, "x": 751}, {"y": 0.0, "x": 761}, {"y": 0.0, "x": 771}, {"y": 0.0, "x": 781}, {"y": 0.0, "x": 791}, {"y": 0.0, "x": 801}, {"y": 0.0, "x": 811}, {"y": 0.0, "x": 821}, {"y": 0.0, "x": 831}, {"y": 0.0, "x": 841}, {"y": 0.0, "x": 851}, {"y": 0.0, "x": 861}, {"y": 0.0, "x": 871}, {"y": 1.101, "x": 881}, {"y": 0.84699999999999998, "x": 891}, {"y": 0.75, "x": 901}, {"y": 0.64600000000000002, "x": 911}, {"y": 0.61799999999999999, "x": 921}, {"y": 0.60499999999999998, "x": 931}, {"y": 0.38500000000000001, "x": 941}, {"y": 0.14399999999999999, "x": 951}, {"y": 0.104, "x": 961}, {"y": 0.063, "x": 971}, {"y": 0.058000000000000003, "x": 981}, {"y": 0.053999999999999999, "x": 991}, {"y": 0.050000000000000003, "x": 1001}, {"y": 0.045999999999999999, "x": 1011}, {"y": 0.043999999999999997, "x": 1021}, {"y": 0.042999999999999997, "x": 1031}, {"y": 0.044999999999999998, "x": 1041}, {"y": 0.042000000000000003, "x": 1051}, {"y": 0.042999999999999997, "x": 1061}, {"y": 0.042999999999999997, "x": 1071}, {"y": 0.042999999999999997, "x": 1081}, {"y": 0.042000000000000003, "x": 1091}, {"y": 0.042999999999999997, "x": 1101}, {"y": 0.041000000000000002, "x": 1111}, {"y": 0.042000000000000003, "x": 1121}, {"y": 0.041000000000000002, "x": 1131}, {"y": 0.041000000000000002, "x": 1141}, {"y": 0.042000000000000003, "x": 1151}, {"y": 0.042000000000000003, "x": 1161}, {"y": 0.041000000000000002, "x": 1171}, {"y": 0.042000000000000003, "x": 1181}, {"y": 0.042999999999999997, "x": 1191}, {"y": 0.042000000000000003, "x": 1201}, {"y": 0.042000000000000003, "x": 1211}, {"y": 0.042999999999999997, "x": 1221}, {"y": 0.042999999999999997, "x": 1231}, {"y": 0.042000000000000003, "x": 1241}, {"y": 0.042999999999999997, "x": 1251}, {"y": 0.042000000000000003, "x": 1261}, {"y": 0.043999999999999997, "x": 1271}, {"y": 0.042000000000000003, "x": 1281}, {"y": 0.042000000000000003, "x": 1291}, {"y": 0.042000000000000003, "x": 1301}, {"y": 0.041000000000000002, "x": 1311}, {"y": 0.043999999999999997, "x": 1321}, {"y": 0.043999999999999997, "x": 1331}, {"y": 0.041000000000000002, "x": 1341}, {"y": 0.042999999999999997, "x": 1351}, {"y": 0.042999999999999997, "x": 1361}, {"y": 0.042999999999999997, "x": 1371}, {"y": 0.043999999999999997, "x": 1381}, {"y": 0.042999999999999997, "x": 1391}, {"y": 0.043999999999999997, "x": 1401}, {"y": 0.042000000000000003, "x": 1411}, {"y": 0.042000000000000003, "x": 1421}, {"y": 0.042000000000000003, "x": 1431}, {"y": 0.042000000000000003, "x": 1441}, {"y": 0.041000000000000002, "x": 1451}, {"y": 0.042000000000000003, "x": 1461}, {"y": 0.042000000000000003, "x": 1471}, {"y": 0.042000000000000003, "x": 1481}, {"y": 0.042000000000000003, "x": 1491}, {"y": 0.041000000000000002, "x": 1501}, {"y": 0.042000000000000003, "x": 1511}, {"y": 0.042999999999999997, "x": 1521}, {"y": 0.042999999999999997, "x": 1531}, {"y": 0.042999999999999997, "x": 1541}, {"y": 0.042000000000000003, "x": 1551}, {"y": 0.042000000000000003, "x": 1561}, {"y": 0.042000000000000003, "x": 1571}, {"y": 0.042999999999999997, "x": 1581}, {"y": 0.042000000000000003, "x": 1591}, {"y": 0.040000000000000001, "x": 1601}, {"y": 0.041000000000000002, "x": 1611}, {"y": 0.042000000000000003, "x": 1621}, {"y": 0.042000000000000003, "x": 1631}, {"y": 0.042999999999999997, "x": 1641}, {"y": 0.042000000000000003, "x": 1651}, {"y": 0.042000000000000003, "x": 1661}, {"y": 0.041000000000000002, "x": 1671}, {"y": 0.041000000000000002, "x": 1681}, {"y": 0.042000000000000003, "x": 1691}, {"y": 0.042000000000000003, "x": 1701}, {"y": 0.042000000000000003, "x": 1711}, {"y": 0.042999999999999997, "x": 1721}, {"y": 0.042999999999999997, "x": 1731}, {"y": 0.042000000000000003, "x": 1741}, {"y": 0.040000000000000001, "x": 1751}, {"y": 0.042999999999999997, "x": 1761}, {"y": 0.042000000000000003, "x": 1771}, {"y": 0.042999999999999997, "x": 1781}, {"y": 0.041000000000000002, "x": 1791}, {"y": 0.042000000000000003, "x": 1801}, {"y": 0.042999999999999997, "x": 1811}, {"y": 0.043999999999999997, "x": 1821}, {"y": 0.042000000000000003, "x": 1831}, {"y": 0.043999999999999997, "x": 1841}, {"y": 0.042999999999999997, "x": 1851}, {"y": 0.041000000000000002, "x": 1861}, {"y": 0.042999999999999997, "x": 1871}, {"y": 0.042000000000000003, "x": 1881}, {"y": 0.042000000000000003, "x": 1891}, {"y": 0.041000000000000002, "x": 1901}, {"y": 0.041000000000000002, "x": 1911}, {"y": 0.042999999999999997, "x": 1921}, {"y": 0.041000000000000002, "x": 1931}, {"y": 0.042000000000000003, "x": 1941}, {"y": 0.042000000000000003, "x": 1951}, {"y": 0.043999999999999997, "x": 1961}, {"y": 0.042000000000000003, "x": 1971}, {"y": 0.042999999999999997, "x": 1981}, {"y": 0.041000000000000002, "x": 1991}, {"y": 0.042000000000000003, "x": 2001}, {"y": 0.041000000000000002, "x": 2011}, {"y": 0.042000000000000003, "x": 2021}, {"y": 0.042000000000000003, "x": 2031}, {"y": 0.042000000000000003, "x": 2041}, {"y": 0.042000000000000003, "x": 2051}, {"y": 0.042999999999999997, "x": 2061}, {"y": 0.042000000000000003, "x": 2071}], "key": "read average latency(ms)", "yAxis": "1"}, {"type": "line", "values": [{"y": 0.0, "x": 0}, {"y": 0.0, "x": 10}, {"y": 0.0, "x": 20}, {"y": 0.0, "x": 30}, {"y": 0.0, "x": 40}, {"y": 0.0, "x": 50}, {"y": 0.0, "x": 60}, {"y": 0.0, "x": 70}, {"y": 0.0, "x": 80}, {"y": 0.0, "x": 90}, {"y": 0.0, "x": 100}, {"y": 0.0, "x": 110}, {"y": 0.0, "x": 120}, {"y": 0.0, "x": 130}, {"y": 0.0, "x": 140}, {"y": 0.0, "x": 150}, {"y": 0.0, "x": 160}, {"y": 0.0, "x": 170}, {"y": 0.0, "x": 180}, {"y": 0.0, "x": 190}, {"y": 0.0, "x": 200}, {"y": 0.0, "x": 210}, {"y": 0.0, "x": 220}, {"y": 0.0, "x": 230}, {"y": 0.0, "x": 240}, {"y": 0.0, "x": 250}, {"y": 0.0, "x": 260}, {"y": 0.0, "x": 270}, {"y": 0.0, "x": 280}, {"y": 0.0, "x": 290}, {"y": 0.0, "x": 300}, {"y": 0.0, "x": 310}, {"y": 0.0, "x": 320}, {"y": 0.0, "x": 330}, {"y": 0.0, "x": 340}, {"y": 0.0, "x": 350}, {"y": 0.0, "x": 360}, {"y": 0.0, "x": 370}, {"y": 0.0, "x": 380}, {"y": 0.0, "x": 390}, {"y": 0.0, "x": 400}, {"y": 0.0, "x": 410}, {"y": 0.0, "x": 420}, {"y": 0.0, "x": 430}, {"y": 0.0, "x": 440}, {"y": 0.0, "x": 450}, {"y": 0.0, "x": 460}, {"y": 0.0, "x": 470}, {"y": 0.0, "x": 480}, {"y": 0.0, "x": 490}, {"y": 0.0, "x": 500}, {"y": 0.0, "x": 510}, {"y": 0.0, "x": 520}, {"y": 0.0, "x": 530}, {"y": 0.0, "x": 540}, {"y": 0.0, "x": 550}, {"y": 0.0, "x": 560}, {"y": 0.0, "x": 570}, {"y": 0.0, "x": 580}, {"y": 0.0, "x": 590}, {"y": 0.0, "x": 600}, {"y": 0.0, "x": 610}, {"y": 0.0, "x": 620}, {"y": 0.0, "x": 630}, {"y": 0.0, "x": 640}, {"y": 0.0, "x": 650}, {"y": 0.0, "x": 660}, {"y": 0.0, "x": 670}, {"y": 0.0, "x": 680}, {"y": 0.0, "x": 690}, {"y": 0.0, "x": 700}, {"y": 0.0, "x": 710}, {"y": 0.0, "x": 720}, {"y": 0.0, "x": 730}, {"y": 0.0, "x": 741}, {"y": 0.0, "x": 751}, {"y": 0.0, "x": 761}, {"y": 0.0, "x": 771}, {"y": 0.0, "x": 781}, {"y": 0.0, "x": 791}, {"y": 0.0, "x": 801}, {"y": 0.0, "x": 811}, {"y": 0.0, "x": 821}, {"y": 0.0, "x": 831}, {"y": 0.0, "x": 841}, {"y": 0.0, "x": 851}, {"y": 0.0, "x": 861}, {"y": 0.0, "x": 871}, {"y": 30.933, "x": 881}, {"y": 61.572000000000003, "x": 891}, {"y": 15.273, "x": 901}, {"y": 12.179, "x": 911}, {"y": 19.329000000000001, "x": 921}, {"y": 78.674000000000007, "x": 931}, {"y": 12.013, "x": 941}, {"y": 20.245999999999999, "x": 951}, {"y": 3.2709999999999999, "x": 961}, {"y": 3.1930000000000001, "x": 971}, {"y": 9.8190000000000008, "x": 981}, {"y": 8.7899999999999991, "x": 991}, {"y": 8.1349999999999998, "x": 1001}, {"y": 22.398, "x": 1011}, {"y": 5.9790000000000001, "x": 1021}, {"y": 3.5409999999999999, "x": 1031}, {"y": 3.2410000000000001, "x": 1041}, {"y": 5.9669999999999996, "x": 1051}, {"y": 6.1959999999999997, "x": 1061}, {"y": 3.4060000000000001, "x": 1071}, {"y": 3.0409999999999999, "x": 1081}, {"y": 3.1190000000000002, "x": 1091}, {"y": 45.298000000000002, "x": 1101}, {"y": 3.8519999999999999, "x": 1111}, {"y": 3.1150000000000002, "x": 1121}, {"y": 3.452, "x": 1131}, {"y": 3.1680000000000001, "x": 1141}, {"y": 4.9950000000000001, "x": 1151}, {"y": 6.0250000000000004, "x": 1161}, {"y": 5.8789999999999996, "x": 1171}, {"y": 2.1379999999999999, "x": 1181}, {"y": 4.484, "x": 1191}, {"y": 3.5049999999999999, "x": 1201}, {"y": 6.3029999999999999, "x": 1211}, {"y": 5.2629999999999999, "x": 1221}, {"y": 6.3579999999999997, "x": 1231}, {"y": 6.5369999999999999, "x": 1241}, {"y": 4.7530000000000001, "x": 1251}, {"y": 3.4159999999999999, "x": 1261}, {"y": 4.7800000000000002, "x": 1271}, {"y": 5.359, "x": 1281}, {"y": 3.3290000000000002, "x": 1291}, {"y": 3.4550000000000001, "x": 1301}, {"y": 3.1190000000000002, "x": 1311}, {"y": 4.0129999999999999, "x": 1321}, {"y": 6.3899999999999997, "x": 1331}, {"y": 4.0430000000000001, "x": 1341}, {"y": 8.1809999999999992, "x": 1351}, {"y": 5.3810000000000002, "x": 1361}, {"y": 3.6269999999999998, "x": 1371}, {"y": 3.1680000000000001, "x": 1381}, {"y": 3.3170000000000002, "x": 1391}, {"y": 7.9560000000000004, "x": 1401}, {"y": 5.2530000000000001, "x": 1411}, {"y": 3.5350000000000001, "x": 1421}, {"y": 3.4169999999999998, "x": 1431}, {"y": 3.214, "x": 1441}, {"y": 3.1030000000000002, "x": 1451}, {"y": 11.988, "x": 1461}, {"y": 9.2870000000000008, "x": 1471}, {"y": 8.3369999999999997, "x": 1481}, {"y": 5.556, "x": 1491}, {"y": 3.2629999999999999, "x": 1501}, {"y": 3.8199999999999998, "x": 1511}, {"y": 6.2839999999999998, "x": 1521}, {"y": 5.0970000000000004, "x": 1531}, {"y": 6.5229999999999997, "x": 1541}, {"y": 3.6520000000000001, "x": 1551}, {"y": 4.1749999999999998, "x": 1561}, {"y": 7.577, "x": 1571}, {"y": 6.2969999999999997, "x": 1581}, {"y": 3.677, "x": 1591}, {"y": 4.327, "x": 1601}, {"y": 3.6240000000000001, "x": 1611}, {"y": 3.5779999999999998, "x": 1621}, {"y": 3.1899999999999999, "x": 1631}, {"y": 11.692, "x": 1641}, {"y": 8.5489999999999995, "x": 1651}, {"y": 5.9269999999999996, "x": 1661}, {"y": 3.569, "x": 1671}, {"y": 3.149, "x": 1681}, {"y": 3.2839999999999998, "x": 1691}, {"y": 6.1619999999999999, "x": 1701}, {"y": 3.8759999999999999, "x": 1711}, {"y": 4.3010000000000002, "x": 1721}, {"y": 5.1959999999999997, "x": 1731}, {"y": 4.3869999999999996, "x": 1741}, {"y": 8.8460000000000001, "x": 1751}, {"y": 8.9779999999999998, "x": 1761}, {"y": 5.9580000000000002, "x": 1771}, {"y": 4.4829999999999997, "x": 1781}, {"y": 4.1989999999999998, "x": 1791}, {"y": 5.1630000000000003, "x": 1801}, {"y": 6.774, "x": 1811}, {"y": 7.8570000000000002, "x": 1821}, {"y": 3.6659999999999999, "x": 1831}, {"y": 7.0990000000000002, "x": 1841}, {"y": 3.423, "x": 1851}, {"y": 3.4780000000000002, "x": 1861}, {"y": 5.7400000000000002, "x": 1871}, {"y": 4.726, "x": 1881}, {"y": 4.3490000000000002, "x": 1891}, {"y": 7.8979999999999997, "x": 1901}, {"y": 3.915, "x": 1911}, {"y": 3.0489999999999999, "x": 1921}, {"y": 3.3130000000000002, "x": 1931}, {"y": 8.8100000000000005, "x": 1941}, {"y": 5.133, "x": 1951}, {"y": 4.3330000000000002, "x": 1961}, {"y": 4.5970000000000004, "x": 1971}, {"y": 6.3769999999999998, "x": 1981}, {"y": 6.0220000000000002, "x": 1991}, {"y": 3.8570000000000002, "x": 2001}, {"y": 6.1500000000000004, "x": 2011}, {"y": 4.2729999999999997, "x": 2021}, {"y": 6.3490000000000002, "x": 2031}, {"y": 4.8319999999999999, "x": 2041}, {"y": 5.1630000000000003, "x": 2051}, {"y": 6.4139999999999997, "x": 2061}, {"y": 4.9240000000000004, "x": 2071}], "key": "read maximum latency(ms)", "yAxis": "1"}, {"type": "line", "values": [{"y": 0.0, "x": 0}, {"y": 0.0, "x": 10}, {"y": 0.0, "x": 20}, {"y": 0.0, "x": 30}, {"y": 0.0, "x": 40}, {"y": 0.0, "x": 50}, {"y": 0.0, "x": 60}, {"y": 0.0, "x": 70}, {"y": 0.0, "x": 80}, {"y": 0.0, "x": 90}, {"y": 0.0, "x": 100}, {"y": 0.0, "x": 110}, {"y": 0.0, "x": 120}, {"y": 0.0, "x": 130}, {"y": 0.0, "x": 140}, {"y": 0.0, "x": 150}, {"y": 0.0, "x": 160}, {"y": 0.0, "x": 170}, {"y": 0.0, "x": 180}, {"y": 0.0, "x": 190}, {"y": 0.0, "x": 200}, {"y": 0.0, "x": 210}, {"y": 0.0, "x": 220}, {"y": 0.0, "x": 230}, {"y": 0.0, "x": 240}, {"y": 0.0, "x": 250}, {"y": 0.0, "x": 260}, {"y": 0.0, "x": 270}, {"y": 0.0, "x": 280}, {"y": 0.0, "x": 290}, {"y": 0.0, "x": 300}, {"y": 0.0, "x": 310}, {"y": 0.0, "x": 320}, {"y": 0.0, "x": 330}, {"y": 0.0, "x": 340}, {"y": 0.0, "x": 350}, {"y": 0.0, "x": 360}, {"y": 0.0, "x": 370}, {"y": 0.0, "x": 380}, {"y": 0.0, "x": 390}, {"y": 0.0, "x": 400}, {"y": 0.0, "x": 410}, {"y": 0.0, "x": 420}, {"y": 0.0, "x": 430}, {"y": 0.0, "x": 440}, {"y": 0.0, "x": 450}, {"y": 0.0, "x": 460}, {"y": 0.0, "x": 470}, {"y": 0.0, "x": 480}, {"y": 0.0, "x": 490}, {"y": 0.0, "x": 500}, {"y": 0.0, "x": 510}, {"y": 0.0, "x": 520}, {"y": 0.0, "x": 530}, {"y": 0.0, "x": 540}, {"y": 0.0, "x": 550}, {"y": 0.0, "x": 560}, {"y": 0.0, "x": 570}, {"y": 0.0, "x": 580}, {"y": 0.0, "x": 590}, {"y": 0.0, "x": 600}, {"y": 0.0, "x": 610}, {"y": 0.0, "x": 620}, {"y": 0.0, "x": 630}, {"y": 0.0, "x": 640}, {"y": 0.0, "x": 650}, {"y": 0.0, "x": 660}, {"y": 0.0, "x": 670}, {"y": 0.0, "x": 680}, {"y": 0.0, "x": 690}, {"y": 0.0, "x": 700}, {"y": 0.0, "x": 710}, {"y": 0.0, "x": 720}, {"y": 0.0, "x": 730}, {"y": 0.0, "x": 741}, {"y": 0.0, "x": 751}, {"y": 0.0, "x": 761}, {"y": 0.0, "x": 771}, {"y": 0.0, "x": 781}, {"y": 0.0, "x": 791}, {"y": 0.0, "x": 801}, {"y": 0.0, "x": 811}, {"y": 0.0, "x": 821}, {"y": 0.0, "x": 831}, {"y": 0.0, "x": 841}, {"y": 0.0, "x": 851}, {"y": 0.0, "x": 861}, {"y": 0.0, "x": 871}, {"y": 0.017999999999999999, "x": 881}, {"y": 0.014, "x": 891}, {"y": 0.012999999999999999, "x": 901}, {"y": 0.012999999999999999, "x": 911}, {"y": 0.012999999999999999, "x": 921}, {"y": 0.012, "x": 931}, {"y": 0.012, "x": 941}, {"y": 0.010999999999999999, "x": 951}, {"y": 0.012, "x": 961}, {"y": 0.010999999999999999, "x": 971}, {"y": 0.012, "x": 981}, {"y": 0.012, "x": 991}, {"y": 0.012, "x": 1001}, {"y": 0.012, "x": 1011}, {"y": 0.012, "x": 1021}, {"y": 0.012, "x": 1031}, {"y": 0.01, "x": 1041}, {"y": 0.010999999999999999, "x": 1051}, {"y": 0.010999999999999999, "x": 1061}, {"y": 0.010999999999999999, "x": 1071}, {"y": 0.010999999999999999, "x": 1081}, {"y": 0.012, "x": 1091}, {"y": 0.01, "x": 1101}, {"y": 0.010999999999999999, "x": 1111}, {"y": 0.010999999999999999, "x": 1121}, {"y": 0.012999999999999999, "x": 1131}, {"y": 0.012, "x": 1141}, {"y": 0.012999999999999999, "x": 1151}, {"y": 0.012, "x": 1161}, {"y": 0.010999999999999999, "x": 1171}, {"y": 0.012999999999999999, "x": 1181}, {"y": 0.012, "x": 1191}, {"y": 0.012, "x": 1201}, {"y": 0.012999999999999999, "x": 1211}, {"y": 0.012, "x": 1221}, {"y": 0.012, "x": 1231}, {"y": 0.010999999999999999, "x": 1241}, {"y": 0.012, "x": 1251}, {"y": 0.012999999999999999, "x": 1261}, {"y": 0.012, "x": 1271}, {"y": 0.012999999999999999, "x": 1281}, {"y": 0.010999999999999999, "x": 1291}, {"y": 0.012999999999999999, "x": 1301}, {"y": 0.012, "x": 1311}, {"y": 0.010999999999999999, "x": 1321}, {"y": 0.012, "x": 1331}, {"y": 0.010999999999999999, "x": 1341}, {"y": 0.012999999999999999, "x": 1351}, {"y": 0.012, "x": 1361}, {"y": 0.012, "x": 1371}, {"y": 0.012, "x": 1381}, {"y": 0.012, "x": 1391}, {"y": 0.012, "x": 1401}, {"y": 0.012999999999999999, "x": 1411}, {"y": 0.010999999999999999, "x": 1421}, {"y": 0.010999999999999999, "x": 1431}, {"y": 0.012999999999999999, "x": 1441}, {"y": 0.012999999999999999, "x": 1451}, {"y": 0.012, "x": 1461}, {"y": 0.010999999999999999, "x": 1471}, {"y": 0.012999999999999999, "x": 1481}, {"y": 0.012, "x": 1491}, {"y": 0.012, "x": 1501}, {"y": 0.012, "x": 1511}, {"y": 0.012999999999999999, "x": 1521}, {"y": 0.012, "x": 1531}, {"y": 0.012, "x": 1541}, {"y": 0.01, "x": 1551}, {"y": 0.010999999999999999, "x": 1561}, {"y": 0.012, "x": 1571}, {"y": 0.010999999999999999, "x": 1581}, {"y": 0.012, "x": 1591}, {"y": 0.012, "x": 1601}, {"y": 0.012999999999999999, "x": 1611}, {"y": 0.012, "x": 1621}, {"y": 0.012, "x": 1631}, {"y": 0.012999999999999999, "x": 1641}, {"y": 0.01, "x": 1651}, {"y": 0.010999999999999999, "x": 1661}, {"y": 0.01, "x": 1671}, {"y": 0.01, "x": 1681}, {"y": 0.010999999999999999, "x": 1691}, {"y": 0.012999999999999999, "x": 1701}, {"y": 0.012999999999999999, "x": 1711}, {"y": 0.012, "x": 1721}, {"y": 0.012, "x": 1731}, {"y": 0.012999999999999999, "x": 1741}, {"y": 0.010999999999999999, "x": 1751}, {"y": 0.012999999999999999, "x": 1761}, {"y": 0.012, "x": 1771}, {"y": 0.010999999999999999, "x": 1781}, {"y": 0.010999999999999999, "x": 1791}, {"y": 0.010999999999999999, "x": 1801}, {"y": 0.010999999999999999, "x": 1811}, {"y": 0.01, "x": 1821}, {"y": 0.012, "x": 1831}, {"y": 0.010999999999999999, "x": 1841}, {"y": 0.012999999999999999, "x": 1851}, {"y": 0.012, "x": 1861}, {"y": 0.01, "x": 1871}, {"y": 0.012, "x": 1881}, {"y": 0.012999999999999999, "x": 1891}, {"y": 0.010999999999999999, "x": 1901}, {"y": 0.010999999999999999, "x": 1911}, {"y": 0.0089999999999999993, "x": 1921}, {"y": 0.012, "x": 1931}, {"y": 0.012, "x": 1941}, {"y": 0.012999999999999999, "x": 1951}, {"y": 0.012999999999999999, "x": 1961}, {"y": 0.012999999999999999, "x": 1971}, {"y": 0.012, "x": 1981}, {"y": 0.012, "x": 1991}, {"y": 0.010999999999999999, "x": 2001}, {"y": 0.010999999999999999, "x": 2011}, {"y": 0.012, "x": 2021}, {"y": 0.012999999999999999, "x": 2031}, {"y": 0.012, "x": 2041}, {"y": 0.012, "x": 2051}, {"y": 0.012999999999999999, "x": 2061}, {"y": 0.010999999999999999, "x": 2071}], "key": "read minimum latency(ms)", "yAxis": "1"}, {"type": "line", "values": [{"y": 0.0, "x": 0}, {"y": 0.0, "x": 10}, {"y": 0.0, "x": 20}, {"y": 0.0, "x": 30}, {"y": 0.0, "x": 40}, {"y": 0.0, "x": 50}, {"y": 0.0, "x": 60}, {"y": 0.0, "x": 70}, {"y": 0.0, "x": 80}, {"y": 0.0, "x": 90}, {"y": 0.0, "x": 100}, {"y": 0.0, "x": 110}, {"y": 0.0, "x": 120}, {"y": 0.0, "x": 130}, {"y": 0.0, "x": 140}, {"y": 0.0, "x": 150}, {"y": 0.0, "x": 160}, {"y": 0.0, "x": 170}, {"y": 0.0, "x": 180}, {"y": 0.0, "x": 190}, {"y": 0.0, "x": 200}, {"y": 0.0, "x": 210}, {"y": 0.0, "x": 220}, {"y": 0.0, "x": 230}, {"y": 0.0, "x": 240}, {"y": 0.0, "x": 250}, {"y": 0.0, "x": 260}, {"y": 0.0, "x": 270}, {"y": 0.0, "x": 280}, {"y": 0.0, "x": 290}, {"y": 0.0, "x": 300}, {"y": 0.0, "x": 310}, {"y": 0.0, "x": 320}, {"y": 0.0, "x": 330}, {"y": 0.0, "x": 340}, {"y": 0.0, "x": 350}, {"y": 0.0, "x": 360}, {"y": 0.0, "x": 370}, {"y": 0.0, "x": 380}, {"y": 0.0, "x": 390}, {"y": 0.0, "x": 400}, {"y": 0.0, "x": 410}, {"y": 0.0, "x": 420}, {"y": 0.0, "x": 430}, {"y": 0.0, "x": 440}, {"y": 0.0, "x": 450}, {"y": 0.0, "x": 460}, {"y": 0.0, "x": 470}, {"y": 0.0, "x": 480}, {"y": 0.0, "x": 490}, {"y": 0.0, "x": 500}, {"y": 0.0, "x": 510}, {"y": 0.0, "x": 520}, {"y": 0.0, "x": 530}, {"y": 0.0, "x": 540}, {"y": 0.0, "x": 550}, {"y": 0.0, "x": 560}, {"y": 0.0, "x": 570}, {"y": 0.0, "x": 580}, {"y": 0.0, "x": 590}, {"y": 0.0, "x": 600}, {"y": 0.0, "x": 610}, {"y": 0.0, "x": 620}, {"y": 0.0, "x": 630}, {"y": 0.0, "x": 640}, {"y": 0.0, "x": 650}, {"y": 0.0, "x": 660}, {"y": 0.0, "x": 670}, {"y": 0.0, "x": 680}, {"y": 0.0, "x": 690}, {"y": 0.0, "x": 700}, {"y": 0.0, "x": 710}, {"y": 0.0, "x": 720}, {"y": 0.0, "x": 730}, {"y": 0.0, "x": 741}, {"y": 0.0, "x": 751}, {"y": 0.0, "x": 761}, {"y": 0.0, "x": 771}, {"y": 0.0, "x": 781}, {"y": 0.0, "x": 791}, {"y": 0.0, "x": 801}, {"y": 0.0, "x": 811}, {"y": 0.0, "x": 821}, {"y": 0.0, "x": 831}, {"y": 0.0, "x": 841}, {"y": 0.0, "x": 851}, {"y": 0.0, "x": 861}, {"y": 0.0, "x": 871}, {"y": 3.605, "x": 881}, {"y": 19.199999999999999, "x": 891}, {"y": 21.533000000000001, "x": 901}, {"y": 23.922999999999998, "x": 911}, {"y": 25.114000000000001, "x": 921}, {"y": 26.917000000000002, "x": 931}, {"y": 41.347000000000001, "x": 941}, {"y": 111.246, "x": 951}, {"y": 157.40000000000001, "x": 961}, {"y": 255.22900000000001, "x": 971}, {"y": 283.75900000000001, "x": 981}, {"y": 305.666, "x": 991}, {"y": 328.60899999999998, "x": 1001}, {"y": 355.88400000000001, "x": 1011}, {"y": 371.68799999999999, "x": 1021}, {"y": 373.90800000000002, "x": 1031}, {"y": 363.649, "x": 1041}, {"y": 384.64400000000001, "x": 1051}, {"y": 378.98599999999999, "x": 1061}, {"y": 379.32499999999999, "x": 1071}, {"y": 378.90899999999999, "x": 1081}, {"y": 386.42399999999998, "x": 1091}, {"y": 376.96499999999997, "x": 1101}, {"y": 387.32999999999998, "x": 1111}, {"y": 382.38600000000002, "x": 1121}, {"y": 387.464, "x": 1131}, {"y": 392.87299999999999, "x": 1141}, {"y": 386.84800000000001, "x": 1151}, {"y": 383.71199999999999, "x": 1161}, {"y": 391.548, "x": 1171}, {"y": 385.15499999999997, "x": 1181}, {"y": 375.13900000000001, "x": 1191}, {"y": 382.947, "x": 1201}, {"y": 384.32799999999997, "x": 1211}, {"y": 382.19600000000003, "x": 1221}, {"y": 374.32499999999999, "x": 1231}, {"y": 384.601, "x": 1241}, {"y": 380.40800000000002, "x": 1251}, {"y": 385.35199999999998, "x": 1261}, {"y": 370.44, "x": 1271}, {"y": 388.63400000000001, "x": 1281}, {"y": 386.09100000000001, "x": 1291}, {"y": 379.709, "x": 1301}, {"y": 391.226, "x": 1311}, {"y": 373.60300000000001, "x": 1321}, {"y": 370.94099999999997, "x": 1331}, {"y": 389.06799999999998, "x": 1341}, {"y": 379.78199999999998, "x": 1351}, {"y": 380.00700000000001, "x": 1361}, {"y": 376.95800000000003, "x": 1371}, {"y": 372.17099999999999, "x": 1381}, {"y": 380.36200000000002, "x": 1391}, {"y": 369.084, "x": 1401}, {"y": 381.61799999999999, "x": 1411}, {"y": 384.11700000000002, "x": 1421}, {"y": 386.59800000000001, "x": 1431}, {"y": 383.54700000000003, "x": 1441}, {"y": 393.863, "x": 1451}, {"y": 378.93700000000001, "x": 1461}, {"y": 385.12400000000002, "x": 1471}, {"y": 383.40800000000002, "x": 1481}, {"y": 384.255, "x": 1491}, {"y": 389.76799999999997, "x": 1501}, {"y": 384.13299999999998, "x": 1511}, {"y": 377.74299999999999, "x": 1521}, {"y": 372.42700000000002, "x": 1531}, {"y": 380.21600000000001, "x": 1541}, {"y": 378.96899999999999, "x": 1551}, {"y": 386.17200000000003, "x": 1561}, {"y": 383.96300000000002, "x": 1571}, {"y": 378.827, "x": 1581}, {"y": 384.07600000000002, "x": 1591}, {"y": 399.54399999999998, "x": 1601}, {"y": 393.96499999999997, "x": 1611}, {"y": 380.92599999999999, "x": 1621}, {"y": 383.10700000000003, "x": 1631}, {"y": 381.74700000000001, "x": 1641}, {"y": 389.72399999999999, "x": 1651}, {"y": 380.12099999999998, "x": 1661}, {"y": 388.584, "x": 1671}, {"y": 393.76400000000001, "x": 1681}, {"y": 387.93700000000001, "x": 1691}, {"y": 385.62599999999998, "x": 1701}, {"y": 382.89999999999998, "x": 1711}, {"y": 382.86700000000002, "x": 1721}, {"y": 375.23700000000002, "x": 1731}, {"y": 385.97300000000001, "x": 1741}, {"y": 397.327, "x": 1751}, {"y": 379.33999999999997, "x": 1761}, {"y": 383.00999999999999, "x": 1771}, {"y": 380.03500000000003, "x": 1781}, {"y": 393.18900000000002, "x": 1791}, {"y": 383.64499999999998, "x": 1801}, {"y": 379.05399999999997, "x": 1811}, {"y": 373.01400000000001, "x": 1821}, {"y": 377.68799999999999, "x": 1831}, {"y": 367.43900000000002, "x": 1841}, {"y": 376.536, "x": 1851}, {"y": 391.65699999999998, "x": 1861}, {"y": 376.45100000000002, "x": 1871}, {"y": 383.62400000000002, "x": 1881}, {"y": 384.54000000000002, "x": 1891}, {"y": 396.17000000000002, "x": 1901}, {"y": 389.86200000000002, "x": 1911}, {"y": 379.23500000000001, "x": 1921}, {"y": 390.33699999999999, "x": 1931}, {"y": 390.75400000000002, "x": 1941}, {"y": 387.67399999999998, "x": 1951}, {"y": 370.26600000000002, "x": 1961}, {"y": 383.75999999999999, "x": 1971}, {"y": 376.649, "x": 1981}, {"y": 398.45400000000001, "x": 1991}, {"y": 385.15499999999997, "x": 2001}, {"y": 388.87400000000002, "x": 2011}, {"y": 389.06400000000002, "x": 2021}, {"y": 386.21499999999997, "x": 2031}, {"y": 386.79199999999997, "x": 2041}, {"y": 383.74599999999998, "x": 2051}, {"y": 381.88, "x": 2061}, {"y": 387.495, "x": 2071}], "key": "read ops per second (thousands)", "yAxis": "2"}]; -</script> - -</body> diff --git a/src/docs/spell.ok b/src/docs/spell.ok index 6af9a19fbf2..f333a8fff58 100644 --- a/src/docs/spell.ok +++ b/src/docs/spell.ok @@ -71,7 +71,6 @@ Seward's SiH TXT URIs -Vv WiredTiger WiredTiger's WiredTigerCheckpoint @@ -234,6 +233,7 @@ io ip je jemalloc +jitter jni jrx json @@ -325,6 +325,8 @@ oltp oob optype os +outlier +outliers ovfl pareto pcoll @@ -350,6 +352,7 @@ putValue putValueString py qnx +rVv rdbms rdlock readlock diff --git a/src/docs/wtperf.dox b/src/docs/wtperf.dox index 2cadc6ed05b..f8d0a9bf5f7 100644 --- a/src/docs/wtperf.dox +++ b/src/docs/wtperf.dox @@ -100,8 +100,9 @@ Enabling monitoring causes \c wtperf to create a file \c monitor in the database home directory (or another directory as specified using the \c -m option to \c wtperf). -A visualization tool to see the performance over the course of the \c wtperf -run can be found in \c tools/wtperf_stats/wtperf_stats.py. +The same visualization tool, \c wtstats, can be used to view a combined +chart with both the \c monitor output and the statistics logging output +at the same time. The following example shows how to run the \c medium-btree.wtperf configuration with monitoring enabled, and then generate a graph. @@ -123,25 +124,17 @@ bench/wtperf/wtperf \ -O bench/wtperf/runners/medium-btree.wtperf # Use the visualization tool to create HTML graph output; the output file is -# named wtperf_stats.html. -python tools/wtperf_stats/wtperf_stats.py WTPERF_RUN/monitor +# named wtstats.html. +python tools/wtstats/wtstats.py WTPERF_RUN/monitor + +# Possible alternatives if statistics logging also enabled: +# python tools/wtstats/wtstats.py WTPERF_RUN/monitor WTPERF_RUN/WiredTigerStat* +# python tools/wtstats/wtstats.py WTPERF_RUN @endcode -The python command creates a file named \c wtperf_stats.html in the current +The python command creates a file named \c wtstats.html in the current working directory. You can open the generated HTML document in your browser -and see the generated statistics. The keys in the graph are clickable, -including the graph included on this page. Double clicking on one -of the keys will show only that item. Single clicking will -enable or disable a particular item. - -The script has a few optional arguments: <code>-o file</code> or -<code>--output file</code> to redirect the HTML output to another -filename, and \c --abstime to display time on the X-axis in absolute -time. - -The following is example HTML output: - -\htmlinclude wtperf-sample.html +and see the generated statistics. @section config Wtperf configuration options diff --git a/src/docs/wtstats.dox b/src/docs/wtstats.dox index a07dfd4ca8f..c4218049a4a 100644 --- a/src/docs/wtstats.dox +++ b/src/docs/wtstats.dox @@ -2,7 +2,7 @@ The WiredTiger distribution includes the \b wtstats tool that can be used to examine information generated using statistics logging (see -@ref statistics_log). +@ref statistics_log) and wtperf monitoring output (see @ref wtperf). After running an application with statistics logging configured, the statistics output files will be in the database home directory. By diff --git a/src/evict/evict_file.c b/src/evict/evict_file.c index 5aa85872a3b..910aef070ca 100644 --- a/src/evict/evict_file.c +++ b/src/evict/evict_file.c @@ -36,8 +36,8 @@ __wt_evict_file(WT_SESSION_IMPL *session, int syncop) /* Walk the tree, discarding pages. */ next_ref = NULL; - WT_ERR(__wt_tree_walk( - session, &next_ref, WT_READ_CACHE | WT_READ_NO_EVICT)); + WT_ERR(__wt_tree_walk(session, &next_ref, NULL, + WT_READ_CACHE | WT_READ_NO_EVICT)); while ((ref = next_ref) != NULL) { page = ref->page; @@ -73,8 +73,8 @@ __wt_evict_file(WT_SESSION_IMPL *session, int syncop) * the reconciliation, the next walk call could miss a page in * the tree. */ - WT_ERR(__wt_tree_walk( - session, &next_ref, WT_READ_CACHE | WT_READ_NO_EVICT)); + WT_ERR(__wt_tree_walk(session, &next_ref, NULL, + WT_READ_CACHE | WT_READ_NO_EVICT)); switch (syncop) { case WT_SYNC_CLOSE: diff --git a/src/evict/evict_lru.c b/src/evict/evict_lru.c index c6b962f9f5d..83a9aa5c8c5 100644 --- a/src/evict/evict_lru.c +++ b/src/evict/evict_lru.c @@ -225,6 +225,46 @@ err: WT_PANIC_MSG(session, ret, "cache eviction server error"); } /* + * __evict_workers_resize -- + * Resize the array of eviction workers (as needed after a reconfigure). + * We don't do this during the reconfigure because the eviction server + * thread owns these structures. + */ +static int +__evict_workers_resize(WT_SESSION_IMPL *session) +{ + WT_CONNECTION_IMPL *conn; + WT_DECL_RET; + WT_EVICT_WORKER *workers; + size_t alloc; + u_int i; + + conn = S2C(session); + + alloc = conn->evict_workers_alloc * sizeof(*workers); + WT_RET(__wt_realloc(session, &alloc, + conn->evict_workers_max * sizeof(*workers), &conn->evict_workctx)); + workers = conn->evict_workctx; + + for (i = conn->evict_workers_alloc; i < conn->evict_workers_max; i++) { + WT_ERR(__wt_open_internal_session(conn, + "eviction-worker", 0, 0, &workers[i].session)); + workers[i].id = i; + F_SET(workers[i].session, WT_SESSION_CAN_WAIT); + + if (i < conn->evict_workers_min) { + ++conn->evict_workers; + F_SET(&workers[i], WT_EVICT_WORKER_RUN); + WT_ERR(__wt_thread_create(workers[i].session, + &workers[i].tid, __evict_worker, &workers[i])); + } + } + +err: conn->evict_workers_alloc = conn->evict_workers_max; + return (ret); +} + +/* * __wt_evict_create -- * Start the eviction server thread. */ @@ -232,8 +272,6 @@ int __wt_evict_create(WT_SESSION_IMPL *session) { WT_CONNECTION_IMPL *conn; - WT_EVICT_WORKER *workers; - u_int i; conn = S2C(session); @@ -253,27 +291,6 @@ __wt_evict_create(WT_SESSION_IMPL *session) if (conn->evict_workers_max == 0) F_SET(session, WT_SESSION_CAN_WAIT); - if (conn->evict_workers_max > 0) { - WT_RET(__wt_calloc_def( - session, conn->evict_workers_max, &workers)); - conn->evict_workctx = workers; - - for (i = 0; i < conn->evict_workers_max; i++) { - WT_RET(__wt_open_internal_session(conn, - "eviction-worker", 0, 0, &workers[i].session)); - workers[i].id = i; - F_SET(workers[i].session, WT_SESSION_CAN_WAIT); - - if (i < conn->evict_workers_min) { - ++conn->evict_workers; - F_SET(&workers[i], WT_EVICT_WORKER_RUN); - WT_RET(__wt_thread_create( - workers[i].session, &workers[i].tid, - __evict_worker, &workers[i])); - } - } - } - /* * Start the primary eviction server thread after the worker threads * have started to avoid it starting additional worker threads before @@ -312,11 +329,12 @@ __wt_evict_destroy(WT_SESSION_IMPL *session) WT_TRET(__wt_cond_signal(session, cache->evict_waiter_cond)); WT_TRET(__wt_thread_join(session, workers[i].tid)); } - /* Handle shutdown when cleaning up after a failed open */ + /* Handle shutdown when cleaning up after a failed open. */ if (conn->evict_workctx != NULL) { - for (i = 0; i < conn->evict_workers_max; i++) { + for (i = 0; i < conn->evict_workers_alloc; i++) { wt_session = &conn->evict_workctx[i].session->iface; - WT_TRET(wt_session->close(wt_session, NULL)); + if (wt_session != NULL) + WT_TRET(wt_session->close(wt_session, NULL)); } __wt_free(session, conn->evict_workctx); } @@ -398,7 +416,7 @@ __evict_has_work(WT_SESSION_IMPL *session, uint32_t *flagsp) * target or the dirty target. */ bytes_inuse = __wt_cache_bytes_inuse(cache); - dirty_inuse = cache->bytes_dirty; + dirty_inuse = __wt_cache_dirty_inuse(cache); bytes_max = conn->cache_size; /* Check to see if the eviction server should run. */ @@ -435,9 +453,9 @@ __evict_pass(WT_SESSION_IMPL *session) WT_CACHE *cache; WT_CONNECTION_IMPL *conn; WT_EVICT_WORKER *worker; - int loop; + uint64_t pages_evicted; uint32_t flags; - uint64_t bytes_inuse, dirty_target_size, pages_evicted, target_size; + int loop; conn = S2C(session); cache = conn->cache; @@ -469,16 +487,13 @@ __evict_pass(WT_SESSION_IMPL *session) * Start a worker if we have capacity and we haven't reached * the eviction targets. */ - bytes_inuse = __wt_cache_bytes_inuse(cache); - target_size = (conn->cache_size * cache->eviction_target) / 100; - dirty_target_size = - (conn->cache_size * cache->eviction_dirty_target) / 100; - if ((bytes_inuse > target_size || - cache->bytes_dirty > dirty_target_size) && + if (LF_ISSET(WT_EVICT_PASS_ALL | WT_EVICT_PASS_DIRTY) && conn->evict_workers < conn->evict_workers_max) { WT_RET(__wt_verbose(session, WT_VERB_EVICTSERVER, "Starting evict worker: %"PRIu32"\n", conn->evict_workers)); + if (conn->evict_workers >= conn->evict_workers_alloc) + WT_RET(__evict_workers_resize(session)); worker = &conn->evict_workctx[conn->evict_workers++]; F_SET(worker, WT_EVICT_WORKER_RUN); WT_RET(__wt_thread_create(session, @@ -488,7 +503,7 @@ __evict_pass(WT_SESSION_IMPL *session) WT_RET(__wt_verbose(session, WT_VERB_EVICTSERVER, "Eviction pass with: Max: %" PRIu64 " In use: %" PRIu64 " Dirty: %" PRIu64, - conn->cache_size, bytes_inuse, cache->bytes_dirty)); + conn->cache_size, cache->bytes_inmem, cache->bytes_dirty)); WT_RET(__evict_lru_walk(session, flags)); WT_RET(__evict_server_work(session)); @@ -839,7 +854,7 @@ __evict_walk(WT_SESSION_IMPL *session, uint32_t flags) WT_CONNECTION_IMPL *conn; WT_DATA_HANDLE *dhandle; WT_DECL_RET; - u_int max_entries, old_slot, retries, slot; + u_int max_entries, prev_slot, retries, slot, start_slot, spins; int incr, dhandle_locked; WT_DECL_SPINLOCK_ID(id); @@ -869,7 +884,7 @@ __evict_walk(WT_SESSION_IMPL *session, uint32_t flags) * Set the starting slot in the queue and the maximum pages added * per walk. */ - slot = cache->evict_entries; + start_slot = slot = cache->evict_entries; max_entries = slot + WT_EVICT_WALK_INCR; retry: while (slot < max_entries && ret == 0) { @@ -885,8 +900,16 @@ retry: while (slot < max_entries && ret == 0) { * reference count to keep it alive while we sweep. */ if (!dhandle_locked) { - if ((ret = __wt_spin_trylock( - session, &conn->dhandle_lock, &id)) != 0) + for (spins = 0; (ret = __wt_spin_trylock( + session, &conn->dhandle_lock, &id)) == EBUSY && + !F_ISSET(cache, WT_EVICT_CLEAR_WALKS); + spins++) { + if (spins < 1000) + __wt_yield(); + else + __wt_sleep(0, 1000); + } + if (ret != 0) break; dhandle_locked = 1; } @@ -926,10 +949,10 @@ retry: while (slot < max_entries && ret == 0) { continue; /* - * Also skip files that are configured to stick in cache until - * we get aggressive. + * Also skip files that are checkpointing or configured to + * stick in cache until we get aggressive. */ - if (btree->evict_priority != 0 && + if ((btree->checkpointing || btree->evict_priority != 0) && !LF_ISSET(WT_EVICT_PASS_AGGRESSIVE)) continue; @@ -942,7 +965,7 @@ retry: while (slot < max_entries && ret == 0) { btree->evict_walk_skips++ < btree->evict_walk_period) continue; btree->evict_walk_skips = 0; - old_slot = slot; + prev_slot = slot; (void)WT_ATOMIC_ADD4(dhandle->session_inuse, 1); incr = 1; @@ -964,15 +987,14 @@ retry: while (slot < max_entries && ret == 0) { __wt_spin_unlock(session, &cache->evict_walk_lock); /* - * If we didn't find enough candidates in the file, skip it - * next time. + * If we didn't find any candidates in the file, skip it next + * time. */ - if (slot >= old_slot + WT_EVICT_WALK_PER_FILE || - slot >= max_entries) - btree->evict_walk_period = 0; - else + if (slot == prev_slot) btree->evict_walk_period = WT_MIN( - WT_MAX(1, 2 * btree->evict_walk_period), 1000); + WT_MAX(1, 2 * btree->evict_walk_period), 100); + else + btree->evict_walk_period = 0; } if (incr) { @@ -988,15 +1010,16 @@ retry: while (slot < max_entries && ret == 0) { /* * Walk the list of files a few times if we don't find enough pages. - * Try two passes through all the files, then only keep going if we - * are finding more candidates. Take care not to skip files on - * subsequent passes. + * Try two passes through all the files, give up when we have some + * candidates and we aren't finding more. Take care not to skip files + * on subsequent passes. */ if (!F_ISSET(cache, WT_EVICT_CLEAR_WALKS) && ret == 0 && slot < max_entries && (retries < 2 || - (!LF_ISSET(WT_EVICT_PASS_WOULD_BLOCK) && - retries < 10 && slot > 0))) { + (!LF_ISSET(WT_EVICT_PASS_WOULD_BLOCK) && retries < 10 && + (slot == cache->evict_entries || slot > start_slot)))) { cache->evict_file_next = NULL; + start_slot = slot; ++retries; goto retry; } @@ -1064,9 +1087,10 @@ __evict_walk_file(WT_SESSION_IMPL *session, u_int *slotp, uint32_t flags) * Get some more eviction candidate pages. */ for (evict = start, pages_walked = 0, internal_pages = restarts = 0; - evict < end && (ret == 0 || ret == WT_NOTFOUND); - ret = __wt_tree_walk(session, &btree->evict_ref, walk_flags), - ++pages_walked) { + evict < end && pages_walked < WT_EVICT_MAX_PER_FILE && + (ret == 0 || ret == WT_NOTFOUND); + ret = __wt_tree_walk( + session, &btree->evict_ref, &pages_walked, walk_flags)) { if (btree->evict_ref == NULL) { /* * Take care with terminating this loop. @@ -1126,12 +1150,8 @@ __evict_walk_file(WT_SESSION_IMPL *session, u_int *slotp, uint32_t flags) continue; } -fast: /* - * If the file is being checkpointed, there's a period of time - * where we can't discard dirty pages because of possible races - * with the checkpointing thread. - */ - if (modified && btree->checkpointing) +fast: /* If the page can't be evicted, give up. */ + if (!__wt_page_can_evict(session, page, 0)) continue; /* @@ -1447,7 +1467,7 @@ __wt_cache_dump(WT_SESSION_IMPL *session) next_walk = NULL; session->dhandle = dhandle; while (__wt_tree_walk(session, - &next_walk, WT_READ_CACHE | WT_READ_NO_WAIT) == 0 && + &next_walk, NULL, WT_READ_CACHE | WT_READ_NO_WAIT) == 0 && next_walk != NULL) { page = next_walk->page; if (page->type == WT_PAGE_COL_INT || diff --git a/src/evict/evict_page.c b/src/evict/evict_page.c index 0cff584f2ab..9ba1af897a4 100644 --- a/src/evict/evict_page.c +++ b/src/evict/evict_page.c @@ -320,13 +320,11 @@ static int __evict_review(WT_SESSION_IMPL *session, WT_REF *ref, int exclusive, int top, int *inmem_splitp, int *istreep) { - WT_BTREE *btree; WT_DECL_RET; WT_PAGE *page; WT_PAGE_MODIFY *mod; uint32_t flags; - btree = S2BT(session); flags = WT_EVICTING; /* @@ -370,19 +368,8 @@ __evict_review(WT_SESSION_IMPL *session, WT_REF *ref, } /* - * If the tree was deepened, there's a requirement that newly created - * internal pages not be evicted until all threads are known to have - * exited the original page index array, because evicting an internal - * page discards its WT_REF array, and a thread traversing the original - * page index array might see an freed WT_REF. During the split we set - * a transaction value, once that's globally visible, we know we can - * evict the created page. - */ - if (!exclusive && mod != NULL && WT_PAGE_IS_INTERNAL(page) && - !__wt_txn_visible_all(session, mod->mod_split_txn)) - return (EBUSY); - - /* + * Check whether the page can be evicted. + * * If the file is being checkpointed, we can't evict dirty pages: * if we write a page and free the previous version of the page, that * previous version might be referenced by an internal page already @@ -402,13 +389,8 @@ __evict_review(WT_SESSION_IMPL *session, WT_REF *ref, * internal page acquires hazard pointers on child pages it reads, and * is blocked by the exclusive lock. */ - if (mod != NULL && btree->checkpointing && - (__wt_page_is_modified(page) || - F_ISSET(mod, WT_PM_REC_MULTIBLOCK))) { - WT_STAT_FAST_CONN_INCR(session, cache_eviction_checkpoint); - WT_STAT_FAST_DATA_INCR(session, cache_eviction_checkpoint); + if (!exclusive && !__wt_page_can_evict(session, page, 0)) return (EBUSY); - } /* * Check for an append-only workload needing an in-memory split. @@ -448,29 +430,18 @@ __evict_review(WT_SESSION_IMPL *session, WT_REF *ref, * If we have an exclusive lock (we're discarding the tree), assert * there are no updates we cannot read. * - * Otherwise, if the top-level page we're evicting is a leaf page, set - * the update-restore flag, so reconciliation will write blocks it can - * write and create a list of skipped updates for blocks it cannot - * write. This is how forced eviction of huge pages works: we take a - * big page and reconcile it into blocks, some of which we write and - * discard, the rest of which we re-create as smaller in-memory pages, - * (restoring the updates that stopped us from writing the block), and - * inserting the whole mess into the page's parent. + * Otherwise, if the top-level page we're evicting is a leaf page + * marked for forced eviction, set the update-restore flag, so + * reconciliation will write blocks it can write and create a list of + * skipped updates for blocks it cannot write. This is how forced + * eviction of huge pages works: we take a big page and reconcile it + * into blocks, some of which we write and discard, the rest of which + * we re-create as smaller in-memory pages, (restoring the updates that + * stopped us from writing the block), and inserting the whole mess + * into the page's parent. * * Don't set the update-restore flag for internal pages, they don't * have updates that can be saved and restored. - * - * Don't set the update-restore flag for small pages. (If a small - * page were selected by eviction and then modified, and we configure it - * for update-restore, we'll end up splitting one or two pages into the - * parent, which is a waste of effort. If we don't set update-restore, - * eviction will return EBUSY, which makes more sense, the page was just - * modified.) - * - * Don't set the update-restore flag for any page other than the - * top one; only the reconciled top page goes through the split path - * (and child pages are pages we expect to merge into the top page, they - * they are not expected to split). */ if (__wt_page_is_modified(page)) { if (exclusive) diff --git a/src/include/api.h b/src/include/api.h index e610b3b3e1b..70068e32b9b 100644 --- a/src/include/api.h +++ b/src/include/api.h @@ -76,7 +76,7 @@ } \ } \ break; \ -} while (ret == 0) +} while (1) /* End a transactional API call, retry on deadlock. */ #define TXN_API_END(s, ret) TXN_API_END_RETRY(s, ret, 1) diff --git a/src/include/btree.i b/src/include/btree.i index 2896620e503..032178b4755 100644 --- a/src/include/btree.i +++ b/src/include/btree.i @@ -941,6 +941,70 @@ __wt_ref_info(WT_SESSION_IMPL *session, } /* + * __wt_page_can_evict -- + * Check whether a page can be evicted. + */ +static inline int +__wt_page_can_evict(WT_SESSION_IMPL *session, WT_PAGE *page, int check_splits) +{ + WT_BTREE *btree; + WT_PAGE_MODIFY *mod; + + btree = S2BT(session); + mod = page->modify; + + /* Pages that have never been modified can always be evicted. */ + if (mod == NULL) + return (1); + + /* + * If the tree was deepened, there's a requirement that newly created + * internal pages not be evicted until all threads are known to have + * exited the original page index array, because evicting an internal + * page discards its WT_REF array, and a thread traversing the original + * page index array might see an freed WT_REF. During the split we set + * a transaction value, once that's globally visible, we know we can + * evict the created page. + */ + if (WT_PAGE_IS_INTERNAL(page) && + !__wt_txn_visible_all(session, mod->mod_split_txn)) + return (0); + + /* + * If the file is being checkpointed, we can't evict dirty pages: + * if we write a page and free the previous version of the page, that + * previous version might be referenced by an internal page already + * been written in the checkpoint, leaving the checkpoint inconsistent. + */ + if (btree->checkpointing && + (__wt_page_is_modified(page) || + F_ISSET(mod, WT_PM_REC_MULTIBLOCK))) { + WT_STAT_FAST_CONN_INCR(session, cache_eviction_checkpoint); + WT_STAT_FAST_DATA_INCR(session, cache_eviction_checkpoint); + return (0); + } + + /* + * If we aren't (potentially) doing eviction that can restore updates + * and the updates on this page are too recent, give up. + */ + if (page->read_gen != WT_READGEN_OLDEST && + !__wt_txn_visible_all(session, __wt_page_is_modified(page) ? + mod->update_txn : mod->rec_max_txn)) + return (0); + + /* + * If the page was recently split in-memory, don't force it out: we + * hope eviction will find it first. + */ + if (check_splits && + !__wt_txn_visible_all(session, mod->inmem_split_txn)) + return (0); + + return (1); +} + +/* * __wt_page_release_evict -- * Attempt to release and immediately evict a page. */ @@ -1010,10 +1074,9 @@ __wt_page_release(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags) /* * Attempt to evict pages with the special "oldest" read generation. - * * This is set for pages that grow larger than the configured - * memory_page_max setting, and when we are attempting to scan without - * trashing the cache. + * memory_page_max setting, when we see many deleted items, and when we + * are attempting to scan without trashing the cache. * * Skip this if eviction is disabled for this operation or this tree, * or if there is no chance of eviction succeeding for dirty pages due @@ -1021,12 +1084,10 @@ __wt_page_release(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags) * it contains an update that isn't stable. Also skip forced eviction * if we just did an in-memory split. */ - if (LF_ISSET(WT_READ_NO_EVICT) || - page->read_gen != WT_READGEN_OLDEST || + if (page->read_gen != WT_READGEN_OLDEST || + LF_ISSET(WT_READ_NO_EVICT) || F_ISSET(btree, WT_BTREE_NO_EVICTION) || - (__wt_page_is_modified(page) && (btree->checkpointing || - !__wt_txn_visible_all(session, page->modify->first_dirty_txn) || - !__wt_txn_visible_all(session, page->modify->inmem_split_txn)))) + !__wt_page_can_evict(session, page, 1)) return (__wt_hazard_clear(session, page)); WT_RET_BUSY_OK(__wt_page_release_evict(session, ref)); diff --git a/src/include/cache.h b/src/include/cache.h index de6faad608a..84b18082a25 100644 --- a/src/include/cache.h +++ b/src/include/cache.h @@ -13,9 +13,10 @@ #define WT_EVICT_INT_SKEW (1<<20) /* Prefer leaf pages over internal pages by this many increments of the read generation. */ -#define WT_EVICT_WALK_PER_FILE 10 /* Pages to visit per file */ -#define WT_EVICT_WALK_BASE 300 /* Pages tracked across file visits */ -#define WT_EVICT_WALK_INCR 100 /* Pages added each walk */ +#define WT_EVICT_WALK_PER_FILE 10 /* Pages to queue per file */ +#define WT_EVICT_MAX_PER_FILE 100 /* Max pages to visit per file */ +#define WT_EVICT_WALK_BASE 300 /* Pages tracked across file visits */ +#define WT_EVICT_WALK_INCR 100 /* Pages added each walk */ #define WT_EVICT_PASS_AGGRESSIVE 0x01 #define WT_EVICT_PASS_ALL 0x02 @@ -82,6 +83,8 @@ struct __wt_cache { u_int eviction_target; /* Percent to end eviction */ u_int eviction_dirty_target; /* Percent to allow dirty */ + u_int overhead_pct; /* Cache percent adjustment */ + /* * LRU eviction list information. */ diff --git a/src/include/cache.i b/src/include/cache.i index 4bceb5c0d6c..f952f1bf698 100644 --- a/src/include/cache.i +++ b/src/include/cache.i @@ -62,7 +62,32 @@ __wt_cache_pages_inuse(WT_CACHE *cache) static inline uint64_t __wt_cache_bytes_inuse(WT_CACHE *cache) { - return (cache->bytes_inmem); + uint64_t bytes_inuse; + + /* Adjust the cache size to take allocation overhead into account. */ + bytes_inuse = cache->bytes_inmem; + if (cache->overhead_pct != 0) + bytes_inuse += + (bytes_inuse * (uint64_t)cache->overhead_pct) / 100; + + return (bytes_inuse); +} + +/* + * __wt_cache_dirty_inuse -- + * Return the number of dirty bytes in use. + */ +static inline uint64_t +__wt_cache_dirty_inuse(WT_CACHE *cache) +{ + uint64_t dirty_inuse; + + dirty_inuse = cache->bytes_dirty; + if (cache->overhead_pct != 0) + dirty_inuse += + (dirty_inuse * (uint64_t)cache->overhead_pct) / 100; + + return (dirty_inuse); } /* @@ -87,13 +112,9 @@ __wt_eviction_check(WT_SESSION_IMPL *session, int *fullp, int wake) * in a shared cache. */ bytes_inuse = __wt_cache_bytes_inuse(cache); - dirty_inuse = cache->bytes_dirty; + dirty_inuse = __wt_cache_dirty_inuse(cache); bytes_max = conn->cache_size + 1; - /* Adjust the cache size to take allocation overhead into account. */ - if (conn->cache_overhead != 0) - bytes_max -= (bytes_max * (uint64_t)conn->cache_overhead) / 100; - /* Calculate the cache full percentage. */ *fullp = (int)((100 * bytes_inuse) / bytes_max); diff --git a/src/include/config.h b/src/include/config.h index 65757c2ef6d..046f515188c 100644 --- a/src/include/config.h +++ b/src/include/config.h @@ -73,14 +73,15 @@ struct __wt_config_parser_impl { #define WT_CONFIG_ENTRY_session_rename 24 #define WT_CONFIG_ENTRY_session_rollback_transaction 25 #define WT_CONFIG_ENTRY_session_salvage 26 -#define WT_CONFIG_ENTRY_session_truncate 27 -#define WT_CONFIG_ENTRY_session_upgrade 28 -#define WT_CONFIG_ENTRY_session_verify 29 -#define WT_CONFIG_ENTRY_table_meta 30 -#define WT_CONFIG_ENTRY_wiredtiger_open 31 -#define WT_CONFIG_ENTRY_wiredtiger_open_all 32 -#define WT_CONFIG_ENTRY_wiredtiger_open_basecfg 33 -#define WT_CONFIG_ENTRY_wiredtiger_open_usercfg 34 +#define WT_CONFIG_ENTRY_session_strerror 27 +#define WT_CONFIG_ENTRY_session_truncate 28 +#define WT_CONFIG_ENTRY_session_upgrade 29 +#define WT_CONFIG_ENTRY_session_verify 30 +#define WT_CONFIG_ENTRY_table_meta 31 +#define WT_CONFIG_ENTRY_wiredtiger_open 32 +#define WT_CONFIG_ENTRY_wiredtiger_open_all 33 +#define WT_CONFIG_ENTRY_wiredtiger_open_basecfg 34 +#define WT_CONFIG_ENTRY_wiredtiger_open_usercfg 35 /* * configuration section: END * DO NOT EDIT: automatically built by dist/flags.py. diff --git a/src/include/connection.h b/src/include/connection.h index dd97ea50ce9..9cb42ae7c80 100644 --- a/src/include/connection.h +++ b/src/include/connection.h @@ -227,7 +227,6 @@ struct __wt_connection_impl { uint32_t hazard_max; /* Hazard array size */ WT_CACHE *cache; /* Page cache */ - int cache_overhead; /* Cache percent adjustment */ uint64_t cache_size; /* Configured cache size */ WT_TXN_GLOBAL txn_global; /* Global transaction state */ @@ -290,6 +289,7 @@ struct __wt_connection_impl { wt_thread_t evict_tid; /* Eviction server thread ID */ int evict_tid_set; /* Eviction server thread ID set */ + uint32_t evict_workers_alloc;/* Allocated eviction workers */ uint32_t evict_workers_max;/* Max eviction workers */ uint32_t evict_workers_min;/* Min eviction workers */ uint32_t evict_workers; /* Number of eviction workers */ @@ -310,6 +310,7 @@ struct __wt_connection_impl { #define WT_CONN_LOG_ENABLED 0x02 /* Logging is enabled */ #define WT_CONN_LOG_EXISTED 0x04 /* Log files found */ #define WT_CONN_LOG_PREALLOC 0x08 /* Pre-allocation is enabled */ +#define WT_CONN_LOG_RECOVER_ERR 0x10 /* Error if recovery required */ uint32_t log_flags; /* Global logging configuration */ WT_CONDVAR *log_cond; /* Log server wait mutex */ WT_SESSION_IMPL *log_session; /* Log server session */ diff --git a/src/include/extern.h b/src/include/extern.h index 7716336bff1..23bb36623e5 100644 --- a/src/include/extern.h +++ b/src/include/extern.h @@ -157,7 +157,7 @@ extern int __wt_upgrade(WT_SESSION_IMPL *session, const char *cfg[]); extern int __wt_verify(WT_SESSION_IMPL *session, const char *cfg[]); extern int __wt_verify_dsk_image(WT_SESSION_IMPL *session, const char *addr, const WT_PAGE_HEADER *dsk, size_t size, int empty_page_ok); extern int __wt_verify_dsk(WT_SESSION_IMPL *session, const char *addr, WT_ITEM *buf); -extern int __wt_tree_walk(WT_SESSION_IMPL *session, WT_REF **refp, uint32_t flags); +extern int __wt_tree_walk(WT_SESSION_IMPL *session, WT_REF **refp, uint64_t *walkcntp, uint32_t flags); extern int __wt_col_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, uint64_t recno, WT_ITEM *value, WT_UPDATE *upd, int is_remove); extern int __wt_col_search(WT_SESSION_IMPL *session, uint64_t recno, WT_REF *leaf, WT_CURSOR_BTREE *cbt); extern int __wt_row_leaf_keys(WT_SESSION_IMPL *session, WT_PAGE *page); @@ -198,6 +198,7 @@ extern void __wt_conn_config_discard(WT_SESSION_IMPL *session); extern int __wt_ext_config_parser_open(WT_EXTENSION_API *wt_ext, WT_SESSION *wt_session, const char *config, size_t len, WT_CONFIG_PARSER **config_parserp); extern int __wt_ext_config_get(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, WT_CONFIG_ARG *cfg_arg, const char *key, WT_CONFIG_ITEM *cval); extern int __wt_config_upgrade(WT_SESSION_IMPL *session, WT_ITEM *buf); +extern const char *__wt_wiredtiger_error(int error); extern int __wt_collator_config(WT_SESSION_IMPL *session, const char *uri, WT_CONFIG_ITEM *cname, WT_CONFIG_ITEM *metadata, WT_COLLATOR **collatorp, int *ownp); extern int __wt_conn_remove_collator(WT_SESSION_IMPL *session); extern int __wt_conn_remove_compressor(WT_SESSION_IMPL *session); @@ -304,6 +305,7 @@ extern void __wt_cache_dump(WT_SESSION_IMPL *session); extern int __wt_evict(WT_SESSION_IMPL *session, WT_REF *ref, int exclusive); extern void __wt_evict_page_clean_update(WT_SESSION_IMPL *session, WT_REF *ref); extern int __wt_log_ckpt(WT_SESSION_IMPL *session, WT_LSN *ckp_lsn); +extern int __wt_log_needs_recovery(WT_SESSION_IMPL *session, WT_LSN *ckp_lsn, int *rec); extern void __wt_log_written_reset(WT_SESSION_IMPL *session); extern int __wt_log_get_all_files(WT_SESSION_IMPL *session, char ***filesp, u_int *countp, uint32_t *maxid, int active_only); extern void __wt_log_files_free(WT_SESSION_IMPL *session, char **files, u_int count); @@ -436,8 +438,7 @@ extern int __wt_dlopen(WT_SESSION_IMPL *session, const char *path, WT_DLH **dlhp extern int __wt_dlsym(WT_SESSION_IMPL *session, WT_DLH *dlh, const char *name, int fail, void *sym_ret); extern int __wt_dlclose(WT_SESSION_IMPL *session, WT_DLH *dlh); extern int __wt_errno(void); -extern const char *__wt_strerror(int error); -extern int __wt_strerror_r(int error, char *buf, size_t buflen); +extern const char *__wt_strerror(WT_SESSION_IMPL *session, int error, char *errbuf, size_t errlen); extern int __wt_exist(WT_SESSION_IMPL *session, const char *filename, int *existp); extern void __wt_fallocate_config(WT_SESSION_IMPL *session, WT_FH *fh); extern int __wt_fallocate( WT_SESSION_IMPL *session, WT_FH *fh, wt_off_t offset, wt_off_t len); @@ -573,6 +574,7 @@ extern void __wt_errx(WT_SESSION_IMPL *session, const char *fmt, ...) WT_GCC_FUN extern int __wt_ext_err_printf( WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4))); extern int __wt_msg(WT_SESSION_IMPL *session, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 2, 3))); extern int __wt_ext_msg_printf( WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4))); +extern const char *__wt_ext_strerror(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, int error); extern int __wt_progress(WT_SESSION_IMPL *session, const char *s, uint64_t v); extern void __wt_assert(WT_SESSION_IMPL *session, int error, const char *file_name, int line_number, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 5, 6))); extern int __wt_panic(WT_SESSION_IMPL *session); diff --git a/src/include/session.h b/src/include/session.h index 909f1daf5a4..36df35a104e 100644 --- a/src/include/session.h +++ b/src/include/session.h @@ -111,6 +111,8 @@ struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_session_impl { } *scratch_track; #endif + WT_ITEM err; /* Error buffer */ + WT_TXN_ISOLATION isolation; WT_TXN txn; /* Transaction state */ u_int ncursors; /* Count of active file cursors. */ diff --git a/src/include/wiredtiger.in b/src/include/wiredtiger.in index 80e0975d7e6..d0d0f9eec77 100644 --- a/src/include/wiredtiger.in +++ b/src/include/wiredtiger.in @@ -835,6 +835,17 @@ struct __wt_session { int __F(reconfigure)(WT_SESSION *session, const char *config); /*! + * Return information about an error as a string. + * + * @snippet ex_all.c Display an error thread safe + * + * @param session the session handle + * @param error a return value from a WiredTiger function + * @returns a string representation of the error + */ + const char *__F(strerror)(WT_SESSION *session, int error); + + /*! * @name Cursor handles * @{ */ @@ -1319,6 +1330,9 @@ struct __wt_session { * @config{dump_pages, Display the contents of in-memory pages as they * are verified\, using the application's message handler\, intended for * debugging., a boolean flag; default \c false.} + * @config{dump_shape, Display the shape of the tree after + * verification\, using the application's message handler\, intended for + * debugging., a boolean flag; default \c false.} * @configend * @ebusy_errors */ @@ -1543,14 +1557,14 @@ struct __wt_connection { * integer between 1 and 20; default \c 2.} * @config{ ),,} * @config{cache_overhead, assume the heap allocator overhead is the - * specified percentage\, and adjust the cache size by that amount (for - * example\, if the cache size is 100GB\, a percentage of 10 means - * WiredTiger limits itself to allocating 90GB of memory). This value is - * configurable because different heap allocators have different - * overhead and different workloads will have different heap allocation - * sizes and patterns\, therefore applications may need to adjust this - * value based on allocator choice and behavior in measured workloads., - * an integer between 0 and 30; default \c 8.} + * specified percentage\, and adjust the cache usage by that amount (for + * example\, if there is 10GB of data in cache\, a percentage of 10 + * means WiredTiger treats this as 11GB). This value is configurable + * because different heap allocators have different overhead and + * different workloads will have different heap allocation sizes and + * patterns\, therefore applications may need to adjust this value based + * on allocator choice and behavior in measured workloads., an integer + * between 0 and 30; default \c 8.} * @config{cache_size, maximum heap memory to allocate for the cache. A * database should configure either \c cache_size or \c shared_cache but * not both., an integer between 1MB and 10TB; default \c 100MB.} @@ -1879,13 +1893,13 @@ struct __wt_connection { * should be used (4KB on Linux systems\, zero elsewhere)., an integer between * -1 and 1MB; default \c -1.} * @config{cache_overhead, assume the heap allocator overhead is the specified - * percentage\, and adjust the cache size by that amount (for example\, if the - * cache size is 100GB\, a percentage of 10 means WiredTiger limits itself to - * allocating 90GB of memory). This value is configurable because different heap - * allocators have different overhead and different workloads will have - * different heap allocation sizes and patterns\, therefore applications may - * need to adjust this value based on allocator choice and behavior in measured - * workloads., an integer between 0 and 30; default \c 8.} + * percentage\, and adjust the cache usage by that amount (for example\, if + * there is 10GB of data in cache\, a percentage of 10 means WiredTiger treats + * this as 11GB). This value is configurable because different heap allocators + * have different overhead and different workloads will have different heap + * allocation sizes and patterns\, therefore applications may need to adjust + * this value based on allocator choice and behavior in measured workloads., an + * integer between 0 and 30; default \c 8.} * @config{cache_size, maximum heap memory to allocate for the cache. A * database should configure either \c cache_size or \c shared_cache but not * both., an integer between 1MB and 10TB; default \c 100MB.} @@ -1973,6 +1987,9 @@ struct __wt_connection { * are created relative to the database home., a string; default empty.} * @config{ prealloc, pre-allocate log files., a boolean * flag; default \c true.} + * @config{ recover, run recovery + * or error if recovery needs to run after an unclean shutdown., a string\, + * chosen from the following options: \c "error"\, \c "on"; default \c on.} * @config{ ),,} * @config{lsm_manager = (, configure database wide options for LSM tree * management., a set of related configuration options defined below.} @@ -2086,18 +2103,6 @@ int wiredtiger_open(const char *home, */ const char *wiredtiger_strerror(int error); -/*! - * Return information about a WiredTiger error as a string, thread-safe version. - * - * @snippet ex_all.c Display an error thread safe - * - * @param error a return value from a WiredTiger call - * @param buf a buffer of at least \c buflen bytes - * @param buflen the length of the buffer - * @returns zero for success, non-zero to indicate an error. - */ -int wiredtiger_strerror_r(int error, char *buf, size_t buflen); - #if !defined(SWIG) /*! * The interface implemented by applications to accept notifications @@ -2618,6 +2623,12 @@ const char *wiredtiger_version(int *majorp, int *minorp, int *patchp); /*! Restart the operation (internal). */ #define WT_RESTART -31805 /*! @endcond */ +/*! + * Recovery must be run to continue. + * This error is generated when wiredtiger_open is configured to return an error + * if recovery is required to use the database. + */ +#define WT_RUN_RECOVERY -31806 /* * Error return section: END * DO NOT EDIT: automatically built by dist/api_err.py. diff --git a/src/include/wiredtiger_ext.h b/src/include/wiredtiger_ext.h index ee27b32ddf7..28fd8e18329 100644 --- a/src/include/wiredtiger_ext.h +++ b/src/include/wiredtiger_ext.h @@ -118,16 +118,17 @@ struct __wt_extension_api { WT_EXTENSION_API *, WT_SESSION *session, const char *fmt, ...); /*! - * Return information about an error as a string; the strerror method - * is a superset of the ISO C99/POSIX 1003.1-2001 function strerror. + * Return information about an error as a string. * * @snippet ex_data_source.c WT_EXTENSION_API strerror * - * @param err a return value from a WiredTiger, C library or POSIX - * function + * @param wt_api the extension handle + * @param session the session handle (or NULL if none available) + * @param error a return value from a WiredTiger function * @returns a string representation of the error */ - const char *(*strerror)(int err); + const char *(*strerror)( + WT_EXTENSION_API *, WT_SESSION *session, int error); /*! * Allocate short-term use scratch memory. diff --git a/src/log/log.c b/src/log/log.c index 38c953d7835..f6c8602faff 100644 --- a/src/log/log.c +++ b/src/log/log.c @@ -36,6 +36,45 @@ __wt_log_ckpt(WT_SESSION_IMPL *session, WT_LSN *ckp_lsn) } /* + * __wt_log_needs_recovery -- + * Return 0 if we encounter a clean shutdown and 1 if recovery + * must be run in the given variable. + */ +int +__wt_log_needs_recovery(WT_SESSION_IMPL *session, WT_LSN *ckp_lsn, int *rec) +{ + WT_CONNECTION_IMPL *conn; + WT_CURSOR *c; + WT_DECL_RET; + WT_LOG *log; + + conn = S2C(session); + log = conn->log; + c = NULL; + /* + * Default is to run recovery always. + */ + *rec = 1; + + if (log == NULL) + return (0); + WT_RET(__wt_curlog_open(session, "log:", NULL, &c)); + c->set_key(c, ckp_lsn->file, ckp_lsn->offset, 0); + WT_ERR(c->search(c)); + /* + * If the checkpoint LSN we're given is the last record, + * then recovery is not needed. + */ + if ((ret = c->next(c)) == WT_NOTFOUND) { + *rec = 0; + ret = 0; + } +err: if (c != NULL) + (void)c->close(c); + return (ret); +} + +/* * __wt_log_written_reset -- * Interface to reset the amount of log written during this * checkpoint period. Called from the checkpoint code. diff --git a/src/lsm/lsm_merge.c b/src/lsm/lsm_merge.c index 923d8fd2837..6ca1b0f04ab 100644 --- a/src/lsm/lsm_merge.c +++ b/src/lsm/lsm_merge.c @@ -511,7 +511,8 @@ err: if (locked) "Merge aborted due to close")); else WT_TRET(__wt_verbose(session, WT_VERB_LSM, - "Merge failed with %s", wiredtiger_strerror(ret))); + "Merge failed with %s", + __wt_strerror(session, ret, NULL, 0))); } F_CLR(session, WT_SESSION_NO_CACHE | WT_SESSION_NO_CACHE_CHECK); return (ret); diff --git a/src/lsm/lsm_meta.c b/src/lsm/lsm_meta.c index fa051413c5e..2fcced4d1c4 100644 --- a/src/lsm/lsm_meta.c +++ b/src/lsm/lsm_meta.c @@ -153,10 +153,11 @@ __wt_lsm_meta_read(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) } WT_ERR_NOTFOUND_OK(ret); lsm_tree->nold_chunks = nchunks; - /* Values included for backward compatibility */ - } else if (WT_STRING_MATCH("merge_threads", ck.str, ck.len)) { - } else - WT_ERR(__wt_illegal_value(session, "LSM metadata")); + } + /* + * Ignore any other values: the metadata entry might have been + * created by a future release, with unknown options. + */ } WT_ERR_NOTFOUND_OK(ret); diff --git a/src/os_posix/os_errno.c b/src/os_posix/os_errno.c index a58f13583ce..229b68e0008 100644 --- a/src/os_posix/os_errno.c +++ b/src/os_posix/os_errno.c @@ -24,46 +24,32 @@ __wt_errno(void) /* * __wt_strerror -- - * POSIX implementation of wiredtiger_strerror. + * POSIX implementation of WT_SESSION.strerror and wiredtiger_strerror. */ const char * -__wt_strerror(int error) +__wt_strerror(WT_SESSION_IMPL *session, int error, char *errbuf, size_t errlen) { const char *p; /* - * POSIX errors are non-negative integers; check for 0 explicitly - * in-case the underlying strerror doesn't handle 0, some don't. + * Check for a WiredTiger or POSIX constant string, no buffer needed. */ - if (error == 0) - return ("Successful return: 0"); - if (error > 0 && (p = strerror(error)) != NULL) + if ((p = __wt_wiredtiger_error(error)) != NULL) return (p); - return (NULL); -} - -/* - * __wt_strerror_r -- - * POSIX implementation of wiredtiger_strerror_r. - */ -int -__wt_strerror_r(int error, char *buf, size_t buflen) -{ - const char *p; - - /* Require at least 2 bytes, printable character and trailing nul. */ - if (buflen < 2) - return (ENOMEM); /* - * Check for POSIX errors then fallback to something generic. Copy the - * string into the user's buffer, return success if anything printed. + * When called from wiredtiger_strerror, write a passed-in buffer. + * When called from WT_SESSION.strerror, write the session's buffer. + * + * Fallback to a generic message. */ - p = __wt_strerror(error); - if (p != NULL && snprintf(buf, buflen, "%s", p) > 0) - return (0); - - /* Fallback to a generic message, then guess it's a memory problem. */ - return ( - snprintf(buf, buflen, "error return: %d", error) > 0 ? 0 : ENOMEM); + if (session == NULL && + snprintf(errbuf, errlen, "error return: %d", error) > 0) + return (errbuf); + if (session != NULL && __wt_buf_fmt( + session, &session->err, "error return: %d", error) == 0) + return (session->err.data); + + /* Defeated. */ + return ("Unable to return error string"); } diff --git a/src/os_win/os_errno.c b/src/os_win/os_errno.c index 00ee638fbe3..81bcdf9089e 100644 --- a/src/os_win/os_errno.c +++ b/src/os_win/os_errno.c @@ -58,47 +58,27 @@ __wt_errno(void) /* * __wt_strerror -- - * Windows implementation of wiredtiger_strerror. + * Windows implementation of WT_SESSION.strerror and wiredtiger_strerror. */ const char * -__wt_strerror(int error) +__wt_strerror(WT_SESSION_IMPL *session, int error, char *errbuf, size_t errlen) { + DWORD lasterror; const char *p; + char buf[512]; /* - * POSIX errors are non-negative integers; check for 0 explicitly - * in-case the underlying strerror doesn't handle 0, some don't. + * Check for a WiredTiger or POSIX constant string, no buffer needed. */ - if (error == 0) - return ("Successful return: 0"); - if (error > 0 && (p = strerror(error)) != NULL) + if ((p = __wt_wiredtiger_error(error)) != NULL) return (p); - return (NULL); -} - -/* - * __wt_strerror_r -- - * Windows implementation of wiredtiger_strerror_r. - */ -int -__wt_strerror_r(int error, char *buf, size_t buflen) -{ - DWORD lasterror; - const char *p; - - /* Require at least 2 bytes, printable character and trailing nul. */ - if (buflen < 2) - return (ENOMEM); /* - * Check for POSIX errors, Windows errors, then fallback to something - * generic. Copy the string into the user's buffer, return success if - * anything printed. + * When called from wiredtiger_strerror, write a passed-in buffer. + * When called from WT_SESSION.strerror, write the session's buffer. + * + * Check for Windows errors. */ - p = __wt_strerror(error); - if (p != NULL && snprintf(buf, buflen, "%s", p) > 0) - return (0); - if (error < 0) { error = __wt_map_error_to_windows_error(error); @@ -109,16 +89,25 @@ __wt_strerror_r(int error, char *buf, size_t buflen) error, 0, /* let system choose the correct LANGID */ buf, - buflen, + sizeof(buf), NULL); - if (lasterror != 0) - return (0); - - /* Fall through to the fallback error code */ + if (lasterror != 0 && session == NULL && + snprintf(errbuf, errlen, "%s", buf) > 0) + return (errbuf); + if (lasterror != 0 && session != NULL && + __wt_buf_set(session, &session->err, buf, strlen(buf)) == 0) + return (session->err.data); } - /* Fallback to a generic message, then guess it's a memory problem. */ - return ( - snprintf(buf, buflen, "error return: %d", error) > 0 ? 0 : ENOMEM); + /* Fallback to a generic message. */ + if (session == NULL && + snprintf(errbuf, errlen, "error return: %d", error) > 0) + return (errbuf); + if (session != NULL && __wt_buf_fmt( + session, &session->err, "error return: %d", error) == 0) + return (session->err.data); + + /* Defeated. */ + return ("Unable to return error string"); } diff --git a/src/session/session_api.c b/src/session/session_api.c index 8ee143133ae..e54553aa071 100644 --- a/src/session/session_api.c +++ b/src/session/session_api.c @@ -128,8 +128,9 @@ __session_close(WT_SESSION *wt_session, const char *config) /* Discard metadata tracking. */ __wt_meta_track_discard(session); - /* Discard scratch buffers. */ + /* Discard scratch buffers, error memory. */ __wt_scr_discard(session); + __wt_buf_free(session, &session->err); /* Free transaction information. */ __wt_txn_destroy(session); @@ -670,7 +671,11 @@ __session_truncate(WT_SESSION *wt_session, done: err: TXN_API_END_RETRY(session, ret, 0); - return ((ret) == WT_NOTFOUND ? ENOENT : (ret)); + + /* + * Only map WT_NOTFOUND to ENOENT if a URI was specified. + */ + return (ret == WT_NOTFOUND && uri != NULL ? ENOENT : ret); } /* @@ -898,6 +903,20 @@ err: F_CLR(session, WT_SESSION_CAN_WAIT | WT_SESSION_NO_CACHE_CHECK); } /* + * __session_strerror -- + * WT_SESSION->strerror method. + */ +static const char * +__session_strerror(WT_SESSION *wt_session, int error) +{ + WT_SESSION_IMPL *session; + + session = (WT_SESSION_IMPL *)wt_session; + + return (__wt_strerror(session, error, NULL, 0)); +} + +/* * __wt_open_internal_session -- * Allocate a session for WiredTiger's use. */ @@ -959,6 +978,7 @@ __wt_open_session(WT_CONNECTION_IMPL *conn, NULL, __session_close, __session_reconfigure, + __session_strerror, __session_open_cursor, __session_create, __session_compact, diff --git a/src/support/err.c b/src/support/err.c index d766fcba33a..49a3891c58a 100644 --- a/src/support/err.c +++ b/src/support/err.c @@ -409,6 +409,20 @@ __wt_ext_msg_printf( } /* + * __wt_ext_strerror -- + * Extension API call to return an error as a string. + */ +const char * +__wt_ext_strerror(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, int error) +{ + if (wt_session == NULL) + wt_session = (WT_SESSION *) + ((WT_CONNECTION_IMPL *)wt_api->conn)->default_session; + + return (wt_session->strerror(wt_session, error)); +} + +/* * __wt_progress -- * Progress message. */ diff --git a/src/txn/txn_recover.c b/src/txn/txn_recover.c index 71f5df9dda2..2c834083691 100644 --- a/src/txn/txn_recover.c +++ b/src/txn/txn_recover.c @@ -412,7 +412,7 @@ __wt_txn_recover(WT_SESSION_IMPL *session) WT_RECOVERY r; struct WT_RECOVERY_FILE *metafile; char *config; - int was_backup; + int needs_rec, was_backup; conn = S2C(session); WT_CLEAR(r); @@ -483,14 +483,25 @@ __wt_txn_recover(WT_SESSION_IMPL *session) WT_ERR(__wt_verbose(session, WT_VERB_RECOVERY, "Main recovery loop: starting at %u/%" PRIuMAX, r.ckpt_lsn.file, (uintmax_t)r.ckpt_lsn.offset)); + WT_ERR(__wt_log_needs_recovery(session, &r.ckpt_lsn, &needs_rec)); + /* + * Check if the database was shut down cleanly. If not + * return an error if the user does not want automatic + * recovery. + */ + if (needs_rec && FLD_ISSET(conn->log_flags, WT_CONN_LOG_RECOVER_ERR)) + WT_ERR(WT_RUN_RECOVERY); + /* + * Always run recovery even if it was a clean shutdown. + * We can consider skipping it in the future. + */ if (WT_IS_INIT_LSN(&r.ckpt_lsn)) WT_ERR(__wt_log_scan(session, NULL, WT_LOGSCAN_FIRST | WT_LOGSCAN_RECOVER, __txn_log_recover, &r)); else WT_ERR(__wt_log_scan(session, &r.ckpt_lsn, - WT_LOGSCAN_RECOVER, - __txn_log_recover, &r)); + WT_LOGSCAN_RECOVER, __txn_log_recover, &r)); conn->next_file_id = r.max_fileid; diff --git a/src/utilities/util_main.c b/src/utilities/util_main.c index ecfffb81e4b..3274f3a0fd0 100644 --- a/src/utilities/util_main.c +++ b/src/utilities/util_main.c @@ -11,11 +11,15 @@ const char *home = "."; /* Home directory */ const char *progname; /* Program name */ /* Global arguments */ -const char *usage_prefix = "[-Vv] [-C config] [-h home]"; +const char *usage_prefix = "[-Vv] [-R] [-C config] [-h home]"; int verbose; /* Verbose flag */ static const char *command; /* Command name */ +#define REC_ERROR "log=(recover=error)" +#define REC_LOGOFF "log=(enabled=false)" +#define REC_RECOVER "log=(recover=on)" + static int usage(void); int @@ -27,7 +31,7 @@ main(int argc, char *argv[]) size_t len; int ch, major_v, minor_v, tret, (*func)(WT_SESSION *, int, char *[]); char *p; - const char *cmd_config, *config; + const char *cmd_config, *config, *rec_config; conn = NULL; p = NULL; @@ -52,9 +56,16 @@ main(int argc, char *argv[]) return (EXIT_FAILURE); } - /* Check for standard options. */ cmd_config = config = NULL; - while ((ch = __wt_getopt(progname, argc, argv, "C:h:Vv")) != EOF) + /* + * We default to returning an error if recovery needs to be run. + * Generally we expect this to be run after a clean shutdown. + * The printlog command disables logging entirely. If recovery is + * needed, the user can specify -R to run recovery. + */ + rec_config = REC_ERROR; + /* Check for standard options. */ + while ((ch = __wt_getopt(progname, argc, argv, "C:h:RVv")) != EOF) switch (ch) { case 'C': /* wiredtiger_open config */ cmd_config = __wt_optarg; @@ -62,6 +73,9 @@ main(int argc, char *argv[]) case 'h': /* home directory */ home = __wt_optarg; break; + case 'R': /* recovery */ + rec_config = REC_RECOVER; + break; case 'V': /* version */ printf("%s\n", wiredtiger_version(NULL, NULL, NULL)); return (EXIT_SUCCESS); @@ -118,8 +132,10 @@ main(int argc, char *argv[]) } break; case 'p': - if (strcmp(command, "printlog") == 0) + if (strcmp(command, "printlog") == 0) { func = util_printlog; + rec_config = REC_LOGOFF; + } break; case 'r': if (strcmp(command, "read") == 0) @@ -154,15 +170,22 @@ main(int argc, char *argv[]) return (usage()); /* Build the configuration string, as necessary. */ - if (config == NULL) - config = cmd_config; - else if (cmd_config != NULL) { - len = strlen(cmd_config) + strlen(config) + 10; + if (cmd_config != NULL || rec_config != NULL) { + len = 10; /* some slop */ + if (config != NULL) + len += strlen(config); + if (cmd_config != NULL) + len += strlen(cmd_config); + if (rec_config != NULL) + len += strlen(rec_config); if ((p = malloc(len)) == NULL) { ret = util_err(errno, NULL); goto err; } - (void)snprintf(p, len, "%s,%s", config, cmd_config); + (void)snprintf(p, len, "%s,%s,%s", + config == NULL ? "" : config, + cmd_config == NULL ? "" : cmd_config, + rec_config == NULL ? "" : rec_config); config = p; } @@ -201,6 +224,7 @@ usage(void) "global options:\n" "\t" "-C\twiredtiger_open configuration\n" "\t" "-h\tdatabase directory\n" + "\t" "-R\trun recovery if configured\n" "\t" "-V\tdisplay library version and exit\n" "\t" "-v\tverbose\n"); fprintf(stderr, diff --git a/src/utilities/util_verify.c b/src/utilities/util_verify.c index 7b3fffdd03b..796d24a8a6d 100644 --- a/src/utilities/util_verify.c +++ b/src/utilities/util_verify.c @@ -10,28 +10,17 @@ static int usage(void); -#undef OPT_ARGS -#undef USAGE_ARGS -#ifdef HAVE_DIAGNOSTIC -#define OPT_ARGS "d:" -#define USAGE_ARGS \ - "[-d dump_address | dump_blocks | dump_offsets=#,# | dump_pages] uri" -#else -#define OPT_ARGS "" -#define USAGE_ARGS "uri" -#endif - int util_verify(WT_SESSION *session, int argc, char *argv[]) { WT_DECL_RET; size_t size; - int ch, dump_address, dump_blocks, dump_pages; + int ch, dump_address, dump_blocks, dump_pages, dump_shape; char *config, *dump_offsets, *name; - dump_address = dump_blocks = dump_pages = 0; + dump_address = dump_blocks = dump_pages = dump_shape = 0; config = dump_offsets = name = NULL; - while ((ch = __wt_getopt(progname, argc, argv, OPT_ARGS)) != EOF) + while ((ch = __wt_getopt(progname, argc, argv, "d:")) != EOF) switch (ch) { case 'd': if (strcmp(__wt_optarg, "dump_address") == 0) @@ -50,6 +39,8 @@ util_verify(WT_SESSION *session, int argc, char *argv[]) __wt_optarg + strlen("dump_offsets="); } else if (strcmp(__wt_optarg, "dump_pages") == 0) dump_pages = 1; + else if (strcmp(__wt_optarg, "dump_shape") == 0) + dump_shape = 1; else return (usage()); break; @@ -67,11 +58,13 @@ util_verify(WT_SESSION *session, int argc, char *argv[]) return (1); /* Build the configuration string as necessary. */ - if (dump_address || dump_blocks || dump_offsets != NULL || dump_pages) { + if (dump_address || + dump_blocks || dump_offsets != NULL || dump_pages || dump_shape) { size = strlen("dump_address,") + strlen("dump_blocks,") + strlen("dump_pages,") + + strlen("dump_shape,") + strlen("dump_offsets[],") + (dump_offsets == NULL ? 0 : strlen(dump_offsets)) + 20; if ((config = malloc(size)) == NULL) { @@ -79,13 +72,14 @@ util_verify(WT_SESSION *session, int argc, char *argv[]) goto err; } snprintf(config, size, - "%s%s%s%s%s%s", + "%s%s%s%s%s%s%s", dump_address ? "dump_address," : "", dump_blocks ? "dump_blocks," : "", dump_offsets != NULL ? "dump_offsets=[" : "", dump_offsets != NULL ? dump_offsets : "", dump_offsets != NULL ? "]," : "", - dump_pages ? "dump_pages" : ""); + dump_pages ? "dump_pages," : "", + dump_shape ? "dump_shape," : ""); } if ((ret = session->verify(session, name, config)) != 0) { fprintf(stderr, "%s: verify(%s): %s\n", @@ -115,6 +109,8 @@ usage(void) (void)fprintf(stderr, "usage: %s %s " "verify %s\n", - progname, usage_prefix, USAGE_ARGS); + progname, usage_prefix, + "[-d dump_address | dump_blocks | " + "dump_offsets=#,# | dump_pages | dump_shape] uri"); return (1); } diff --git a/test/format/recover.sh b/test/format/recover.sh index 391da3c85fb..de908c71e5d 100644 --- a/test/format/recover.sh +++ b/test/format/recover.sh @@ -34,11 +34,15 @@ while true; do rm -rf $rundir2 $tcmd $config -q abort=1 logging=1 timer=$timer - uri='file:wt' - if `$wtcmd -h RUNDIR list | egrep table > /dev/null`; then - uri='table:wt' - fi # Save a copy of the database directory exactly as it was at the crash. cp -rp RUNDIR $rundir2 - $wtcmd -h RUNDIR verify $uri || exit 1 + + # We aborted, so recovery is required + if `$wtcmd -R -h RUNDIR list | egrep table > /dev/null`; then + uri='table:wt' + else + uri='file:wt' + fi + # Force recovery to run. + $wtcmd -R -h RUNDIR verify $uri || exit 1 done diff --git a/test/suite/test_backup04.py b/test/suite/test_backup04.py index 60f8f0df6c2..fe81dc29d45 100644 --- a/test/suite/test_backup04.py +++ b/test/suite/test_backup04.py @@ -89,7 +89,7 @@ class test_backup_target(wttest.WiredTigerTestCase, suite_subprocess): # Compare the original and backed-up files using the wt dump command. def compare(self, uri, dir_full, dir_incr): - #print "Compare: full URI: " + uri_full + " with incremental URI: " + uri_incr + # print "Compare: full URI: " + uri + " with incremental URI " if dir_full == None: full_name='original' else: @@ -99,11 +99,15 @@ class test_backup_target(wttest.WiredTigerTestCase, suite_subprocess): os.remove(full_name) if os.path.exists(incr_name): os.remove(incr_name) + # + # We have been copying the logs only, so we need to force 'wt' to + # run recovery in order to apply all the logs and check the data. + # if dir_full == None: - self.runWt(['dump', uri], outfilename=full_name) + self.runWt(['-R', 'dump', uri], outfilename=full_name) else: - self.runWt(['-h', dir_full, 'dump', uri], outfilename=full_name) - self.runWt(['-h', dir_incr, 'dump', uri], outfilename=incr_name) + self.runWt(['-R', '-h', dir_full, 'dump', uri], outfilename=full_name) + self.runWt(['-R', '-h', dir_incr, 'dump', uri], outfilename=incr_name) self.assertEqual(True, compare_files(self, full_name, incr_name)) diff --git a/test/suite/test_txn02.py b/test/suite/test_txn02.py index 9369f1a0226..bcba27d9584 100644 --- a/test/suite/test_txn02.py +++ b/test/suite/test_txn02.py @@ -191,8 +191,12 @@ class test_txn02(wttest.WiredTigerTestCase, suite_subprocess): self.assertEqual(True, o in cur_logs) # # Run printlog and make sure it exits with zero status. + # Printlog should not run recovery nor advance the logs. Make sure + # it does not. # self.runWt(['-h', self.backup_dir, 'printlog'], outfilename='printlog.out') + pr_logs = fnmatch.filter(os.listdir(self.backup_dir), "*Log*") + self.assertEqual(cur_logs, pr_logs) def test_ops(self): # print "Creating %s with config '%s'" % (self.uri, self.create_params) diff --git a/test/suite/test_txn10.py b/test/suite/test_txn10.py new file mode 100644 index 00000000000..9876a68042e --- /dev/null +++ b/test/suite/test_txn10.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2015 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_txn10.py +# Transactions: commits and rollbacks +# + +import fnmatch, os, shutil, time +from suite_subprocess import suite_subprocess +from wiredtiger import wiredtiger_open +from wtscenario import multiply_scenarios, number_scenarios, prune_scenarios +import wttest + +class test_txn10(wttest.WiredTigerTestCase, suite_subprocess): + t1 = 'table:test_txn10_1' + t2 = 'table:test_txn10_2' + create_params = 'key_format=i,value_format=i' + + # Overrides WiredTigerTestCase, add extra config params + def setUpConnectionOpen(self, dir): + self.conn_config = \ + 'log=(archive=false,enabled,file_max=100K),' + \ + 'transaction_sync=(method=dsync,enabled)' + return wttest.WiredTigerTestCase.setUpConnectionOpen(self, dir) + + def simulate_crash_restart(self, olddir, newdir): + ''' Simulate a crash from olddir and restart in newdir. ''' + # with the connection still open, copy files to new directory + shutil.rmtree(newdir, ignore_errors=True) + os.mkdir(newdir) + for fname in os.listdir(olddir): + fullname = os.path.join(olddir, fname) + if os.path.isfile(fullname): + shutil.copy(fullname, newdir) + # close the original connection and open to new directory + self.close_conn() + self.conn = self.setUpConnectionOpen(newdir) + self.session = self.setUpSessionOpen(self.conn) + + def test_recovery(self): + ''' Check for bugs in file ID allocation. ''' + + # Here's the strategy: + # - Create a table (t1). + # - Do a clean restart. + # - Create another table (t2). + # - Insert data into t2. + # - Make recovery run. + # + # If we aren't tracking file IDs properly, it's possible that + # we'd end up apply the log records for t2 to table t1. + self.session.create(self.t1, self.create_params) + self.reopen_conn() + self.session.create(self.t2, self.create_params) + c = self.session.open_cursor(self.t2, None, None) + for i in range(10000): + c.set_key(i) + c.set_value(i+1) + c.insert() + c.close() + self.simulate_crash_restart(".", "RESTART") + c = self.session.open_cursor(self.t2, None, None) + i = 0 + for key, value in c: + self.assertEqual(i, key) + self.assertEqual(i+1, value) + i += 1 + self.assertEqual(i, 10000) + c.close() + c = self.session.open_cursor(self.t1, None, None) + i = 0 + for key, value in c: + i += 1 + self.assertEqual(i, 0) + c.close() + +if __name__ == '__main__': + wttest.run() diff --git a/tools/wtstats/test/monitor.fixture b/tools/wtstats/test/monitor.fixture new file mode 100644 index 00000000000..5306a4bd1fd --- /dev/null +++ b/tools/wtstats/test/monitor.fixture @@ -0,0 +1,41 @@ +#time,totalsec,read ops per second,insert ops per second,update ops per second,checkpoints,read average latency(uS),read minimum latency(uS),read maximum latency(uS),insert average latency(uS),insert min latency(uS),insert maximum latency(uS),update average latency(uS),update min latency(uS),update maximum latency(uS) +Feb 13 17:55:14,0,0,156871,0,N,0,0,0,49,6,6146,0,0,0 +Feb 13 17:55:17,5,0,102174,0,N,0,0,0,84,10,31917,0,0,0 +Feb 13 17:55:20,5,0,150487,0,N,0,0,0,51,9,9173,0,0,0 +Feb 13 17:55:23,10,0,142844,0,N,0,0,0,52,10,4952,0,0,0 +Feb 13 17:55:26,10,0,136411,0,N,0,0,0,58,10,3541,0,0,0 +Feb 13 17:55:29,15,0,128205,0,N,0,0,0,63,10,34782,0,0,0 +Feb 13 17:55:32,20,0,139891,0,N,0,0,0,57,10,5684,0,0,0 +Feb 13 17:55:35,20,0,146500,0,N,0,0,0,56,9,24824,0,0,0 +Feb 13 17:55:38,25,0,141227,0,N,0,0,0,59,9,17355,0,0,0 +Feb 13 17:55:41,25,0,98194,0,N,0,0,0,94,10,10364,0,0,0 +Feb 13 17:55:44,30,0,107737,0,N,0,0,0,108,10,103687,0,0,0 +Feb 13 17:55:47,35,0,142776,0,N,0,0,0,57,10,9657,0,0,0 +Feb 13 17:55:50,35,0,123185,0,N,0,0,0,61,10,4710,0,0,0 +Feb 13 17:55:53,40,0,112972,0,N,0,0,0,78,10,8038,0,0,0 +Feb 13 17:55:56,40,0,121859,0,N,0,0,0,60,9,5477,0,0,0 +Feb 13 17:55:59,45,0,99269,0,N,0,0,0,81,11,4893,0,0,0 +Feb 13 17:56:02,50,0,92201,0,N,0,0,0,98,10,14297,0,0,0 +Feb 13 17:56:05,50,0,93858,0,N,0,0,0,77,8,28356,0,0,0 +Feb 13 17:56:08,55,0,125126,0,N,0,0,0,67,12,18481,0,0,0 +Feb 13 17:56:11,55,0,116519,0,N,0,0,0,55,12,8024,0,0,0 +Feb 13 17:56:14,60,0,126543,0,N,0,0,0,62,13,6234,0,0,0 +Feb 13 17:56:17,65,0,110727,0,N,0,0,0,75,13,9378,0,0,0 +Feb 13 17:56:20,65,0,117901,0,N,0,0,0,66,11,5893,0,0,0 +Feb 13 17:56:23,70,0,76692,0,N,0,0,0,89,12,22674,0,0,0 +Feb 13 17:56:26,70,0,73809,0,N,0,0,0,101,11,23230,0,0,0 +Feb 13 17:56:29,75,0,107982,0,N,0,0,0,77,10,6785,0,0,0 +Feb 13 17:56:32,80,0,114799,0,N,0,0,0,72,10,10037,0,0,0 +Feb 13 17:56:35,80,0,109781,0,N,0,0,0,68,11,13299,0,0,0 +Feb 13 17:56:38,85,0,127438,0,N,0,0,0,61,11,8089,0,0,0 +Feb 13 17:56:41,85,0,124606,0,N,0,0,0,63,6,7921,0,0,0 +Feb 13 17:56:44,90,0,115639,0,N,0,0,0,53,11,3853,0,0,0 +Feb 13 17:56:47,95,0,91402,0,N,0,0,0,62,4,18367,0,0,0 +Feb 13 17:56:50,95,0,123687,0,N,0,0,0,62,3,9853,0,0,0 +Feb 13 17:56:53,100,0,149589,0,N,0,0,0,54,9,5460,0,0,0 +Feb 13 17:56:56,100,0,133385,0,N,0,0,0,49,9,6573,0,0,0 +Feb 13 17:56:59,105,0,140711,0,N,0,0,0,47,8,3729,0,0,0 +Feb 13 17:57:02,110,0,131385,0,N,0,0,0,56,7,6582,0,0,0 +Feb 13 17:57:05,110,0,76959,0,N,0,0,0,72,7,30412,0,0,0 +Feb 13 17:57:08,115,0,95041,0,N,0,0,0,65,4,21258,0,0,0 +Feb 13 17:57:11,115,0,127172,0,N,0,0,0,59,9,7407,0,0,0 diff --git a/tools/wtstats/test/test_wtstats.py b/tools/wtstats/test/test_wtstats.py index 80078d97667..dd6098fec49 100644 --- a/tools/wtstats/test/test_wtstats.py +++ b/tools/wtstats/test/test_wtstats.py @@ -75,7 +75,8 @@ def helper_run_with_fixture(kwargs=None): # path replacement kwargs['--output'] = os.path.join(test_dir, kwargs['--output']) - statsfile = os.path.join(test_dir, 'WiredTigerStat.fixture') + statsfile = os.path.join(test_dir, kwargs['files'] if 'files' in kwargs else 'WiredTigerStat.fixture') + print "ST", statsfile arglist = ['./wtstats', statsfile] for item in kwargs.items(): @@ -175,7 +176,51 @@ def test_output_option(): outfile = '_foo_bar_baz.html' helper_run_with_fixture({'--output': outfile}) assert os.path.exists(os.path.join(test_dir, outfile)) - + +@with_setup(setUp, tearDown) +def test_monitor_stats_start_with_wtperf(): + """ wtstats should be able to parse wtperf monitor files """ + + outfile = '_foo_bar_baz.html' + helper_run_with_fixture({'files': 'monitor.fixture', '--output': outfile}) + data = helper_get_json_from_file(outfile) + + series_keys = map(lambda x: x['key'], data['series']) + for key in series_keys: + assert key.startswith('wtperf:') + + assert os.path.exists(os.path.join(test_dir, outfile)) + + +@with_setup(setUp, tearDown) +def test_monitor_stats_convert_us_to_ms(): + """ wtstats should convert monitor stats us to ms """ + + outfile = '_foo_bar_baz.html' + helper_run_with_fixture({'files': 'monitor.fixture', '--output': outfile}) + data = helper_get_json_from_file(outfile) + + series_keys = map(lambda x: x['key'], data['series']) + for key in series_keys: + assert '(uS)' not in key + + values = (item['values'] for item in data['series'] if item['key'] == 'wtperf: insert maximum latency (ms)').next().values() + assert max(values) == 103687 / 1000. + + + +@with_setup(setUp, tearDown) +def test_directory_with_wtstats_and_wtperf(): + """ wtstats should be able to parse directories containing both types """ + + outfile = '_test_output_file.html' + helper_run_with_fixture({'files': '.', '--output': outfile}) + data = helper_get_json_from_file(outfile) + + series_keys = map(lambda x: x['key'], data['series']) + assert any(map(lambda title: 'block-manager' in title, series_keys)) + assert any(map(lambda title: 'wtperf' in title, series_keys)) + @with_setup(setUp, tearDown) def test_add_ext_if_missing(): diff --git a/tools/wtstats/wtstats.py b/tools/wtstats/wtstats.py index 031b7cb546f..3749ffd6c63 100755 --- a/tools/wtstats/wtstats.py +++ b/tools/wtstats/wtstats.py @@ -58,15 +58,18 @@ def munge(args, title, values): if title.split(' ')[1] != 'spinlock' and \ title.split(' ', 1)[1] in no_scale_per_second_list: seconds = 1 + elif 'wtperf' in title and 'per second' not in title: + seconds = 1 else: t1, v1 = values[1] seconds = (parsetime(t1) - start_time).seconds - ylabel += ' per second' + if not ylabel.endswith('per second'): + ylabel += ' per second' if seconds == 0: seconds = 1 stats_cleared = False - if args.clear or title.split(' ', 1)[1] in no_clear_list: + if args.clear or title.split(' ', 1)[1] in no_clear_list or 'wtperf' in title: stats_cleared = True # Split the values into a dictionary of y-axis values keyed by the x axis @@ -85,9 +88,186 @@ def munge(args, title, values): return ylabel, ydata + # Parse the command line import argparse +def common_prefix(a, b): + """ compute longest common prefix of a and b """ + while not b.startswith(a): + a = a[:-1] + return a + + +def common_suffix(a, b): + """ compute longest common suffix of a and b """ + while not a.endswith(b): + b = b[1:] + return b + + +def parse_wtstats_file(file, result): + """ parse wtstats file, one stat per line, example format: + Dec 05 14:43:14 0 /data/b block-manager: mapped bytes read + """ + print 'Processing wtstats file: ' + file + + # Parse file + for line in open(file, 'rU'): + month, day, time, v, title = line.strip('\n').split(" ", 4) + result[title].append((month + " " + day + " " + time, v)) + + + +def parse_wtperf_file(file, result): + """ parse wtperf file, all stats on single line, example format: + Feb 13 17:55:14,0,0,156871,0,N,0,0,0,49,6,6146,0,0,0 + """ + print 'Processing wtperf file: ' + file + fh = open(file, 'rU') + + # first line contains headings, replace microseconds with milliseconds + headings = fh.next().strip('\n').split(',')[1:] + headings = map(lambda h: h.replace('(uS)', ' (ms)'), headings) + + # parse rest of file + for line in fh: + month, day, time, values = re.split(r'[ ,]', line.strip('\n'), 3) + values = values.split(',') + for i, v in enumerate(values): + if v == 'N': + v = 0 + # convert us to ms + if '(ms)' in headings[i]: + v = float(v) / 1000.0 + result['wtperf: ' + headings[i]].append((month + " " + day + " " + time, v)) + + +def skip_constants(result): + # Process the series, eliminate constants, delete totalsec for wtperf + items = list(result.iteritems()) + + for title, values in items: + skip = True + t0, v0 = values[0] + for t, v in values: + if v != v0: + skip = False + break + + if title == 'wtperf: totalsec': + skip = True + + if skip: + del result[title] + + return result + + +def parse_files(files_or_dir): + """ walk through file list or directory and parse according to file type (wtstats / wtperf). """ + + result = defaultdict(list) + + for f in files_or_dir: + if os.path.isfile(f): + # peek at first line to determine type + with open(f, 'rU') as fh: + line = fh.readline() + if line.startswith('#time'): + parse_wtperf_file(f, result) + else: + parse_wtstats_file(f, result) + + elif os.path.isdir(f): + for s in glob(os.path.join(f, 'WiredTigerStat*')): + parse_wtstats_file(s, result) + + for s in glob(os.path.join(f, 'monitor*')): + parse_wtperf_file(s, result) + + return result + + + +def output_series(results, args, prefix=None, grouplist=[]): + """ Write the data into the html template """ + + # add .html ending if not present + filename, ext = os.path.splitext(args.output) + if ext == '': + ext = '.html' + + # open the output file based on prefix + if prefix == None: + outputname = filename + ext + elif len(grouplist) == 0: + outputname = filename +'.' + prefix + ext + else: + outputname = filename +'.group.' + prefix + ext + + if prefix != None and len(grouplist) == 0: + this_series = [] + for title, ydata in results: + if not prefix in title: + continue + #print 'Appending to dataset: ' + title + this_series.append((title, ydata)) + elif prefix != None and len(grouplist) > 0: + this_series = [] + for title, ydata in results: + for subgroup in grouplist: + if not subgroup in title: + continue + # print 'Appending to dataset: ' + title + this_series.append((title, ydata)) + else: + this_series = results + + if len(this_series) == 0: + print 'Output: ' + outputname + ' has no data. Do not create.' + return + + + json_output = { "series": [] } + + for title, ydata in this_series: + json_output["series"].append({ + "key": title, + "values": ydata, + }); + + # load template + this_path = os.path.dirname(os.path.realpath(__file__)) + srcfile = os.path.join(this_path, 'wtstats.html.template') + try: + srcfile = open(srcfile) + contents = srcfile.read() + except IOError: + print >>sys.stderr, "Cannot find template file 'wtstats.html." \ + "template'. See ./template/README.md for more information." + sys.exit(-1) + + srcfile.close() + + # if --json write data to <filename>.json + if args.json: + jsonfile = filename + '.json' + with open(jsonfile, 'w') as f: + json.dump(json_output, f) + print "created %s" % jsonfile + + # write output file + dstfile = open(outputname, 'wt') + replaced_contents = contents.replace('"### INSERT DATA HERE ###"', + json.dumps(json_output)) + dstfile.write(replaced_contents) + dstfile.close() + print "created %s" % dstfile.name + + + + def main(): parser = argparse.ArgumentParser(description='Create graphs from' \ 'WiredTiger statistics.') @@ -109,155 +289,51 @@ def main(): 'logging') args = parser.parse_args() - # Read the input file(s) into a dictionary of lists. - def getfiles(l): - for f in l: - if os.path.isfile(f): - yield f - elif os.path.isdir(f): - for s in glob(os.path.join(f, 'WiredTigerStat*')): - print 'Processing ' + s - yield s - - d = defaultdict(list) - for f in getfiles(args.files): - for line in open(f, 'rU'): - month, day, time, v, title = line.strip('\n').split(" ", 4) - d[title].append((month + " " + day + " " + time, v)) - - # Process the series, eliminate constants - for title, values in sorted(d.iteritems()): - skip = True - t0, v0 = values[0] - for t, v in values: - if v != v0: - skip = False - break - if skip: - #print "Skipping", title - del d[title] + # Parse files or directory and skip constants + parsed = skip_constants(parse_files(args.files)) - # Common prefix / suffix elimination + # filter results based on --include, compute common prefix and suffix + results = [] prefix = suffix = None - def common_prefix(a, b): - while not b.startswith(a): - a = a[:-1] - return a - - def common_suffix(a, b): - while not a.endswith(b): - b = b[1:] - return b - - def output_series(results, prefix=None, grouplist=[]): - # add .html ending if not present - filename, ext = os.path.splitext(args.output) - if ext == '': - ext = '.html' - - # open the output file based on prefix - if prefix == None: - outputname = filename + ext - elif len(grouplist) == 0: - outputname = filename +'.' + prefix + ext - else: - outputname = filename +'.group.' + prefix + ext - - if prefix != None and len(grouplist) == 0: - this_series = [] - for title, ydata in results: - if not prefix in title: - continue - #print 'Appending to dataset: ' + title - this_series.append((title, ydata)) - elif prefix != None and len(grouplist) > 0: - this_series = [] - for title, ydata in results: - for subgroup in grouplist: - if not subgroup in title: - continue - # print 'Appending to dataset: ' + title - this_series.append((title, ydata)) - else: - this_series = results - - if len(this_series) == 0: - print 'Output: ' + outputname + ' has no data. Do not create.' - return - - - json_output = { "series": [] } - - for title, ydata in this_series: - json_output["series"].append({ - "key": title, - "values": ydata, - }); - - # load template - this_path = os.path.dirname(os.path.realpath(__file__)) - srcfile = os.path.join(this_path, 'wtstats.html.template') - try: - srcfile = open(srcfile) - contents = srcfile.read() - except IOError: - print >>sys.stderr, "Cannot find template file 'wtstats.html." \ - "template'. See ./template/README.md for more information." - sys.exit(-1) - - srcfile.close() - - # if --json write data to <filename>.json - if args.json: - jsonfile = filename + '.json' - with open(jsonfile, 'w') as f: - json.dump(json_output, f) - print "created %s" % jsonfile - - # write output file - dstfile = open(outputname, 'wt') - replaced_contents = contents.replace('"### INSERT DATA HERE ###"', - json.dumps(json_output)) - dstfile.write(replaced_contents) - dstfile.close() - print "created %s" % dstfile.name - - # Split out the data, convert timestamps - results = [] - for title, values in sorted(d.iteritems()): + for title, values in sorted(parsed.iteritems()): title, ydata = munge(args, title, values) - # Ignore entries if a list of regular expressions was given + + # ignore entries if a list of regular expressions was given if args.include and not [r for r in args.include if r.search(title)]: continue - prefix = title if prefix is None else common_prefix(prefix, title) - suffix = title if suffix is None else common_suffix(title, suffix) + if not 'wtperf' in title: + prefix = title if prefix is None else common_prefix(prefix, title) + suffix = title if suffix is None else common_suffix(title, suffix) results.append((title, ydata)) # Process titles, eliminate common prefixes and suffixes if prefix or suffix: new_results = [] for title, ydata in results: - title = title[len(prefix):] - if suffix: - title = title[:-len(suffix)] + if 'wtperf' not in title: + title = title[len(prefix):] + if suffix: + title = title[:-len(suffix)] new_results.append((title, ydata)) results = new_results # Are we just listing the results? if args.list: + print + print "Parsed stats:" for title, ydata in results: - print title + print " ", title sys.exit(0) - output_series(results) + output_series(results, args) # If the user wants the stats split up by prefix type do so. if args.all: for prefix in prefix_list: - output_series(results, prefix) + output_series(results, args, prefix) for group in groups.keys(): - output_series(results, group, groups[group]) + output_series(results, args, group, groups[group]) if __name__ == '__main__': |