diff options
45 files changed, 862 insertions, 1117 deletions
diff --git a/src/third_party/wiredtiger/.hgtags b/src/third_party/wiredtiger/.hgtags index 21c8b360301..ca12821f6c8 100644 --- a/src/third_party/wiredtiger/.hgtags +++ b/src/third_party/wiredtiger/.hgtags @@ -23,3 +23,5 @@ aff8aabe571be6db68e8bf44bf7670df5d55d1ff 1.5.0 b5c9f28d72fe1f835d24fe427e211a539f8709fe 1.5.2 03ab950d31edfe1b77aa6e7259e64dbdd3e17fe2 1.6.5 828cbc550d7136a3ee67be3781cade8e96ec3af7 2.8-rc3 +5c9acd2584f2657dec2a44fd8b54211bf9c21193 mongodb-2.8-rc5 +f40795b146bd35a623ef57de5b875a817925b7c9 mongodb-2.8-rc5 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/voxer-10k-short.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/overflow-10k-short.wtperf index 47228079db8..47228079db8 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/voxer-10k-short.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/overflow-10k-short.wtperf diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/voxer-10k.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/overflow-10k.wtperf index 9b4ed2acaee..9b4ed2acaee 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/voxer-10k.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/overflow-10k.wtperf diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/voxer-130k-short.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/overflow-130k-short.wtperf index 83f67062bf8..83f67062bf8 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/voxer-130k-short.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/overflow-130k-short.wtperf diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/voxer-130k.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/overflow-130k.wtperf index a3439f0c575..a3439f0c575 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/voxer-130k.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/overflow-130k.wtperf diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index 7e964252c83..0141526285c 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -559,6 +559,20 @@ common_wiredtiger_open = [ ]), ] +cursor_runtime_config = [ + Config('append', 'false', r''' + append the value as a new record, creating a new record + number key; valid only for cursors with record number keys''', + type='boolean'), + Config('overwrite', 'true', r''' + configures whether the cursor's insert, update and remove + methods check the existing state of the record. If \c overwrite + is \c false, WT_CURSOR::insert fails with ::WT_DUPLICATE_KEY + if the record exists, WT_CURSOR::update and WT_CURSOR::remove + fail with ::WT_NOTFOUND if the record does not exist''', + type='boolean'), +] + methods = { 'file.meta' : Method(file_meta), @@ -570,6 +584,8 @@ methods = { 'cursor.close' : Method([]), +'cursor.reconfigure' : Method(cursor_runtime_config), + 'session.close' : Method([]), 'session.compact' : Method([ @@ -600,11 +616,7 @@ methods = { 'session.log_printf' : Method([]), -'session.open_cursor' : Method([ - Config('append', 'false', r''' - append the value as a new record, creating a new record - number key; valid only for cursors with record number keys''', - type='boolean'), +'session.open_cursor' : Method(cursor_runtime_config + [ Config('bulk', 'false', r''' configure the cursor for bulk-loading, a fast, initial load path (see @ref tune_bulk_load for more information). Bulk-load @@ -641,13 +653,6 @@ methods = { WT_CURSOR::next and WT_CURSOR::close methods. See @ref cursor_random for details''', type='boolean'), - Config('overwrite', 'true', r''' - configures whether the cursor's insert, update and remove - methods check the existing state of the record. If \c overwrite - is \c false, WT_CURSOR::insert fails with ::WT_DUPLICATE_KEY - if the record exists, WT_CURSOR::update and WT_CURSOR::remove - fail with ::WT_NOTFOUND if the record does not exist''', - type='boolean'), Config('raw', 'false', r''' ignore the encodings for the key and value, manage data as if the formats were \c "u". See @ref cursor_raw for details''', diff --git a/src/third_party/wiredtiger/dist/flags.py b/src/third_party/wiredtiger/dist/flags.py index 2d68d932c02..a0e307debf6 100644 --- a/src/third_party/wiredtiger/dist/flags.py +++ b/src/third_party/wiredtiger/dist/flags.py @@ -90,6 +90,7 @@ flags = { 'CONN_CKPT_SYNC', 'CONN_EVICTION_RUN', 'CONN_LEAK_MEMORY', + 'CONN_LOG_SERVER_RUN', 'CONN_LSM_MERGE', 'CONN_PANIC', 'CONN_SERVER_RUN', diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index d3717d27331..6c658df8bf0 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -581,6 +581,7 @@ enum's env eof eop +equalp errhandler errno errv diff --git a/src/third_party/wiredtiger/dist/s_wtstats b/src/third_party/wiredtiger/dist/s_wtstats new file mode 100644 index 00000000000..e55157431f1 --- /dev/null +++ b/src/third_party/wiredtiger/dist/s_wtstats @@ -0,0 +1,37 @@ +#! /bin/sh + +# Create wtstats.template.html file + +t=__wt.$$ +trap 'rm -f $t; exit 0' 0 1 2 3 13 15 + +out=wtstats.html.template +# We require npm which may not be installed. +type npm > /dev/null 2>&1 || { + echo 's_wtstats: npm not found' >&2 + echo ' npm is part of node.js from http://nodejs.org' >&2 + exit 1 +} + +cd ../tools/template || exit 1 +rm -f ./$out +# Note: we don't do the npm install here, it downloads files +test -d ./node_modules || { + echo 's_wtstats: missing node_modules directory, install by:' >&2 + echo ' cd ../tools/template; npm install' >&2 + exit 1 +} +npm run build >$t 2>&1 || { + echo "s_wtstats: npm failed" >&2 + cat $t + exit 1 +} +test -f ./$out || { + echo "s_wtstats: $out not created" >&2 + cat $t + exit 1 +} +f=../$out +cmp ./$out $f > /dev/null 2>&1 || + (echo "Building tools/$out" && rm -f $f && cp ./$out $f) +exit 0 diff --git a/src/third_party/wiredtiger/lang/java/java_doc.i b/src/third_party/wiredtiger/lang/java/java_doc.i index 31bad525330..53785a3bab4 100644 --- a/src/third_party/wiredtiger/lang/java/java_doc.i +++ b/src/third_party/wiredtiger/lang/java/java_doc.i @@ -5,6 +5,7 @@ COPYDOC(__wt_cursor, WT_CURSOR, get_value) COPYDOC(__wt_cursor, WT_CURSOR, set_key) COPYDOC(__wt_cursor, WT_CURSOR, set_value) COPYDOC(__wt_cursor, WT_CURSOR, compare) +COPYDOC(__wt_cursor, WT_CURSOR, equals) COPYDOC(__wt_cursor, WT_CURSOR, next) COPYDOC(__wt_cursor, WT_CURSOR, prev) COPYDOC(__wt_cursor, WT_CURSOR, reset) @@ -14,6 +15,7 @@ COPYDOC(__wt_cursor, WT_CURSOR, insert) COPYDOC(__wt_cursor, WT_CURSOR, update) COPYDOC(__wt_cursor, WT_CURSOR, remove) COPYDOC(__wt_cursor, WT_CURSOR, close) +COPYDOC(__wt_cursor, WT_CURSOR, reconfigure) COPYDOC(__wt_async_op, WT_ASYNC_OP, get_key) COPYDOC(__wt_async_op, WT_ASYNC_OP, get_value) COPYDOC(__wt_async_op, WT_ASYNC_OP, set_key) diff --git a/src/third_party/wiredtiger/lang/java/wiredtiger.i b/src/third_party/wiredtiger/lang/java/wiredtiger.i index 09290a70c67..a92247c7ebf 100644 --- a/src/third_party/wiredtiger/lang/java/wiredtiger.i +++ b/src/third_party/wiredtiger/lang/java/wiredtiger.i @@ -293,6 +293,8 @@ WT_CLASS(struct __wt_async_op, WT_ASYNC_OP, op) %ignore __wt_cursor::compare(WT_CURSOR *, WT_CURSOR *, int *); %rename (compare_wrap) __wt_cursor::compare; +%ignore __wt_cursor::equals(WT_CURSOR *, WT_CURSOR *, int *); +%rename (equals_wrap) __wt_cursor::equals; %rename (AsyncOpType) WT_ASYNC_OPTYPE; %rename (getKeyFormat) __wt_async_op::getKey_format; %rename (getValueFormat) __wt_async_op::getValue_format; @@ -1134,6 +1136,13 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler}; return cmp; } + int equals_wrap(JNIEnv *jenv, WT_CURSOR *other) { + int cmp, ret = $self->equals($self, other, &cmp); + if (ret != 0) + throwWiredTigerException(jenv, ret); + return cmp; + } + %javamethodmodifiers java_init "protected"; int java_init(jobject jcursor) { JAVA_CALLBACK *jcb = (JAVA_CALLBACK *)$self->lang_private; @@ -1622,6 +1631,16 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler}; } /** + * Compare this cursor's position to another Cursor. + * + * \return The result of the comparison. + */ + public int equals(Cursor other) + throws WiredTigerException { + return equals_wrap(other); + } + + /** * Retrieve the next item in the table. * * \return The result of the comparison. diff --git a/src/third_party/wiredtiger/lang/python/wiredtiger.i b/src/third_party/wiredtiger/lang/python/wiredtiger.i index de5afb0a0fa..bf726ceac0a 100644 --- a/src/third_party/wiredtiger/lang/python/wiredtiger.i +++ b/src/third_party/wiredtiger/lang/python/wiredtiger.i @@ -390,6 +390,7 @@ NOTFOUND_OK(__wt_cursor::search) NOTFOUND_OK(__wt_cursor::update) COMPARE_OK(__wt_cursor::compare) +COMPARE_OK(__wt_cursor::equals) COMPARE_NOTFOUND_OK(__wt_cursor::search_near) /* Lastly, some methods need no (additional) error checking. */ @@ -424,6 +425,7 @@ COMPARE_NOTFOUND_OK(__wt_cursor::search_near) /* Next, override methods that return integers via arguments. */ %ignore __wt_cursor::compare(WT_CURSOR *, WT_CURSOR *, int *); +%ignore __wt_cursor::equals(WT_CURSOR *, WT_CURSOR *, int *); %ignore __wt_cursor::search_near(WT_CURSOR *, int *); /* SWIG magic to turn Python byte strings into data / size. */ @@ -682,7 +684,7 @@ typedef int int_void; return (ret); } - /* compare and search_near need special handling. */ + /* compare: special handling. */ int compare(WT_CURSOR *other) { int cmp = 0; int ret = 0; @@ -700,21 +702,40 @@ typedef int int_void; * Map less-than-zero to -1 and greater-than-zero to 1 * to avoid colliding with other errors. */ - ret = ((ret != 0) ? ret : - (cmp < 0) ? -1 : (cmp == 0) ? 0 : 1); + ret = (ret != 0) ? ret : + ((cmp < 0) ? -1 : (cmp == 0) ? 0 : 1); } return (ret); } + /* equals: special handling. */ + int equals(WT_CURSOR *other) { + int cmp = 0; + int ret = 0; + if (other == NULL) { + SWIG_Error(SWIG_NullReferenceError, + "in method 'Cursor_equals', " + "argument 1 of type 'struct __wt_cursor *' " + "is None"); + ret = EINVAL; /* any non-zero value will do. */ + } + else { + ret = $self->equals($self, other, &cmp); + if (ret == 0) + ret = cmp; + } + return (ret); + } + + /* search_near: special handling. */ int search_near() { int cmp = 0; int ret = $self->search_near($self, &cmp); /* * Map less-than-zero to -1 and greater-than-zero to 1 to avoid - * colliding with WT_NOTFOUND. + * colliding with other errors. */ - return ((ret != 0) ? ret : - (cmp < 0) ? -1 : (cmp == 0) ? 0 : 1); + return ((ret != 0) ? ret : (cmp < 0) ? -1 : (cmp == 0) ? 0 : 1); } int _freecb() { diff --git a/src/third_party/wiredtiger/src/block/block_compact.c b/src/third_party/wiredtiger/src/block/block_compact.c index c7f2b09aa23..79494a274a9 100644 --- a/src/third_party/wiredtiger/src/block/block_compact.c +++ b/src/third_party/wiredtiger/src/block/block_compact.c @@ -137,8 +137,6 @@ __wt_block_compact_page_skip(WT_SESSION_IMPL *session, /* Crack the cookie. */ WT_RET(__wt_block_buffer_to_addr(block, addr, &offset, &size, &cksum)); - __wt_spin_lock(session, &block->live_lock); - /* * If this block is in the last 10% of the file and there's a block on * the available list that's in the first 90% of the file, rewrite the @@ -146,6 +144,7 @@ __wt_block_compact_page_skip(WT_SESSION_IMPL *session, * the block would extend the file), but there's an obvious race if the * file is sufficiently busy. */ + __wt_spin_lock(session, &block->live_lock); ninety = fh->size - fh->size / 10; if (offset > ninety) { el = &block->live.avail; @@ -155,7 +154,6 @@ __wt_block_compact_page_skip(WT_SESSION_IMPL *session, break; } } - __wt_spin_unlock(session, &block->live_lock); return (ret); @@ -176,7 +174,7 @@ __block_dump_avail(WT_SESSION_IMPL *session, WT_BLOCK *block) el = &block->live.avail; size = block->fh->size; - WT_RET(__wt_verbose(session, WT_VERB_BLOCK, + WT_RET(__wt_verbose(session, WT_VERB_COMPACT, "file size %" PRIuMAX "MB (%" PRIuMAX ") with %" PRIuMAX "%% space available %" PRIuMAX "MB (%" PRIuMAX ")", (uintmax_t)size / WT_MEGABYTE, (uintmax_t)size, @@ -202,7 +200,7 @@ __block_dump_avail(WT_SESSION_IMPL *session, WT_BLOCK *block) #ifdef __VERBOSE_OUTPUT_PERCENTILE for (i = 0; i < WT_ELEMENTS(percentile); ++i) { v = percentile[i] * 512; - WT_RET(__wt_verbose(session, WT_VERB_BLOCK, + WT_RET(__wt_verbose(session, WT_VERB_COMPACT, "%2u%%: %12" PRIuMAX "MB, (%" PRIuMAX "B, %" PRIuMAX "%%)", i, (uintmax_t)v / WT_MEGABYTE, (uintmax_t)v, @@ -211,7 +209,7 @@ __block_dump_avail(WT_SESSION_IMPL *session, WT_BLOCK *block) #endif for (i = 0; i < WT_ELEMENTS(decile); ++i) { v = decile[i] * 512; - WT_RET(__wt_verbose(session, WT_VERB_BLOCK, + WT_RET(__wt_verbose(session, WT_VERB_COMPACT, "%2u%%: %12" PRIuMAX "MB, (%" PRIuMAX "B, %" PRIuMAX "%%)", i * 10, (uintmax_t)v / WT_MEGABYTE, (uintmax_t)v, diff --git a/src/third_party/wiredtiger/src/btree/bt_cursor.c b/src/third_party/wiredtiger/src/btree/bt_cursor.c index 4aee1883ae0..704b258a7dd 100644 --- a/src/third_party/wiredtiger/src/btree/bt_cursor.c +++ b/src/third_party/wiredtiger/src/btree/bt_cursor.c @@ -800,16 +800,19 @@ err: if (ret != 0) int __wt_btcur_compare(WT_CURSOR_BTREE *a_arg, WT_CURSOR_BTREE *b_arg, int *cmpp) { - WT_BTREE *btree; WT_CURSOR *a, *b; WT_SESSION_IMPL *session; a = (WT_CURSOR *)a_arg; b = (WT_CURSOR *)b_arg; - btree = a_arg->btree; session = (WT_SESSION_IMPL *)a->session; - switch (btree->type) { + /* Confirm both cursors reference the same object. */ + if (a_arg->btree != b_arg->btree) + WT_RET_MSG( + session, EINVAL, "Cursors must reference the same object"); + + switch (a_arg->btree->type) { case BTREE_COL_FIX: case BTREE_COL_VAR: /* @@ -826,7 +829,7 @@ __wt_btcur_compare(WT_CURSOR_BTREE *a_arg, WT_CURSOR_BTREE *b_arg, int *cmpp) break; case BTREE_ROW: WT_RET(__wt_compare( - session, btree->collator, &a->key, &b->key, cmpp)); + session, a_arg->btree->collator, &a->key, &b->key, cmpp)); break; WT_ILLEGAL_VALUE(session); } @@ -837,7 +840,7 @@ __wt_btcur_compare(WT_CURSOR_BTREE *a_arg, WT_CURSOR_BTREE *b_arg, int *cmpp) * __cursor_equals -- * Return if two cursors reference the same row. */ -static int +static inline int __cursor_equals(WT_CURSOR_BTREE *a, WT_CURSOR_BTREE *b) { switch (a->btree->type) { @@ -867,6 +870,43 @@ __cursor_equals(WT_CURSOR_BTREE *a, WT_CURSOR_BTREE *b) } /* + * __wt_btcur_equals -- + * Return an equality comparison between two cursors. + */ +int +__wt_btcur_equals( + WT_CURSOR_BTREE *a_arg, WT_CURSOR_BTREE *b_arg, int *equalp) +{ + WT_CURSOR *a, *b; + WT_SESSION_IMPL *session; + int cmp; + + a = (WT_CURSOR *)a_arg; + b = (WT_CURSOR *)b_arg; + session = (WT_SESSION_IMPL *)a->session; + + /* Confirm both cursors reference the same object. */ + if (a_arg->btree != b_arg->btree) + WT_RET_MSG( + session, EINVAL, "Cursors must reference the same object"); + + /* + * The reason for an equals method is because we can avoid doing + * a full key comparison in some cases. If both cursors point into the + * tree, take the fast path, otherwise fall back to the slower compare + * method; in both cases, return 1 if the cursors are equal, 0 if they + * are not. + */ + if (F_ISSET(a, WT_CURSTD_KEY_INT) && F_ISSET(b, WT_CURSTD_KEY_INT)) + *equalp = __cursor_equals(a_arg, b_arg); + else { + WT_RET(__wt_btcur_compare(a_arg, b_arg, &cmp)); + *equalp = (cmp == 0) ? 1 : 0; + } + return (0); +} + +/* * __cursor_truncate -- * Discard a cursor range from row-store or variable-width column-store * tree. diff --git a/src/third_party/wiredtiger/src/btree/bt_split.c b/src/third_party/wiredtiger/src/btree/bt_split.c index 4041bbb71b0..395ff447035 100644 --- a/src/third_party/wiredtiger/src/btree/bt_split.c +++ b/src/third_party/wiredtiger/src/btree/bt_split.c @@ -257,12 +257,13 @@ __split_ovfl_key_cleanup(WT_SESSION_IMPL *session, WT_PAGE *page, WT_REF *ref) } /* - * __split_ref_instantiate -- - * Instantiate key/address pairs in memory in service of a split. + * __split_ref_deepen_move -- + * Move a WT_REF from a parent to a child in service of a split to deepen + * the tree, including updating the accounting information. */ static int -__split_ref_instantiate(WT_SESSION_IMPL *session, - WT_PAGE *page, WT_REF *ref, size_t *parent_decrp, size_t *child_incrp) +__split_ref_deepen_move(WT_SESSION_IMPL *session, + WT_PAGE *parent, WT_REF *ref, size_t *parent_decrp, size_t *child_incrp) { WT_ADDR *addr; WT_CELL_UNPACK unpack; @@ -279,8 +280,6 @@ __split_ref_instantiate(WT_SESSION_IMPL *session, * of child pages, and so we can no longer reference the block image * that remains with the page being split. * - * Track how much memory the parent is losing and the child gaining. - * * No locking is required to update the WT_REF structure because we're * the only thread splitting the parent page, and there's no way for * readers to race with our updates of single pointers. The changes @@ -289,13 +288,13 @@ __split_ref_instantiate(WT_SESSION_IMPL *session, * * Row-store keys, first. */ - if (page->type == WT_PAGE_ROW_INT) { + if (parent->type == WT_PAGE_ROW_INT) { if ((ikey = __wt_ref_key_instantiated(ref)) == NULL) { - __wt_ref_key(page, ref, &key, &size); + __wt_ref_key(parent, ref, &key, &size); WT_RET(__wt_row_ikey(session, 0, key, size, &ikey)); ref->key.ikey = ikey; } else { - WT_RET(__split_ovfl_key_cleanup(session, page, ref)); + WT_RET(__split_ovfl_key_cleanup(session, parent, ref)); WT_MEMSIZE_ADD(*parent_decrp, sizeof(WT_IKEY) + ikey->size); } @@ -307,12 +306,8 @@ __split_ref_instantiate(WT_SESSION_IMPL *session, * address has been instantiated, there's no work to do. Otherwise, * get the address from the on-page cell. */ - if ((addr = ref->addr) == NULL) - return (0); - if (__wt_off_page(page, addr)) - WT_MEMSIZE_TRANSFER(*parent_decrp, *child_incrp, - sizeof(WT_ADDR) + addr->size); - else { + addr = ref->addr; + if (addr != NULL && !__wt_off_page(parent, addr)) { __wt_cell_unpack((WT_CELL *)ref->addr, &unpack); WT_RET(__wt_calloc_one(session, &addr)); if ((ret = __wt_strndup( @@ -324,8 +319,11 @@ __split_ref_instantiate(WT_SESSION_IMPL *session, addr->type = unpack.raw == WT_CELL_ADDR_INT ? WT_ADDR_INT : WT_ADDR_LEAF; ref->addr = addr; - WT_MEMSIZE_ADD(*child_incrp, sizeof(WT_ADDR) + addr->size); } + + /* And finally, the WT_REF itself. */ + WT_MEMSIZE_TRANSFER(*parent_decrp, *child_incrp, sizeof(WT_REF)); + return (0); } @@ -502,12 +500,9 @@ __split_deepen(WT_SESSION_IMPL *session, WT_PAGE *parent, uint32_t children) child_incr = 0; child_pindex = WT_INTL_INDEX_COPY(child); for (child_refp = child_pindex->index, j = 0; j < slots; ++j) { - WT_ERR(__split_ref_instantiate(session, + WT_ERR(__split_ref_deepen_move(session, parent, *parent_refp, &parent_decr, &child_incr)); *child_refp++ = *parent_refp++; - - WT_MEMSIZE_TRANSFER( - parent_decr, child_incr, sizeof(WT_REF)); } __wt_cache_page_inmem_incr(session, child, child_incr); } @@ -600,9 +595,10 @@ __split_deepen(WT_SESSION_IMPL *session, WT_PAGE *parent, uint32_t children) * be using the new index. */ size = sizeof(WT_PAGE_INDEX) + pindex->entries * sizeof(WT_REF *); - WT_MEMSIZE_ADD(parent_decr, size); WT_ERR(__split_safe_free(session, 0, pindex, size)); + WT_MEMSIZE_ADD(parent_decr, size); +#if 0 /* * Adjust the parent's memory footprint. This may look odd, but we * have already taken the allocation overhead into account, and an @@ -611,6 +607,19 @@ __split_deepen(WT_SESSION_IMPL *session, WT_PAGE *parent, uint32_t children) */ __wt_cache_page_inmem_incr(session, parent, parent_incr); __wt_cache_page_inmem_decr(session, parent, parent_decr); +#else + /* + * XXX + * The code to track page sizes is fundamentally flawed in the face of + * splits: for example, we don't add in an overhead allocation constant + * when allocating WT_REF structures as pages are created, but the + * calculations during split assume that correction. For now, ignore + * our carefully calculated values and force the internal page size to + * 5% of its current value. + */ + size = parent->memory_footprint - (parent->memory_footprint / 20); + __wt_cache_page_inmem_decr(session, parent, size); +#endif if (0) { err: __wt_free_ref_index(session, parent, alloc_index, 1); @@ -774,13 +783,11 @@ __wt_multi_to_ref(WT_SESSION_IMPL *session, * the confusion. */ WT_RET(__wt_calloc_one(session, &addr)); - WT_MEMSIZE_ADD(incr, sizeof(WT_ADDR)); ref->addr = addr; addr->size = multi->addr.size; addr->type = multi->addr.type; WT_RET(__wt_strndup(session, multi->addr.addr, addr->size, &addr->addr)); - WT_MEMSIZE_ADD(incr, addr->size); } else WT_RET(__split_multi_inmem(session, page, ref, multi)); @@ -973,14 +980,17 @@ __split_parent(WT_SESSION_IMPL *session, WT_REF *ref, WT_REF **ref_new, WT_TRET(__split_ovfl_key_cleanup( session, parent, next_ref)); ikey = __wt_ref_key_instantiated(next_ref); - if (ikey != NULL) - WT_TRET(__split_safe_free(session, 0, - ikey, - sizeof(WT_IKEY) + ikey->size)); + if (ikey != NULL) { + size = sizeof(WT_IKEY) + ikey->size; + WT_TRET(__split_safe_free( + session, 0, ikey, size)); + WT_MEMSIZE_ADD(parent_decr, size); + } } WT_TRET(__split_safe_free( session, 0, next_ref, sizeof(WT_REF))); + WT_MEMSIZE_ADD(parent_decr, sizeof(WT_REF)); } } @@ -1096,15 +1106,16 @@ __wt_split_insert(WT_SESSION_IMPL *session, WT_REF *ref, int *splitp) WT_PAGE *page, *right; WT_REF *child, *split_ref[2] = { NULL, NULL }; WT_UPDATE *upd; - size_t page_decr, parent_incr, right_incr, size; + size_t page_decr, parent_decr, parent_incr, right_incr; int i; *splitp = 0; btree = S2BT(session); page = ref->page; + ikey = NULL; right = NULL; - page_decr = parent_incr = right_incr = 0; + page_decr = parent_decr = parent_incr = right_incr = 0; /* * Check for pages with append-only workloads. A common application @@ -1205,9 +1216,19 @@ __wt_split_insert(WT_SESSION_IMPL *session, WT_REF *ref, int *splitp) WT_ERR(__wt_row_ikey(session, 0, WT_INSERT_KEY(moved_ins), WT_INSERT_KEY_SIZE(moved_ins), &child->key.ikey)); + + /* + * We're swapping WT_REFs in the parent, adjust the accounting, and + * row store pages may have instantiated keys. + */ WT_MEMSIZE_ADD(parent_incr, sizeof(WT_REF)); - WT_MEMSIZE_ADD(parent_incr, sizeof(WT_IKEY)); - WT_MEMSIZE_ADD(parent_incr, WT_INSERT_KEY_SIZE(moved_ins)); + WT_MEMSIZE_ADD( + parent_incr, sizeof(WT_IKEY) + WT_INSERT_KEY_SIZE(moved_ins)); + WT_MEMSIZE_ADD(parent_decr, sizeof(WT_REF)); + if (page->type == WT_PAGE_ROW_LEAF || page->type == WT_PAGE_ROW_INT) + if ((ikey = __wt_ref_key_instantiated(ref)) != NULL) + WT_MEMSIZE_ADD( + parent_decr, sizeof(WT_IKEY) + ikey->size); /* The new page is dirty by definition. */ WT_ERR(__wt_page_modify_init(session, right)); @@ -1229,14 +1250,11 @@ __wt_split_insert(WT_SESSION_IMPL *session, WT_REF *ref, int *splitp) */ for (i = 0; i < WT_SKIP_MAXDEPTH && ins_head->tail[i] == moved_ins; ++i) ; - size = ((size_t)i - 1) * sizeof(WT_INSERT *); - size += sizeof(WT_INSERT) + WT_INSERT_KEY_SIZE(moved_ins); + WT_MEMSIZE_TRANSFER(page_decr, right_incr, sizeof(WT_INSERT) + + (size_t)i * sizeof(WT_INSERT *) + WT_INSERT_KEY_SIZE(moved_ins)); for (upd = moved_ins->upd; upd != NULL; upd = upd->next) - size += sizeof(WT_UPDATE) + upd->size; - WT_MEMSIZE_ADD(right_incr, size); - WT_MEMSIZE_ADD(page_decr, size); - __wt_cache_page_inmem_decr(session, page, page_decr); - __wt_cache_page_inmem_incr(session, right, right_incr); + WT_MEMSIZE_TRANSFER( + page_decr, right_incr, sizeof(WT_UPDATE) + upd->size); /* * Allocation operations completed, move the last insert list item from @@ -1323,10 +1341,23 @@ __wt_split_insert(WT_SESSION_IMPL *session, WT_REF *ref, int *splitp) #endif /* - * Split into the parent. + * Save the transaction ID when the split happened. Application + * threads will not try to forcibly evict the page again until + * all concurrent transactions commit. + */ + page->modify->inmem_split_txn = __wt_txn_new_id(session); + + /* Update the page accounting. */ + __wt_cache_page_inmem_decr(session, page, page_decr); + __wt_cache_page_inmem_incr(session, right, right_incr); + + /* + * Split into the parent. After this, the original page is no + * longer locked, so we cannot safely look at it. */ + page = NULL; if ((ret = __split_parent( - session, ref, split_ref, 2, 0, parent_incr, 0, 0)) != 0) { + session, ref, split_ref, 2, parent_decr, parent_incr, 0, 0)) != 0) { /* * Move the insert list element back to the original page list. * For simplicity, the previous skip list pointers originally @@ -1349,13 +1380,6 @@ __wt_split_insert(WT_SESSION_IMPL *session, WT_REF *ref, int *splitp) WT_ERR(ret); } - /* - * Save the transaction ID when the split happened. Application - * threads will not try to forcibly evict the page again until - * all concurrent transactions commit. - */ - page->modify->inmem_split_txn = __wt_txn_new_id(session); - /* Let our caller know that we split. */ *splitp = 1; @@ -1367,13 +1391,8 @@ __wt_split_insert(WT_SESSION_IMPL *session, WT_REF *ref, int *splitp) * structure and instantiated key, there may be threads using them. * Add them to the session discard list, to be freed once we know it's * safe. - * - * After the split, we're going to discard the WT_REF, account for the - * change in memory footprint. Row store pages have keys that may be - * instantiated, check for that. */ - if ((page->type == WT_PAGE_ROW_LEAF || page->type == WT_PAGE_ROW_INT) && - (ikey = __wt_ref_key_instantiated(ref)) != NULL) + if (ikey != NULL) WT_TRET(__split_safe_free( session, 0, ikey, sizeof(WT_IKEY) + ikey->size)); WT_TRET(__split_safe_free(session, 0, ref, sizeof(WT_REF))); @@ -1458,7 +1477,7 @@ __wt_split_multi(WT_SESSION_IMPL *session, WT_REF *ref, int exclusive) WT_PAGE *page; WT_PAGE_MODIFY *mod; WT_REF **ref_new; - size_t ikey_size, parent_decr, parent_incr; + size_t parent_decr, parent_incr; uint32_t i, new_entries; page = ref->page; @@ -1466,7 +1485,7 @@ __wt_split_multi(WT_SESSION_IMPL *session, WT_REF *ref, int exclusive) new_entries = mod->mod_multi_entries; ikey = NULL; - ikey_size = parent_decr = parent_incr = 0; + parent_decr = parent_incr = 0; /* * Convert the split page's multiblock reconciliation information into @@ -1482,12 +1501,11 @@ __wt_split_multi(WT_SESSION_IMPL *session, WT_REF *ref, int exclusive) * change in memory footprint. Row store pages have keys that may be * instantiated, check for that. */ - if ((page->type == WT_PAGE_ROW_LEAF || page->type == WT_PAGE_ROW_INT) && - (ikey = __wt_ref_key_instantiated(ref)) != NULL) { - ikey_size = sizeof(WT_IKEY) + ikey->size; - WT_MEMSIZE_ADD(parent_decr, ikey_size); - } WT_MEMSIZE_ADD(parent_decr, sizeof(WT_REF)); + if (page->type == WT_PAGE_ROW_LEAF || page->type == WT_PAGE_ROW_INT) + if ((ikey = __wt_ref_key_instantiated(ref)) != NULL) + WT_MEMSIZE_ADD( + parent_decr, sizeof(WT_IKEY) + ikey->size); /* Split into the parent. */ WT_ERR(__split_parent(session, @@ -1514,7 +1532,8 @@ __wt_split_multi(WT_SESSION_IMPL *session, WT_REF *ref, int exclusive) * safe. */ if (ikey != NULL) - WT_TRET(__split_safe_free(session, exclusive, ikey, ikey_size)); + WT_TRET(__split_safe_free( + session, exclusive, ikey, sizeof(WT_IKEY) + ikey->size)); WT_TRET(__split_safe_free(session, exclusive, ref, sizeof(WT_REF))); /* diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index 9d2e5ba9c86..ec44e8839b0 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -110,6 +110,12 @@ static const WT_CONFIG_CHECK confchk_connection_reconfigure[] = { { NULL, NULL, NULL, NULL } }; +static const WT_CONFIG_CHECK confchk_cursor_reconfigure[] = { + { "append", "boolean", NULL, NULL }, + { "overwrite", "boolean", NULL, NULL }, + { NULL, NULL, NULL, NULL } +}; + static const WT_CONFIG_CHECK confchk_file_meta[] = { { "allocation_size", "int", "min=512B,max=128MB", NULL }, { "app_metadata", "string", NULL, NULL }, @@ -561,6 +567,10 @@ static const WT_CONFIG_ENTRY config_entries[] = { "", NULL }, + { "cursor.reconfigure", + "append=0,overwrite=", + confchk_cursor_reconfigure + }, { "file.meta", "allocation_size=4KB,app_metadata=,block_allocation=best," "block_compressor=,cache_resident=0,checkpoint=,checkpoint_lsn=," diff --git a/src/third_party/wiredtiger/src/conn/conn_log.c b/src/third_party/wiredtiger/src/conn/conn_log.c index d15b27f9bc9..f70a9a4a60c 100644 --- a/src/third_party/wiredtiger/src/conn/conn_log.c +++ b/src/third_party/wiredtiger/src/conn/conn_log.c @@ -278,6 +278,72 @@ err: } /* + * __log_close_server -- + * The log close server thread. + */ +static void * +__log_close_server(void *arg) +{ + WT_CONNECTION_IMPL *conn; + WT_DECL_RET; + WT_FH *close_fh; + WT_LOG *log; + WT_LSN close_end_lsn, close_lsn; + WT_SESSION_IMPL *session; + int locked; + + session = arg; + conn = S2C(session); + log = conn->log; + locked = 0; + while (F_ISSET(conn, WT_CONN_LOG_SERVER_RUN)) { + /* + * If there is a log file to close, make sure any outstanding + * write operations have completed, then fsync and close it. + */ + if ((close_fh = log->log_close_fh) != NULL && + (ret = __wt_log_extract_lognum(session, close_fh->name, + &close_lsn.file)) == 0 && + close_lsn.file < log->write_lsn.file) { + /* + * We've copied the file handle, clear out the one in + * log structure to allow it to be set again. + */ + log->log_close_fh = NULL; + /* + * Set the close_end_lsn to the LSN immediately after + * ours. That is, the beginning of the next log file. + * We need to know the LSN file number of our own close + * in case earlier calls are still in progress and the + * next one to move the sync_lsn into the next file for + * later syncs. + */ + close_lsn.offset = 0; + close_end_lsn = close_lsn; + close_end_lsn.file++; + WT_ERR(__wt_fsync(session, close_fh)); + __wt_spin_lock(session, &log->log_sync_lock); + locked = 1; + WT_ERR(__wt_close(session, close_fh)); + log->sync_lsn = close_end_lsn; + WT_ERR(__wt_cond_signal(session, log->log_sync_cond)); + locked = 0; + __wt_spin_unlock(session, &log->log_sync_lock); + } else + /* Wait until the next event. */ + WT_ERR(__wt_cond_wait(session, + conn->log_close_cond, 10000)); + } + + if (0) { +err: __wt_err(session, ret, "log close server error"); + } + if (locked) + __wt_spin_unlock(session, &log->log_sync_lock); + return (NULL); +} + +/* * __log_server -- * The log server thread. */ @@ -294,7 +360,7 @@ __log_server(void *arg) conn = S2C(session); log = conn->log; locked = 0; - while (F_ISSET(conn, WT_CONN_SERVER_RUN)) { + while (F_ISSET(conn, WT_CONN_LOG_SERVER_RUN)) { /* * Perform log pre-allocation. */ @@ -322,7 +388,7 @@ __log_server(void *arg) } if (0) { -err: __wt_err(session, ret, "log archive server error"); +err: __wt_err(session, ret, "log server error"); } if (locked) (void)__wt_writeunlock(session, log->log_archive_lock); @@ -386,7 +452,7 @@ __wt_logmgr_create(WT_SESSION_IMPL *session, const char *cfg[]) /* * __wt_logmgr_open -- - * Start the log subsystem and archive server thread. + * Start the log service threads. */ int __wt_logmgr_open(WT_SESSION_IMPL *session) @@ -396,14 +462,33 @@ __wt_logmgr_open(WT_SESSION_IMPL *session) conn = S2C(session); /* If no log thread services are configured, we're done. */ - if (!FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED) || - !FLD_ISSET(conn->log_flags, + if (!FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED)) + return (0); + + /* + * Start the log close thread. It is not configurable. + * If logging is enabled, this thread runs. + */ + WT_RET(__wt_open_internal_session( + conn, "log-close-server", 0, 0, &conn->log_close_session)); + WT_RET(__wt_cond_alloc(conn->log_close_session, + "log close server", 0, &conn->log_close_cond)); + + /* + * Start the thread. + */ + WT_RET(__wt_thread_create(conn->log_close_session, + &conn->log_close_tid, __log_close_server, conn->log_close_session)); + conn->log_close_tid_set = 1; + + /* If no log thread services are configured, we're done. */ + if (!FLD_ISSET(conn->log_flags, (WT_CONN_LOG_ARCHIVE | WT_CONN_LOG_PREALLOC))) return (0); /* * If a log server thread exists, the user may have reconfigured - * archiving ore pre-allocation. Signal the thread. Otherwise the + * archiving or pre-allocation. Signal the thread. Otherwise the * user wants archiving and/or allocation and we need to start up * the thread. */ @@ -457,6 +542,17 @@ __wt_logmgr_destroy(WT_SESSION_IMPL *session) conn->log_tid_set = 0; } WT_TRET(__wt_cond_destroy(session, &conn->log_cond)); + if (conn->log_close_tid_set) { + WT_TRET(__wt_cond_signal(session, conn->log_close_cond)); + WT_TRET(__wt_thread_join(session, conn->log_close_tid)); + conn->log_close_tid_set = 0; + } + WT_TRET(__wt_cond_destroy(session, &conn->log_close_cond)); + if (conn->log_close_session != NULL) { + wt_session = &conn->log_close_session->iface; + WT_TRET(wt_session->close(wt_session, NULL)); + conn->log_close_session = NULL; + } WT_TRET(__wt_log_close(session)); diff --git a/src/third_party/wiredtiger/src/conn/conn_open.c b/src/third_party/wiredtiger/src/conn/conn_open.c index b425376d6ae..ab873cc36a9 100644 --- a/src/third_party/wiredtiger/src/conn/conn_open.c +++ b/src/third_party/wiredtiger/src/conn/conn_open.c @@ -25,7 +25,7 @@ __wt_connection_open(WT_CONNECTION_IMPL *conn, const char *cfg[]) * Tell internal server threads to run: this must be set before opening * any sessions. */ - F_SET(conn, WT_CONN_SERVER_RUN); + F_SET(conn, WT_CONN_SERVER_RUN | WT_CONN_LOG_SERVER_RUN); /* WT_SESSION_IMPL array. */ WT_RET(__wt_calloc(session, @@ -130,6 +130,7 @@ __wt_connection_close(WT_CONNECTION_IMPL *conn) if (FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED)) WT_TRET(__wt_txn_checkpoint_log( session, 1, WT_TXN_LOG_CKPT_STOP, NULL)); + F_CLR(conn, WT_CONN_LOG_SERVER_RUN); WT_TRET(__wt_logmgr_destroy(session)); /* Free memory for collators, compressors, data sources. */ diff --git a/src/third_party/wiredtiger/src/cursor/cur_backup.c b/src/third_party/wiredtiger/src/cursor/cur_backup.c index 3c4c5322534..c85e9e24fb2 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_backup.c +++ b/src/third_party/wiredtiger/src/cursor/cur_backup.c @@ -109,6 +109,7 @@ __wt_curbackup_open(WT_SESSION_IMPL *session, __wt_cursor_notsup, /* set-key */ __wt_cursor_notsup, /* set-value */ __wt_cursor_notsup, /* compare */ + __wt_cursor_notsup, /* equals */ __curbackup_next, /* next */ __wt_cursor_notsup, /* prev */ __curbackup_reset, /* reset */ @@ -117,6 +118,7 @@ __wt_curbackup_open(WT_SESSION_IMPL *session, __wt_cursor_notsup, /* insert */ __wt_cursor_notsup, /* update */ __wt_cursor_notsup, /* remove */ + __wt_cursor_notsup, /* reconfigure */ __curbackup_close); /* close */ WT_CURSOR *cursor; WT_CURSOR_BACKUP *cb; diff --git a/src/third_party/wiredtiger/src/cursor/cur_config.c b/src/third_party/wiredtiger/src/cursor/cur_config.c index f011f6db126..348cfbab1dd 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_config.c +++ b/src/third_party/wiredtiger/src/cursor/cur_config.c @@ -32,6 +32,7 @@ __wt_curconfig_open(WT_SESSION_IMPL *session, __wt_cursor_set_key, /* set-key */ __wt_cursor_set_value, /* set-value */ __wt_cursor_notsup, /* compare */ + __wt_cursor_notsup, /* equals */ __wt_cursor_notsup, /* next */ __wt_cursor_notsup, /* prev */ __wt_cursor_noop, /* reset */ @@ -40,6 +41,7 @@ __wt_curconfig_open(WT_SESSION_IMPL *session, __wt_cursor_notsup, /* insert */ __wt_cursor_notsup, /* update */ __wt_cursor_notsup, /* remove */ + __wt_cursor_notsup, /* reconfigure */ __curconfig_close); WT_CURSOR_CONFIG *cconfig; WT_CURSOR *cursor; diff --git a/src/third_party/wiredtiger/src/cursor/cur_ds.c b/src/third_party/wiredtiger/src/cursor/cur_ds.c index 2cb791de85d..cc3e23570d5 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_ds.c +++ b/src/third_party/wiredtiger/src/cursor/cur_ds.c @@ -454,6 +454,7 @@ __wt_curds_open( __wt_cursor_set_key, /* set-key */ __wt_cursor_set_value, /* set-value */ __curds_compare, /* compare */ + __wt_cursor_equal, /* equals */ __curds_next, /* next */ __curds_prev, /* prev */ __curds_reset, /* reset */ @@ -462,6 +463,7 @@ __wt_curds_open( __curds_insert, /* insert */ __curds_update, /* update */ __curds_remove, /* remove */ + __wt_cursor_notsup, /* reconfigure */ __curds_close); /* close */ WT_CONFIG_ITEM cval, metadata; WT_CURSOR *cursor, *source; diff --git a/src/third_party/wiredtiger/src/cursor/cur_dump.c b/src/third_party/wiredtiger/src/cursor/cur_dump.c index 2be1d91f0f3..00281054d22 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_dump.c +++ b/src/third_party/wiredtiger/src/cursor/cur_dump.c @@ -352,6 +352,7 @@ __wt_curdump_create(WT_CURSOR *child, WT_CURSOR *owner, WT_CURSOR **cursorp) __curdump_set_key, /* set-key */ __curdump_set_value, /* set-value */ __wt_cursor_notsup, /* compare */ + __wt_cursor_notsup, /* equals */ __curdump_next, /* next */ __curdump_prev, /* prev */ __curdump_reset, /* reset */ @@ -360,6 +361,7 @@ __wt_curdump_create(WT_CURSOR *child, WT_CURSOR *owner, WT_CURSOR **cursorp) __curdump_insert, /* insert */ __curdump_update, /* update */ __curdump_remove, /* remove */ + __wt_cursor_notsup, /* reconfigure */ __curdump_close); /* close */ WT_CURSOR *cursor; WT_CURSOR_DUMP *cdump; diff --git a/src/third_party/wiredtiger/src/cursor/cur_file.c b/src/third_party/wiredtiger/src/cursor/cur_file.c index 10d1151c30c..2f24d8ed59a 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_file.c +++ b/src/third_party/wiredtiger/src/cursor/cur_file.c @@ -46,10 +46,11 @@ __curfile_compare(WT_CURSOR *a, WT_CURSOR *b, int *cmpp) CURSOR_API_CALL(a, session, compare, cbt->btree); /* - * Confirm both cursors refer to the same source and have keys, then - * call the underlying object to compare them. + * Check both cursors are a "file:" type then call the underlying + * function, it can handle cursors pointing to different objects. */ - if (strcmp(a->internal_uri, b->internal_uri) != 0) + if (!WT_PREFIX_MATCH(a->internal_uri, "file:") || + !WT_PREFIX_MATCH(b->internal_uri, "file:")) WT_ERR_MSG(session, EINVAL, "Cursors must reference the same object"); @@ -63,6 +64,38 @@ err: API_END_RET(session, ret); } /* + * __curfile_equals -- + * WT_CURSOR->equals method for the btree cursor type. + */ +static int +__curfile_equals(WT_CURSOR *a, WT_CURSOR *b, int *equalp) +{ + WT_CURSOR_BTREE *cbt; + WT_DECL_RET; + WT_SESSION_IMPL *session; + + cbt = (WT_CURSOR_BTREE *)a; + CURSOR_API_CALL(a, session, equals, cbt->btree); + + /* + * Check both cursors are a "file:" type then call the underlying + * function, it can handle cursors pointing to different objects. + */ + if (!WT_PREFIX_MATCH(a->internal_uri, "file:") || + !WT_PREFIX_MATCH(b->internal_uri, "file:")) + WT_ERR_MSG(session, EINVAL, + "Cursors must reference the same object"); + + WT_CURSOR_CHECKKEY(a); + WT_CURSOR_CHECKKEY(b); + + ret = __wt_btcur_equals( + (WT_CURSOR_BTREE *)a, (WT_CURSOR_BTREE *)b, equalp); + +err: API_END_RET(session, ret); +} + +/* * __curfile_next -- * WT_CURSOR->next method for the btree cursor type. */ @@ -356,6 +389,7 @@ __wt_curfile_create(WT_SESSION_IMPL *session, __wt_cursor_set_key, /* set-key */ __wt_cursor_set_value, /* set-value */ __curfile_compare, /* compare */ + __curfile_equals, /* equals */ __curfile_next, /* next */ __curfile_prev, /* prev */ __curfile_reset, /* reset */ @@ -364,6 +398,7 @@ __wt_curfile_create(WT_SESSION_IMPL *session, __curfile_insert, /* insert */ __curfile_update, /* update */ __curfile_remove, /* remove */ + __wt_cursor_reconfigure, /* reconfigure */ __curfile_close); /* close */ WT_BTREE *btree; WT_CONFIG_ITEM cval; diff --git a/src/third_party/wiredtiger/src/cursor/cur_index.c b/src/third_party/wiredtiger/src/cursor/cur_index.c index 007f3a5ae5b..abc6a106cc9 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_index.c +++ b/src/third_party/wiredtiger/src/cursor/cur_index.c @@ -343,6 +343,7 @@ __wt_curindex_open(WT_SESSION_IMPL *session, __wt_cursor_set_key, /* set-key */ __curindex_set_value, /* set-value */ __wt_cursor_notsup, /* compare */ + __wt_cursor_notsup, /* equals */ __curindex_next, /* next */ __curindex_prev, /* prev */ __curindex_reset, /* reset */ @@ -351,6 +352,7 @@ __wt_curindex_open(WT_SESSION_IMPL *session, __wt_cursor_notsup, /* insert */ __wt_cursor_notsup, /* update */ __wt_cursor_notsup, /* remove */ + __wt_cursor_notsup, /* reconfigure */ __curindex_close); /* close */ WT_CURSOR_INDEX *cindex; WT_CURSOR *cursor; diff --git a/src/third_party/wiredtiger/src/cursor/cur_log.c b/src/third_party/wiredtiger/src/cursor/cur_log.c index e3089e9fb83..2e2a2530df6 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_log.c +++ b/src/third_party/wiredtiger/src/cursor/cur_log.c @@ -344,6 +344,7 @@ __wt_curlog_open(WT_SESSION_IMPL *session, __wt_cursor_set_key, /* set-key */ __wt_cursor_set_value, /* set-value */ __curlog_compare, /* compare */ + __wt_cursor_equal, /* equals */ __curlog_next, /* next */ __wt_cursor_notsup, /* prev */ __curlog_reset, /* reset */ @@ -352,6 +353,7 @@ __wt_curlog_open(WT_SESSION_IMPL *session, __wt_cursor_notsup, /* insert */ __wt_cursor_notsup, /* update */ __wt_cursor_notsup, /* remove */ + __wt_cursor_notsup, /* reconfigure */ __curlog_close); /* close */ WT_CURSOR *cursor; WT_CURSOR_LOG *cl; diff --git a/src/third_party/wiredtiger/src/cursor/cur_metadata.c b/src/third_party/wiredtiger/src/cursor/cur_metadata.c index 618c678558a..31c96e3087a 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_metadata.c +++ b/src/third_party/wiredtiger/src/cursor/cur_metadata.c @@ -409,6 +409,7 @@ __wt_curmetadata_open(WT_SESSION_IMPL *session, __wt_cursor_set_key, /* set-key */ __wt_cursor_set_value, /* set-value */ __curmetadata_compare, /* compare */ + __wt_cursor_equal, /* equals */ __curmetadata_next, /* next */ __curmetadata_prev, /* prev */ __curmetadata_reset, /* reset */ @@ -417,10 +418,12 @@ __wt_curmetadata_open(WT_SESSION_IMPL *session, __curmetadata_insert, /* insert */ __curmetadata_update, /* update */ __curmetadata_remove, /* remove */ + __wt_cursor_notsup, /* reconfigure */ __curmetadata_close); /* close */ WT_CURSOR *cursor; WT_CURSOR_METADATA *mdc; WT_DECL_RET; + WT_CONFIG_ITEM cval; WT_RET(__wt_calloc_one(session, &mdc)); @@ -435,8 +438,16 @@ __wt_curmetadata_open(WT_SESSION_IMPL *session, WT_ERR(__wt_cursor_init(cursor, uri, owner, cfg, cursorp)); - /* Metadata cursors default to read only. */ - WT_ERR(__wt_cursor_config_readonly(cursor, cfg, 1)); + /* + * Metadata cursors default to readonly; if not set to not-readonly, + * they are permanently readonly and cannot be reconfigured. + */ + WT_ERR(__wt_config_gets_def(session, cfg, "readonly", 1, &cval)); + if (cval.val != 0) { + cursor->insert = __wt_cursor_notsup; + cursor->update = __wt_cursor_notsup; + cursor->remove = __wt_cursor_notsup; + } if (0) { err: if (mdc->file_cursor != NULL) diff --git a/src/third_party/wiredtiger/src/cursor/cur_stat.c b/src/third_party/wiredtiger/src/cursor/cur_stat.c index f7effb87a4f..e3364ad8009 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_stat.c +++ b/src/third_party/wiredtiger/src/cursor/cur_stat.c @@ -408,11 +408,15 @@ __curstat_file_init(WT_SESSION_IMPL *session, * We're likely holding the handle lock inside the statistics * logging thread, not to mention calling __wt_conn_btree_apply * from there as well. Save/restore the handle. + * Take the schema lock now, in case btree apply needs to + * get it later - that would violate lock ordering + * conventions and can lead to deadlocks. */ saved_dhandle = dhandle; - WT_WITH_DHANDLE_LOCK(session, - ret = __wt_conn_btree_apply( - session, 1, dhandle->name, __curstat_checkpoint, cfg_arg)); + WT_WITH_SCHEMA_LOCK(session, + WT_WITH_DHANDLE_LOCK(session, + ret = __wt_conn_btree_apply(session, + 1, dhandle->name, __curstat_checkpoint, cfg_arg))); session->dhandle = saved_dhandle; } @@ -485,6 +489,7 @@ __wt_curstat_open(WT_SESSION_IMPL *session, __curstat_set_key, /* set-key */ __curstat_set_value, /* set-value */ __wt_cursor_notsup, /* compare */ + __wt_cursor_notsup, /* equals */ __curstat_next, /* next */ __curstat_prev, /* prev */ __curstat_reset, /* reset */ @@ -493,6 +498,7 @@ __wt_curstat_open(WT_SESSION_IMPL *session, __wt_cursor_notsup, /* insert */ __wt_cursor_notsup, /* update */ __wt_cursor_notsup, /* remove */ + __wt_cursor_notsup, /* reconfigure */ __curstat_close); /* close */ WT_CONFIG_ITEM cval, sval; WT_CURSOR *cursor; diff --git a/src/third_party/wiredtiger/src/cursor/cur_std.c b/src/third_party/wiredtiger/src/cursor/cur_std.c index 1470cb37100..df38eb9e57d 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_std.c +++ b/src/third_party/wiredtiger/src/cursor/cur_std.c @@ -59,28 +59,6 @@ __wt_cursor_set_notsup(WT_CURSOR *cursor) } /* - * __wt_cursor_config_readonly -- - * Parse read only configuration and setup cursor appropriately. - */ -int -__wt_cursor_config_readonly(WT_CURSOR *cursor, const char *cfg[], int def) -{ - WT_CONFIG_ITEM cval; - WT_SESSION_IMPL *session; - - session = (WT_SESSION_IMPL *)cursor->session; - - WT_RET(__wt_config_gets_def(session, cfg, "readonly", def, &cval)); - if (cval.val != 0) { - /* Reset all cursor methods that could modify data. */ - cursor->insert = __wt_cursor_notsup; - cursor->update = __wt_cursor_notsup; - cursor->remove = __wt_cursor_notsup; - } - return (0); -} - -/* * __wt_cursor_kv_not_set -- * Standard error message for key/values not set. */ @@ -502,29 +480,68 @@ __wt_cursor_close(WT_CURSOR *cursor) } /* - * __cursor_runtime_config -- + * __wt_cursor_equal -- + * WT_CURSOR->equals default implementation. + */ +int +__wt_cursor_equal(WT_CURSOR *cursor, WT_CURSOR *other, int *equalp) +{ + WT_DECL_RET; + WT_SESSION_IMPL *session; + int cmp; + + session = (WT_SESSION_IMPL *)cursor->session; + CURSOR_API_CALL(cursor, session, equals, NULL); + + WT_ERR(cursor->compare(cursor, other, &cmp)); + *equalp = (cmp == 0) ? 1 : 0; + +err: API_END(session, ret); + return (ret); +} + +/* + * __wt_cursor_reconfigure -- * Set runtime-configurable settings. */ -static int -__cursor_runtime_config(WT_CURSOR *cursor, const char *cfg[]) +int +__wt_cursor_reconfigure(WT_CURSOR *cursor, const char *config) { WT_CONFIG_ITEM cval; + WT_DECL_RET; WT_SESSION_IMPL *session; session = (WT_SESSION_IMPL *)cursor->session; - /* - * !!! - * There's no way yet to reconfigure cursor flags at runtime; if, in - * the future there is a way to do that, similar support needs to be - * added for data-source cursors, or, this call needs to return an - * error in the case of a data-source cursor. + /* Reconfiguration resets the cursor. */ + WT_RET(cursor->reset(cursor)); + + /* + * append + * Only relevant to column stores. */ - WT_RET(__wt_config_gets_def(session, cfg, "overwrite", 1, &cval)); - if (cval.val) - F_SET(cursor, WT_CURSTD_OVERWRITE); - else - F_CLR(cursor, WT_CURSTD_OVERWRITE); + if (WT_CURSOR_RECNO(cursor)) { + if ((ret = __wt_config_getones( + session, config, "append", &cval)) == 0) { + if (cval.val) + F_SET(cursor, WT_CURSTD_APPEND); + else + F_CLR(cursor, WT_CURSTD_APPEND); + } else + WT_RET_NOTFOUND_OK(ret); + } + + /* + * overwrite + */ + if ((ret = __wt_config_getones( + session, config, "overwrite", &cval)) == 0) { + if (cval.val) + F_SET(cursor, WT_CURSTD_OVERWRITE); + else + F_CLR(cursor, WT_CURSTD_OVERWRITE); + } else + WT_RET_NOTFOUND_OK(ret); return (0); } @@ -587,9 +604,6 @@ __wt_cursor_init(WT_CURSOR *cursor, if (cursor->internal_uri == NULL) WT_RET(__wt_strdup(session, uri, &cursor->internal_uri)); - /* Set runtime-configurable settings. */ - WT_RET(__cursor_runtime_config(cursor, cfg)); - /* * append * The append flag is only relevant to column stores. @@ -601,14 +615,23 @@ __wt_cursor_init(WT_CURSOR *cursor, } /* - * checkpoint - * Checkpoint cursors are read-only. + * checkpoint, readonly + * Checkpoint cursors are permanently read-only, avoid the extra work + * of two configuration string checks. */ WT_RET(__wt_config_gets_def(session, cfg, "checkpoint", 0, &cval)); if (cval.len != 0) { cursor->insert = __wt_cursor_notsup; cursor->update = __wt_cursor_notsup; cursor->remove = __wt_cursor_notsup; + } else { + WT_RET( + __wt_config_gets_def(session, cfg, "readonly", 0, &cval)); + if (cval.val != 0) { + cursor->insert = __wt_cursor_notsup; + cursor->update = __wt_cursor_notsup; + cursor->remove = __wt_cursor_notsup; + } } /* @@ -635,14 +658,18 @@ __wt_cursor_init(WT_CURSOR *cursor, } else cdump = NULL; + /* overwrite */ + WT_RET(__wt_config_gets_def(session, cfg, "overwrite", 1, &cval)); + if (cval.val) + F_SET(cursor, WT_CURSTD_OVERWRITE); + else + F_CLR(cursor, WT_CURSTD_OVERWRITE); + /* raw */ WT_RET(__wt_config_gets_def(session, cfg, "raw", 0, &cval)); if (cval.val != 0) F_SET(cursor, WT_CURSTD_RAW); - /* readonly */ - WT_RET(__wt_cursor_config_readonly(cursor, cfg, 0)); - /* * Cursors that are internal to some other cursor (such as file cursors * inside a table cursor) should be closed after the containing cursor. diff --git a/src/third_party/wiredtiger/src/cursor/cur_table.c b/src/third_party/wiredtiger/src/cursor/cur_table.c index 4eb362cfcd4..f8e8625b0bd 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_table.c +++ b/src/third_party/wiredtiger/src/cursor/cur_table.c @@ -77,20 +77,22 @@ __curextract_insert(WT_CURSOR *cursor) { static int __apply_idx(WT_CURSOR_TABLE *ctable, size_t func_off, int skip_immutable) { WT_CURSOR_STATIC_INIT(iface, - __wt_cursor_get_key, /* get-key */ - __wt_cursor_get_value, /* get-value */ - __wt_cursor_set_key, /* set-key */ - __wt_cursor_set_value, /* set-value */ - __wt_cursor_notsup, /* compare */ - __wt_cursor_notsup, /* next */ - __wt_cursor_notsup, /* prev */ - __wt_cursor_notsup, /* reset */ - __wt_cursor_notsup, /* search */ - __wt_cursor_notsup, /* search-near */ - __curextract_insert, /* insert */ - __wt_cursor_notsup, /* update */ - __wt_cursor_notsup, /* remove */ - __wt_cursor_notsup); /* close */ + __wt_cursor_get_key, /* get-key */ + __wt_cursor_get_value, /* get-value */ + __wt_cursor_set_key, /* set-key */ + __wt_cursor_set_value, /* set-value */ + __wt_cursor_notsup, /* compare */ + __wt_cursor_notsup, /* equals */ + __wt_cursor_notsup, /* next */ + __wt_cursor_notsup, /* prev */ + __wt_cursor_notsup, /* reset */ + __wt_cursor_notsup, /* search */ + __wt_cursor_notsup, /* search-near */ + __curextract_insert, /* insert */ + __wt_cursor_notsup, /* update */ + __wt_cursor_notsup, /* reconfigure */ + __wt_cursor_notsup, /* remove */ + __wt_cursor_notsup); /* close */ WT_CURSOR **cp; WT_CURSOR_EXTRACTOR extract_cursor; WT_DECL_RET; @@ -755,11 +757,11 @@ __curtable_open_colgroups(WT_CURSOR_TABLE *ctable, const char *cfg_arg[]) WT_TABLE *table; WT_CURSOR **cp; /* - * Underlying column groups are always opened without dump, and only - * the primary is opened with next_random. + * Underlying column groups are always opened without dump or readonly, + * and only the primary is opened with next_random. */ const char *cfg[] = { - cfg_arg[0], cfg_arg[1], "dump=\"\"", NULL, NULL + cfg_arg[0], cfg_arg[1], "dump=\"\",readonly=0", NULL, NULL }; u_int i; int complete; @@ -832,20 +834,22 @@ __wt_curtable_open(WT_SESSION_IMPL *session, const char *uri, const char *cfg[], WT_CURSOR **cursorp) { WT_CURSOR_STATIC_INIT(iface, - __wt_curtable_get_key, /* get-key */ - __wt_curtable_get_value, /* get-value */ - __wt_curtable_set_key, /* set-key */ - __wt_curtable_set_value, /* set-value */ - __curtable_compare, /* compare */ - __curtable_next, /* next */ - __curtable_prev, /* prev */ - __curtable_reset, /* reset */ - __curtable_search, /* search */ - __curtable_search_near, /* search-near */ - __curtable_insert, /* insert */ - __curtable_update, /* update */ - __curtable_remove, /* remove */ - __curtable_close); /* close */ + __wt_curtable_get_key, /* get-key */ + __wt_curtable_get_value, /* get-value */ + __wt_curtable_set_key, /* set-key */ + __wt_curtable_set_value, /* set-value */ + __curtable_compare, /* compare */ + __wt_cursor_equal, /* equals */ + __curtable_next, /* next */ + __curtable_prev, /* prev */ + __curtable_reset, /* reset */ + __curtable_search, /* search */ + __curtable_search_near, /* search-near */ + __curtable_insert, /* insert */ + __curtable_update, /* update */ + __curtable_remove, /* remove */ + __wt_cursor_reconfigure, /* reconfigure */ + __curtable_close); /* close */ WT_CONFIG_ITEM cval; WT_CURSOR *cursor; WT_CURSOR_TABLE *ctable; @@ -892,8 +896,8 @@ __wt_curtable_open(WT_SESSION_IMPL *session, ctable->plan = table->plan; /* Handle projections. */ + WT_ERR(__wt_scr_alloc(session, 0, &tmp)); if (columns != NULL) { - WT_ERR(__wt_scr_alloc(session, 0, &tmp)); WT_ERR(__wt_struct_reformat(session, table, columns, strlen(columns), NULL, 1, tmp)); WT_ERR(__wt_strndup( @@ -935,22 +939,28 @@ __wt_curtable_open(WT_SESSION_IMPL *session, WT_ERR(__curtable_open_colgroups(ctable, cfg)); /* - * We'll need to squirrel away a copy of the cursor configuration - * for if/when we open indices. + * We'll need to squirrel away a copy of the cursor configuration for + * if/when we open indices. * * cfg[0] is the baseline configuration for the cursor open and we can * acquire another copy from the configuration structures, so it would * be reasonable not to copy it here: but I'd rather be safe than sorry. * - * Underlying indices are always opened without dump. + * cfg[1] is the application configuration. + * + * Underlying indices are always opened without dump or readonly; that + * information is appended to cfg[1] so later "fast" configuration calls + * (checking only cfg[0] and cfg[1]) work. I don't expect to see more + * than two configuration strings here, but it's written to compact into + * two configuration strings, a copy of cfg[0] and the rest in cfg[1]. */ - for (cfg_cnt = 0; cfg[cfg_cnt] != NULL; ++cfg_cnt) - ; - WT_ERR(__wt_calloc_def(session, cfg_cnt + 2, &ctable->cfg)); - for (cfg_cnt = 0; cfg[cfg_cnt] != NULL; ++cfg_cnt) - WT_ERR( - __wt_strdup(session, cfg[cfg_cnt], &ctable->cfg[cfg_cnt])); - WT_ERR(__wt_strdup(session, "dump=\"\"", &ctable->cfg[cfg_cnt])); + WT_ERR(__wt_calloc_def(session, 3, &ctable->cfg)); + WT_ERR(__wt_strdup(session, cfg[0], &ctable->cfg[0])); + WT_ERR(__wt_buf_set(session, tmp, "", 0)); + for (cfg_cnt = 1; cfg[cfg_cnt] != NULL; ++cfg_cnt) + WT_ERR(__wt_buf_catfmt(session, tmp, "%s,", cfg[cfg_cnt])); + WT_ERR(__wt_buf_catfmt(session, tmp, "dump=\"\",readonly=0")); + WT_ERR(__wt_strdup(session, tmp->data, &ctable->cfg[1])); if (0) { err: WT_TRET(__curtable_close(cursor)); diff --git a/src/third_party/wiredtiger/src/evict/evict_page.c b/src/third_party/wiredtiger/src/evict/evict_page.c index b3a6f718ca2..99e2a6751be 100644 --- a/src/third_party/wiredtiger/src/evict/evict_page.c +++ b/src/third_party/wiredtiger/src/evict/evict_page.c @@ -355,6 +355,14 @@ __evict_review(WT_SESSION_IMPL *session, WT_REF *ref, * pages after we've written them. */ if (WT_PAGE_IS_INTERNAL(page)) { + /* + * Quit if we're trying to push out a "tree", an internal page + * with live internal pages as children, it's not likely to + * succeed. + */ + if (!top && !exclusive) + return (EBUSY); + WT_WITH_PAGE_INDEX(session, ret = __evict_review_subtree( session, ref, exclusive, inmem_splitp, istreep)); WT_RET(ret); diff --git a/src/third_party/wiredtiger/src/include/config.h b/src/third_party/wiredtiger/src/include/config.h index 277a22063d1..65757c2ef6d 100644 --- a/src/third_party/wiredtiger/src/include/config.h +++ b/src/third_party/wiredtiger/src/include/config.h @@ -57,29 +57,30 @@ struct __wt_config_parser_impl { #define WT_CONFIG_ENTRY_connection_open_session 8 #define WT_CONFIG_ENTRY_connection_reconfigure 9 #define WT_CONFIG_ENTRY_cursor_close 10 -#define WT_CONFIG_ENTRY_file_meta 11 -#define WT_CONFIG_ENTRY_index_meta 12 -#define WT_CONFIG_ENTRY_session_begin_transaction 13 -#define WT_CONFIG_ENTRY_session_checkpoint 14 -#define WT_CONFIG_ENTRY_session_close 15 -#define WT_CONFIG_ENTRY_session_commit_transaction 16 -#define WT_CONFIG_ENTRY_session_compact 17 -#define WT_CONFIG_ENTRY_session_create 18 -#define WT_CONFIG_ENTRY_session_drop 19 -#define WT_CONFIG_ENTRY_session_log_printf 20 -#define WT_CONFIG_ENTRY_session_open_cursor 21 -#define WT_CONFIG_ENTRY_session_reconfigure 22 -#define WT_CONFIG_ENTRY_session_rename 23 -#define WT_CONFIG_ENTRY_session_rollback_transaction 24 -#define WT_CONFIG_ENTRY_session_salvage 25 -#define WT_CONFIG_ENTRY_session_truncate 26 -#define WT_CONFIG_ENTRY_session_upgrade 27 -#define WT_CONFIG_ENTRY_session_verify 28 -#define WT_CONFIG_ENTRY_table_meta 29 -#define WT_CONFIG_ENTRY_wiredtiger_open 30 -#define WT_CONFIG_ENTRY_wiredtiger_open_all 31 -#define WT_CONFIG_ENTRY_wiredtiger_open_basecfg 32 -#define WT_CONFIG_ENTRY_wiredtiger_open_usercfg 33 +#define WT_CONFIG_ENTRY_cursor_reconfigure 11 +#define WT_CONFIG_ENTRY_file_meta 12 +#define WT_CONFIG_ENTRY_index_meta 13 +#define WT_CONFIG_ENTRY_session_begin_transaction 14 +#define WT_CONFIG_ENTRY_session_checkpoint 15 +#define WT_CONFIG_ENTRY_session_close 16 +#define WT_CONFIG_ENTRY_session_commit_transaction 17 +#define WT_CONFIG_ENTRY_session_compact 18 +#define WT_CONFIG_ENTRY_session_create 19 +#define WT_CONFIG_ENTRY_session_drop 20 +#define WT_CONFIG_ENTRY_session_log_printf 21 +#define WT_CONFIG_ENTRY_session_open_cursor 22 +#define WT_CONFIG_ENTRY_session_reconfigure 23 +#define WT_CONFIG_ENTRY_session_rename 24 +#define WT_CONFIG_ENTRY_session_rollback_transaction 25 +#define WT_CONFIG_ENTRY_session_salvage 26 +#define WT_CONFIG_ENTRY_session_truncate 27 +#define WT_CONFIG_ENTRY_session_upgrade 28 +#define WT_CONFIG_ENTRY_session_verify 29 +#define WT_CONFIG_ENTRY_table_meta 30 +#define WT_CONFIG_ENTRY_wiredtiger_open 31 +#define WT_CONFIG_ENTRY_wiredtiger_open_all 32 +#define WT_CONFIG_ENTRY_wiredtiger_open_basecfg 33 +#define WT_CONFIG_ENTRY_wiredtiger_open_usercfg 34 /* * configuration section: END * DO NOT EDIT: automatically built by dist/flags.py. diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h index c8a3ae6e291..c5723882489 100644 --- a/src/third_party/wiredtiger/src/include/connection.h +++ b/src/third_party/wiredtiger/src/include/connection.h @@ -279,10 +279,14 @@ struct __wt_connection_impl { #define WT_CONN_LOG_EXISTED 0x04 /* Log files found */ #define WT_CONN_LOG_PREALLOC 0x08 /* Pre-allocation is enabled */ uint32_t log_flags; /* Global logging configuration */ - WT_CONDVAR *log_cond; /* Log archive wait mutex */ - WT_SESSION_IMPL *log_session; /* Log archive session */ - wt_thread_t log_tid; /* Log archive thread */ - int log_tid_set; /* Log archive thread set */ + WT_CONDVAR *log_cond; /* Log server wait mutex */ + WT_SESSION_IMPL *log_session; /* Log server session */ + wt_thread_t log_tid; /* Log server thread */ + int log_tid_set; /* Log server thread set */ + WT_CONDVAR *log_close_cond;/* Log close thread wait mutex */ + WT_SESSION_IMPL *log_close_session;/* Log close thread session */ + wt_thread_t log_close_tid; /* Log close thread thread */ + int log_close_tid_set;/* Log close thread set */ WT_LOG *log; /* Logging structure */ WT_COMPRESSOR *log_compressor;/* Logging compressor */ wt_off_t log_file_max; /* Log file max size */ diff --git a/src/third_party/wiredtiger/src/include/cursor.h b/src/third_party/wiredtiger/src/include/cursor.h index e46c1f7de1b..ba53bd23c78 100644 --- a/src/third_party/wiredtiger/src/include/cursor.h +++ b/src/third_party/wiredtiger/src/include/cursor.h @@ -15,6 +15,7 @@ set_key, \ set_value, \ compare, \ + equals, \ next, \ prev, \ reset, \ @@ -23,6 +24,7 @@ insert, \ update, \ remove, \ + reconfigure, \ close) \ static const WT_CURSOR n = { \ NULL, /* session */ \ @@ -34,6 +36,7 @@ (void (*)(WT_CURSOR *, ...))(set_key), \ (void (*)(WT_CURSOR *, ...))(set_value), \ (int (*)(WT_CURSOR *, WT_CURSOR *, int *))(compare), \ + (int (*)(WT_CURSOR *, WT_CURSOR *, int *))(equals), \ next, \ prev, \ reset, \ @@ -43,6 +46,7 @@ update, \ remove, \ close, \ + (int (*)(WT_CURSOR *, const char *))(reconfigure), \ { NULL, NULL }, /* TAILQ_ENTRY q */ \ 0, /* recno key */ \ { 0 }, /* recno raw buffer */ \ diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index d8ed3f5cef1..35b8dfc113e 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -96,6 +96,7 @@ extern int __wt_btcur_remove(WT_CURSOR_BTREE *cbt); extern int __wt_btcur_update(WT_CURSOR_BTREE *cbt); extern int __wt_btcur_next_random(WT_CURSOR_BTREE *cbt); extern int __wt_btcur_compare(WT_CURSOR_BTREE *a_arg, WT_CURSOR_BTREE *b_arg, int *cmpp); +extern int __wt_btcur_equals( WT_CURSOR_BTREE *a_arg, WT_CURSOR_BTREE *b_arg, int *equalp); extern int __wt_btcur_range_truncate(WT_CURSOR_BTREE *start, WT_CURSOR_BTREE *stop); extern int __wt_btcur_close(WT_CURSOR_BTREE *cbt); extern int __wt_debug_set_verbose(WT_SESSION_IMPL *session, const char *v); @@ -265,7 +266,6 @@ extern int __wt_curstat_open(WT_SESSION_IMPL *session, const char *uri, const ch extern int __wt_cursor_notsup(WT_CURSOR *cursor); extern int __wt_cursor_noop(WT_CURSOR *cursor); extern void __wt_cursor_set_notsup(WT_CURSOR *cursor); -extern int __wt_cursor_config_readonly(WT_CURSOR *cursor, const char *cfg[], int def); extern int __wt_cursor_kv_not_set(WT_CURSOR *cursor, int key); extern int __wt_cursor_get_key(WT_CURSOR *cursor, ...); extern void __wt_cursor_set_key(WT_CURSOR *cursor, ...); @@ -280,6 +280,8 @@ extern int __wt_cursor_get_valuev(WT_CURSOR *cursor, va_list ap); extern void __wt_cursor_set_value(WT_CURSOR *cursor, ...); extern void __wt_cursor_set_valuev(WT_CURSOR *cursor, va_list ap); extern int __wt_cursor_close(WT_CURSOR *cursor); +extern int __wt_cursor_equal(WT_CURSOR *cursor, WT_CURSOR *other, int *equalp); +extern int __wt_cursor_reconfigure(WT_CURSOR *cursor, const char *config); extern int __wt_cursor_dup_position(WT_CURSOR *to_dup, WT_CURSOR *cursor); extern int __wt_cursor_init(WT_CURSOR *cursor, const char *uri, WT_CURSOR *owner, const char *cfg[], WT_CURSOR **cursorp); extern int __wt_curtable_get_key(WT_CURSOR *cursor, ...); diff --git a/src/third_party/wiredtiger/src/include/flags.h b/src/third_party/wiredtiger/src/include/flags.h index c7e74885a35..9664fce3f9f 100644 --- a/src/third_party/wiredtiger/src/include/flags.h +++ b/src/third_party/wiredtiger/src/include/flags.h @@ -6,15 +6,16 @@ #define WT_CONN_CKPT_SYNC 0x00000002 #define WT_CONN_EVICTION_RUN 0x00000004 #define WT_CONN_LEAK_MEMORY 0x00000008 -#define WT_CONN_LSM_MERGE 0x00000010 -#define WT_CONN_PANIC 0x00000020 -#define WT_CONN_SERVER_ASYNC 0x00000040 -#define WT_CONN_SERVER_CHECKPOINT 0x00000080 -#define WT_CONN_SERVER_LSM 0x00000100 -#define WT_CONN_SERVER_RUN 0x00000200 -#define WT_CONN_SERVER_STATISTICS 0x00000400 -#define WT_CONN_SERVER_SWEEP 0x00000800 -#define WT_CONN_WAS_BACKUP 0x00001000 +#define WT_CONN_LOG_SERVER_RUN 0x00000010 +#define WT_CONN_LSM_MERGE 0x00000020 +#define WT_CONN_PANIC 0x00000040 +#define WT_CONN_SERVER_ASYNC 0x00000080 +#define WT_CONN_SERVER_CHECKPOINT 0x00000100 +#define WT_CONN_SERVER_LSM 0x00000200 +#define WT_CONN_SERVER_RUN 0x00000400 +#define WT_CONN_SERVER_STATISTICS 0x00000800 +#define WT_CONN_SERVER_SWEEP 0x00001000 +#define WT_CONN_WAS_BACKUP 0x00002000 #define WT_EVICTING 0x00000001 #define WT_FILE_TYPE_CHECKPOINT 0x00000001 #define WT_FILE_TYPE_DATA 0x00000002 diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index 91eb41af4f3..982e850241b 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -270,7 +270,8 @@ struct __wt_cursor { */ /*! * Return the ordering relationship between two cursors: both cursors - * must have the same data source and have valid keys. + * must have the same data source and have valid keys. (When testing + * only for equality, WT_CURSOR::equals may be faster.) * * @snippet ex_all.c Cursor comparison * @@ -286,6 +287,21 @@ struct __wt_cursor { int __F(compare)(WT_CURSOR *cursor, WT_CURSOR *other, int *comparep); /*! + * Return the ordering relationship between two cursors, testing only + * for equality: both cursors must have the same data source and have + * valid keys. + * + * @snippet ex_all.c Cursor equality + * + * @param cursor the cursor handle + * @param other another cursor handle + * @param[out] equalp the status of the comparison: 1 if the cursors + * refer to the same key, otherwise 0. + * @errors + */ + int __F(equals)(WT_CURSOR *cursor, WT_CURSOR *other, int *equalp); + + /*! * Return the next record. * * @snippet ex_all.c Return the next record @@ -495,6 +511,29 @@ struct __wt_cursor { */ int __F(close)(WT_HANDLE_CLOSED(WT_CURSOR) *cursor); + /*! + * Reconfigure the cursor. + * + * The cursor is reset. + * + * @snippet ex_all.c Reconfigure a cursor + * + * @param cursor the cursor handle + * @configstart{cursor.reconfigure, see dist/api_data.py} + * @config{append, append the value as a new record\, creating a new + * record number key; valid only for cursors with record number keys., a + * boolean flag; default \c false.} + * @config{overwrite, configures whether the cursor's insert\, update + * and remove methods check the existing state of the record. If \c + * overwrite is \c false\, WT_CURSOR::insert fails with + * ::WT_DUPLICATE_KEY if the record exists\, WT_CURSOR::update and + * WT_CURSOR::remove fail with ::WT_NOTFOUND if the record does not + * exist., a boolean flag; default \c true.} + * @configend + * @errors + */ + int __F(reconfigure)(WT_CURSOR *cursor, const char *config); + /* * Protected fields, only to be used by cursor implementations. */ @@ -522,6 +561,7 @@ struct __wt_cursor { * user on open. */ const char *internal_uri; + /* Saved modification methods. */ #define WT_CURSTD_APPEND 0x0001 #define WT_CURSTD_BULK 0x0002 @@ -813,10 +853,6 @@ struct __wt_session { * duplicated cursors are positioned at the same place in the data * source as the original. * - * To reconfigure a cursor, duplicate it with a new configuration value: - * - * @snippet ex_all.c Reconfigure a cursor - * * Cursor handles should be discarded by calling WT_CURSOR::close. * * Cursors capable of supporting transactional operations operate in the diff --git a/src/third_party/wiredtiger/src/log/log.c b/src/third_party/wiredtiger/src/log/log.c index 181e77128a9..e75946e9885 100644 --- a/src/third_party/wiredtiger/src/log/log.c +++ b/src/third_party/wiredtiger/src/log/log.c @@ -240,6 +240,7 @@ __log_acquire(WT_SESSION_IMPL *session, uint64_t recsize, WT_LOGSLOT *slot) if (log->log_close_fh != NULL) F_SET(slot, SLOT_CLOSEFH); } + /* * Checkpoints can be configured based on amount of log written. * Add in this log record to the sum and if needed, signal the @@ -857,9 +858,8 @@ __log_release(WT_SESSION_IMPL *session, WT_LOGSLOT *slot) { WT_CONNECTION_IMPL *conn; WT_DECL_RET; - WT_FH *close_fh; WT_LOG *log; - WT_LSN close_end_lsn, close_lsn, sync_lsn; + WT_LSN sync_lsn; size_t write_size; int locked; WT_DECL_SPINLOCK_ID(id); /* Must appear last */ @@ -868,31 +868,6 @@ __log_release(WT_SESSION_IMPL *session, WT_LOGSLOT *slot) log = conn->log; locked = 0; - /* - * If we're going to have to close our log file, make a local copy - * of the file handle structure. - */ - close_fh = NULL; - WT_INIT_LSN(&close_lsn); - WT_INIT_LSN(&close_end_lsn); - if (F_ISSET(slot, SLOT_CLOSEFH)) { - close_fh = log->log_close_fh; - /* - * Set the close_end_lsn to the LSN immediately after ours. - * That is, the beginning of the next log file. We need to - * know the LSN file number of our own close in case earlier - * calls are still in progress and the next one to move the - * sync_lsn into the next file for later syncs. - */ - WT_ERR(__wt_log_extract_lognum(session, close_fh->name, - &close_lsn.file)); - close_lsn.offset = 0; - close_end_lsn = close_lsn; - close_end_lsn.file++; - log->log_close_fh = NULL; - F_CLR(slot, SLOT_CLOSEFH); - } - /* Write the buffered records */ if (F_ISSET(slot, SLOT_BUFFERED)) { write_size = (size_t) @@ -909,32 +884,8 @@ __log_release(WT_SESSION_IMPL *session, WT_LOGSLOT *slot) __wt_yield(); log->write_lsn = slot->slot_end_lsn; - /* - * If we have a file to close, close it now. First fsync so - * that a later sync will be assured all earlier transactions - * in earlier log files are also on disk. We have to do this - * before potentially syncing our own operation. - */ - if (close_fh) { - WT_ERR(__wt_fsync(session, close_fh)); - /* - * This loop guarantees sync_lsn is updated in file order - * so that when sync_lsn is updated, we know all earlier files - * have already been fully processed. - */ - while (log->sync_lsn.file < close_lsn.file || - __wt_spin_trylock(session, &log->log_sync_lock, &id) != 0) { - WT_ERR(__wt_cond_wait( - session, log->log_sync_cond, 10000)); - continue; - } - locked = 1; - WT_ERR(__wt_close(session, close_fh)); - log->sync_lsn = close_end_lsn; - WT_ERR(__wt_cond_signal(session, log->log_sync_cond)); - locked = 0; - __wt_spin_unlock(session, &log->log_sync_lock); - } + if (F_ISSET(slot, SLOT_CLOSEFH)) + WT_ERR(__wt_cond_signal(session, conn->log_close_cond)); /* * Try to consolidate calls to fsync to wait less. Acquire a spin lock @@ -956,10 +907,10 @@ __log_release(WT_SESSION_IMPL *session, WT_LOGSLOT *slot) locked = 1; /* - * Record the current end of log after we grabbed the lock. + * Record the current end of our update after the lock. * That is how far our calls can guarantee. */ - sync_lsn = log->write_lsn; + sync_lsn = slot->slot_end_lsn; /* * Check if we have to sync the parent directory. Some * combinations of sync flags may result in the log file @@ -1506,6 +1457,13 @@ __wt_log_write(WT_SESSION_IMPL *session, WT_ITEM *record, WT_LSN *lsnp, conn = S2C(session); log = conn->log; + /* + * An error during opening the logging subsystem can result in it + * being enabled, but without an open log file. In that case, + * just return. + */ + if (log->log_fh == NULL) + return (0); ip = record; if ((compressor = conn->log_compressor) != NULL && record->size < log->allocsize) diff --git a/src/third_party/wiredtiger/src/lsm/lsm_cursor.c b/src/third_party/wiredtiger/src/lsm/lsm_cursor.c index 0d44b16d85c..39b4b4508b7 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_cursor.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_cursor.c @@ -1429,6 +1429,7 @@ __wt_clsm_open(WT_SESSION_IMPL *session, __wt_cursor_set_key, /* set-key */ __wt_cursor_set_value, /* set-value */ __clsm_compare, /* compare */ + __wt_cursor_equal, /* equals */ __clsm_next, /* next */ __clsm_prev, /* prev */ __clsm_reset, /* reset */ @@ -1437,6 +1438,7 @@ __wt_clsm_open(WT_SESSION_IMPL *session, __clsm_insert, /* insert */ __clsm_update, /* update */ __clsm_remove, /* remove */ + __wt_cursor_reconfigure, /* reconfigure */ __clsm_close); /* close */ WT_CURSOR *cursor; WT_CURSOR_LSM *clsm; @@ -1468,8 +1470,6 @@ __wt_clsm_open(WT_SESSION_IMPL *session, cursor->key_format = lsm_tree->key_format; cursor->value_format = lsm_tree->value_format; - WT_ERR(__wt_cursor_config_readonly(cursor, cfg, 0)); - clsm->lsm_tree = lsm_tree; /* diff --git a/src/third_party/wiredtiger/src/reconcile/rec_write.c b/src/third_party/wiredtiger/src/reconcile/rec_write.c index 5c3ed614850..0300596f90b 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_write.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_write.c @@ -2432,15 +2432,7 @@ no_slots: * the "page" and try again after we accumulate some more rows. */ WT_STAT_FAST_DATA_INCR(session, compress_raw_fail_temporary); - -split_grow: /* - * Double the page size and make sure we accommodate at least - * one more record. The reason for the latter is that we may - * be here because there's a large key/value pair that won't - * fit in our initial page buffer, even at its expanded size. - */ - r->page_size *= 2; - return (__rec_split_grow(session, r, r->page_size + next_len)); + goto split_grow; } /* We have a block, update the boundary counter. */ @@ -2462,6 +2454,22 @@ split_grow: /* } else WT_RET( __rec_split_write(session, r, last, write_ref, last_block)); + + /* + * We got called because there wasn't enough room in the buffer for the + * next key and we might or might not have written a block. In any case, + * make sure the next key fits into the buffer. + */ + if (r->space_avail < next_len) { +split_grow: /* + * Double the page size and make sure we accommodate at least + * one more record. The reason for the latter is that we may + * be here because there's a large key/value pair that won't + * fit in our initial page buffer, even at its expanded size. + */ + r->page_size *= 2; + return (__rec_split_grow(session, r, r->page_size + next_len)); + } return (0); } diff --git a/src/third_party/wiredtiger/src/session/session_compact.c b/src/third_party/wiredtiger/src/session/session_compact.c index acb8b730a7d..2a18eb3be93 100644 --- a/src/third_party/wiredtiger/src/session/session_compact.c +++ b/src/third_party/wiredtiger/src/session/session_compact.c @@ -175,11 +175,13 @@ __compact_file(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) WT_ERR(__wt_epoch(session, &start_time)); /* - * We compact 10% of the file on each pass, try 10 times (which is - * probably overkill), and quit if we make no progress. Check for a - * timeout each time through the loop. + * We compact 10% of the file on each pass (but the overall size of the + * file is decreasing each time, so we're not compacting 10% of the + * original file each time). Try 100 times (which is clearly more than + * we need); quit if we make no progress and check for a timeout each + * time through the loop. */ - for (i = 0; i < 10; ++i) { + for (i = 0; i < 100; ++i) { WT_ERR(wt_session->checkpoint(wt_session, t->data)); session->compaction = 0; diff --git a/src/third_party/wiredtiger/tools/statlog.py b/src/third_party/wiredtiger/tools/statlog.py deleted file mode 100644 index a597eb060ae..00000000000 --- a/src/third_party/wiredtiger/tools/statlog.py +++ /dev/null @@ -1,125 +0,0 @@ -#!/usr/bin/env python -# -# Public Domain 2014-2015 MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# -# This is free and unencumbered software released into the public domain. -# -# Anyone is free to copy, modify, publish, use, compile, sell, or -# distribute this software, either in source code form or as a compiled -# binary, for any purpose, commercial or non-commercial, and by any -# means. -# -# In jurisdictions that recognize copyright laws, the author or authors -# of this software dedicate any and all copyright interest in the -# software to the public domain. We make this dedication for the benefit -# of the public at large and to the detriment of our heirs and -# successors. We intend this dedication to be an overt act of -# relinquishment in perpetuity of all present and future rights to this -# software under copyright law. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -# - -import fileinput, os, shutil, sys, textwrap -from collections import defaultdict -from datetime import datetime -from subprocess import call - -# Import the data describing which statistics should not be scaled -from stat_data import no_scale_per_second_list - -TIMEFMT = "%b %d %H:%M:%S" -reportno = 0 - -# Plot a set of entries for a title. -def plot(title, values): - global reportno - - # Ignore entries where the value never changes. - skip = True - t0, v0 = values[0] - for t, v in values: - if v != v0: - skip = False - break - if skip: - print 'skipping: ' + title - return - - print 'building: ' + title - reportno = reportno + 1 - num = "%03d" % reportno - - ylabel = 'Value' - if title.split(' ')[1] != 'spinlock' and \ - title.split(' ', 1)[1] in no_scale_per_second_list: - seconds = 1 - else: - t1, v1 = values[1] - seconds = (datetime.strptime(t1, TIMEFMT) - - datetime.strptime(t0, TIMEFMT)).seconds - if seconds == 0: - seconds = 1 - ylabel += ' per second' - - # Write the raw data into a file for processing. - of = open("reports/raw/report.%s.raw" % num, "w") - for t, v in sorted(values): - print >>of, "%s %g" % (t, float(v) / seconds) - of.close() - - # Write a command file for gnuplot. - of = open("gnuplot.cmd", "w") - of.write(''' -set terminal png nocrop size 800,600 -set autoscale -set grid -set style data linespoints -set title "%(title)s" -set xlabel "Time" -set xtics rotate by -45 -set xdata time -set timefmt "%(timefmt)s" -set format x "%(timefmt)s" -set ylabel "%(ylabel)s" -set yrange [0:] -set output 'reports/report.%(num)s.png' -plot "reports/raw/report.%(num)s.raw" using 1:4 notitle''' % { - 'num' : num, - 'timefmt' : TIMEFMT, - 'title' : title, - 'ylabel' : ylabel, - }) - of.close() - - # Run gnuplot. - call(["gnuplot", "gnuplot.cmd"]) - - # Remove the command file. - os.remove("gnuplot.cmd") - -# Read the input into a dictionary of lists. -if sys.argv[1:] == []: - print "usage: " + sys.argv[0] + " file ..." - sys.exit(1) - -# Remove and re-create the reports folder. -shutil.rmtree("reports", True) -os.makedirs("reports/raw") - -d = defaultdict(list) -for line in fileinput.input(sys.argv[1:]): - month, day, time, v, desc = line.strip('\n').split(" ", 4) - d[desc].append((month + " " + day + " " + time, v)) - -# Plot each entry in the dictionary. -for items in sorted(d.iteritems()): - plot('\\n'.join(l for l in textwrap.wrap(items[0], 60)), items[1]) - diff --git a/src/third_party/wiredtiger/tools/wt_nvd3_util.py b/src/third_party/wiredtiger/tools/wt_nvd3_util.py deleted file mode 100644 index 435c2ce71e7..00000000000 --- a/src/third_party/wiredtiger/tools/wt_nvd3_util.py +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env python -# -# Public Domain 2014-2015 MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# -# This is free and unencumbered software released into the public domain. -# -# Anyone is free to copy, modify, publish, use, compile, sell, or -# distribute this software, either in source code form or as a compiled -# binary, for any purpose, commercial or non-commercial, and by any -# means. -# -# In jurisdictions that recognize copyright laws, the author or authors -# of this software dedicate any and all copyright interest in the -# software to the public domain. We make this dedication for the benefit -# of the public at large and to the detriment of our heirs and -# successors. We intend this dedication to be an overt act of -# relinquishment in perpetuity of all present and future rights to this -# software under copyright law. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -# -import os, sys -from datetime import datetime - -tool_dir = os.path.split(sys.argv[0])[0] -# Make sure Python finds the NVD3 in our third party directory, to -# avoid compatability issues -sys.path.append(os.path.join(tool_dir, "3rdparty")) - -try: - from nvd3 import lineChart -except ImportError: - print >>sys.stderr, "Could not import nvd3. It should be installed locally." - sys.exit(-1) - -# Add a multiChart type so we can overlay line graphs -class multiChart(lineChart): - def __init__(self, **kwargs): - lineChart.__init__(self, **kwargs) - - # Fix the axes - del self.axislist['yAxis'] - self.create_y_axis('yAxis1', format=kwargs.get('y_axis_format', '.02f')) - self.create_y_axis('yAxis2', format=kwargs.get('y_axis_format', '.02f')) - -TIMEFMT = "%b %d %H:%M:%S" - -thisyear = datetime.today().year -def parsetime(s): - return datetime.strptime(s, TIMEFMT).replace(year=thisyear) - diff --git a/src/third_party/wiredtiger/tools/wtperf_graph.py b/src/third_party/wiredtiger/tools/wtperf_graph.py deleted file mode 100644 index d80eb0cbce5..00000000000 --- a/src/third_party/wiredtiger/tools/wtperf_graph.py +++ /dev/null @@ -1,235 +0,0 @@ -#!/usr/bin/env python -# -# Public Domain 2014-2015 MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# -# This is free and unencumbered software released into the public domain. -# -# Anyone is free to copy, modify, publish, use, compile, sell, or -# distribute this software, either in source code form or as a compiled -# binary, for any purpose, commercial or non-commercial, and by any -# means. -# -# In jurisdictions that recognize copyright laws, the author or authors -# of this software dedicate any and all copyright interest in the -# software to the public domain. We make this dedication for the benefit -# of the public at large and to the detriment of our heirs and -# successors. We intend this dedication to be an overt act of -# relinquishment in perpetuity of all present and future rights to this -# software under copyright law. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -# - -import csv, os, sys -from subprocess import call -# Python script to read wtperf monitor output and create a performance -# graph. - -TIMEFMT = "%b %d %H:%M:%S" - -def process_monitor(fname, sfx, ckptlist, opdict): - # Read the monitor file and figure out when a checkpoint was running. - in_ckpt = 'N' - - ckptlist=[] - - ofname = 'monitor%s.png' % (sfx) - # Monitor output format currently is: - # time,totalsec,read,insert,update,ckpt,...latencies... - ops = ('read', 'insert', 'update') - csvcol = (2, 3, 4) - with open(fname, 'r') as csvfile: - reader = csv.reader(csvfile) - for row in reader: - if row[0].lstrip().startswith('#'): - continue - # Look for checkpoints and operations. - if row[5] != in_ckpt: - ckptlist.append(row[0]) - in_ckpt = row[5] - for op, col in zip(ops, csvcol): - if row[col] != '0' and opdict[op] == 0: - opdict[op] = 1 - - if in_ckpt == 'Y': - ckptlist.append(row[0]) - - # Graph time vs. read, insert and update operations per second. - gcmd = "gnuplot.mon.cmd" - of = open(gcmd, "w") - of.write(''' -set autoscale -set datafile sep ',' -set grid -set style data lines -set terminal png nocrop size 800,600 -set timefmt "%(TIMEFMT)s" -set title "read, insert and update operations per second" -set format x "%(TIMEFMT)s" -set xlabel "Time" -set xtics rotate by -45 -set xdata time -set ylabel "Operations per second (thousands)" -set yrange [0:]\n''' % { - 'TIMEFMT' : TIMEFMT - }) - it = iter(ckptlist) - for start, stop in zip(it, it): - of.write("set object rectangle from first '%s',\ - graph 0 to first '%s',\ - graph 1 fc rgb \"gray\" back\n" % (start, stop)) - of.write('set output "%s"\n' % (ofname)) - of.write("""plot "{name}" using 1:($3/1000) title "Reads", \\ - "{name}" using 1:($4/1000) title "Inserts",\\ - "{name}" using 1:($5/1000) title "Updates" - """.format(name=fname)) - of.close() - call(["gnuplot", gcmd]) - os.remove(gcmd) - -# Graph time vs. average, minimium, maximum latency for an operation. -def plot_latency_operation(name, fname, sfx, ckptlist, col_avg, col_min, col_max): - gcmd = "gnuplot." + name + ".l1.cmd" - of = open(gcmd, "w") - of.write(''' -set autoscale -set datafile sep ',' -set grid -set style data lines -set terminal png nocrop size 800,600 -set timefmt "%(TIMEFMT)s" -set title "%(NAME)s: average, minimum and maximum latency" -set format x "%(TIMEFMT)s" -set xlabel "Time" -set xtics rotate by -45 -set xdata time -set ylabel "Latency (us)" -set logscale y -set yrange [1:]\n''' % { - 'NAME' : name, - 'TIMEFMT' : TIMEFMT - }) - it = iter(ckptlist) - for start, stop in zip(it, it): - of.write('set object rectangle from first \'' + start +\ - '\', graph 0 ' + ' to first \'' + stop +\ - '\', graph 1 fc rgb "gray" back\n') - ofname = name + sfx + '.latency1.png' - of.write('set output "' + ofname + '"\n') - of.write('plot "' +\ - fname + '" using 1:($' + repr(col_avg) +\ - ') title "Average Latency", "' + fname +'" using 1:($' +\ - repr(col_min) + ') title "Minimum Latency", "' +\ - fname + '" using 1:($' + repr(col_max) +\ - ') title "Maximum Latency"\n') - of.close() - call(["gnuplot", gcmd]) - os.remove(gcmd) - - -# Graph latency vs. % operations -def plot_latency_percent(name, dirname, sfx, ckptlist): - lfile = os.path.join(dirname, 'latency.' + name) - if not os.path.exists(lfile): - return - gcmd = "gnuplot." + name + ".l2.cmd" - of = open(gcmd, "w") - of.write(''' -set autoscale -set datafile sep ',' -set grid -set style data points -set terminal png nocrop size 800,600\n''') - of.write('set title "' + name + ': latency distribution"\n') - of.write(''' -set xlabel "Latency (us)" -set xrange [1:] -set xtics rotate by -45 -set logscale x -set ylabel "%% operations" -set yrange [0:]\n''') - ofname = name + sfx + '.latency2.png' - of.write('set output "' + ofname + '"\n') - of.write('plot "' + lfile + sfx +\ - '" using (($2 * 100)/$4) title "' + name + '"\n') - of.close() - call(["gnuplot", gcmd]) - os.remove(gcmd) - - -# Graph latency vs. % operations (cumulative) -def plot_latency_cumulative_percent(name, dirname, sfx, ckptlist): - lfile = os.path.join(dirname, 'latency.' + name) - if not os.path.exists(lfile): - return - # Latency plot: cumulative operations vs. latency - gcmd = "gnuplot." + name + ".l3.cmd" - of = open(gcmd, "w") - of.write(''' -set autoscale -set datafile sep ',' -set grid -set style data lines -set terminal png nocrop size 800,600 -set title "%(NAME)s: cumulative latency distribution" -set xlabel "Latency (us)" -set xrange [1:] -set xtics rotate by -45 -set logscale x -set ylabel "%% operations" -set yrange [0:]\n''' % { - 'NAME' : name - }) - ofname = name + sfx + '.latency3.png' - of.write('set output "' + ofname + '"\n') - of.write('plot "' + lfile + sfx +\ - '" using 1:(($3 * 100)/$4) title "' + name + '"\n') - of.close() - call(["gnuplot", gcmd]) - os.remove(gcmd) - -def process_file(fname): - ckptlist = [] - # NOTE: The operations below must be in this exact order to match - # the operation latency output in the monitor file. - opdict={'read':0, 'insert':0, 'update':0} - - # This assumes the monitor file has the string "monitor" - # and any other (optional) characters in the filename are a suffix. - sfx = os.path.basename(fname).replace('monitor','') - dirname = os.path.dirname(fname) - - process_monitor(fname, sfx, ckptlist, opdict) - column = 7 # average, minimum, maximum start in column 7 - for k, v in opdict.items(): - if v != 0: - plot_latency_operation( - k, fname, sfx, ckptlist, column, column + 1, column + 2) - plot_latency_percent(k, dirname, sfx, ckptlist) - plot_latency_cumulative_percent(k, dirname, sfx, ckptlist) - else: - print fname + ': no ' + k + ' operations found. Skip.' - column = column + 3 - -def main(): - # This program takes a list of monitor files generated by - # wtperf. If no args are given, it looks for a single file - # named 'monitor'. - numargs = len(sys.argv) - if numargs < 2: - process_file('monitor') - else: - d = 1 - while d < numargs: - process_file(sys.argv[d]) - d += 1 - -if __name__ == '__main__': - main() diff --git a/src/third_party/wiredtiger/tools/wtperf_stats.py b/src/third_party/wiredtiger/tools/wtperf_stats.py deleted file mode 100644 index 5e6e1733fb7..00000000000 --- a/src/third_party/wiredtiger/tools/wtperf_stats.py +++ /dev/null @@ -1,178 +0,0 @@ -#!/usr/bin/env python -# -# Public Domain 2014-2015 MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# -# This is free and unencumbered software released into the public domain. -# -# Anyone is free to copy, modify, publish, use, compile, sell, or -# distribute this software, either in source code form or as a compiled -# binary, for any purpose, commercial or non-commercial, and by any -# means. -# -# In jurisdictions that recognize copyright laws, the author or authors -# of this software dedicate any and all copyright interest in the -# software to the public domain. We make this dedication for the benefit -# of the public at large and to the detriment of our heirs and -# successors. We intend this dedication to be an overt act of -# relinquishment in perpetuity of all present and future rights to this -# software under copyright law. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -# - -import os, csv, operator, sys -from time import mktime - -tool_dir = os.path.split(sys.argv[0])[0] -sys.path.append(tool_dir) - -try: - from wt_nvd3_util import multiChart, parsetime -except ImportError: - print >>sys.stderr, "Could not import wt_nvd3_util.py, it should be\ - in the same directory as %s" % sys.argv[0] - sys.exit(-1) - -def timesort(s): - # Sort the timestr via its parsetime() value so that the year gets - # added and it properly sorts. Times are only %b %d %H:%M:%S and - # may improperly sort if the data crosses a month boundary. - t = operator.itemgetter('#time') - timestr = t(s) - return parsetime(timestr) - -# Fixup the names and values in a dictionary read in from a csv file. One -# field must be "#time" - which is used to calculate the interval. -# Input is a dictionary, output is a list of dictionaries with a single entry. -def munge_dict(values_dict, abstime): - sorted_values = sorted(values_dict, key=timesort) - start_time = parsetime(sorted_values[0]['#time']) - - ret = [] - for v in sorted_values: - if abstime: - # Build the time series, milliseconds since the epoch - v['#time'] = int(mktime(parsetime(v['#time']).timetuple())) * 1000 - else: - # Build the time series as seconds since the start of the data - v['#time'] = (parsetime(v['#time']) - start_time).seconds - next_val = {} - for title, value in v.items(): - if title.find('uS') != -1: - title = title.replace('uS', 'ms') - value = float(value) / 1000 - if title == 'totalsec': - value = 0 - if title == 'checkpoints' and value == 'N': - value = 0 - elif title.find('time') != -1: - title = 'time' - elif title.find('latency') == -1 and \ - title.find('checkpoints') == -1: - title = title + ' (thousands)' - value = float(value) / 1000 - next_val[title] = value - ret.append(next_val) - - # After building the series, eliminate constants - d0 = ret[0] - for t0, v0 in d0.items(): - skip = True - for d in ret: - v = d[t0] - if v != v0: - skip = False - break - if skip: - for dicts in ret: - del dicts[t0] - - return ret - -def addPlotsToChart(chart, graph_data, wtstat_chart = False): - # Extract the times - they are the same for all lines. - times = [] - for v in graph_data: - times.append(v['time']) - - # Add a line to the graph for each field in the CSV file in alphabetical - # order, so the key is sorted. - for field in sorted(graph_data[0].keys()): - if field == 'time': - continue - # Split the latency and non-latency measurements onto different scales - axis = "1" - if not wtstat_chart and field.find('latency') == -1: - axis="2" - ydata = [] - for v in graph_data: - ydata.append(v[field]) - chart.add_serie(x=times, y=ydata, name=field, type="line", yaxis=axis) - -# Input parameters are a chart populated with WiredTiger statistics and -# the directory where the wtperf monitor file can be found. -def addPlotsToStatsChart(chart, dirname, abstime): - fname = os.path.join(dirname, 'monitor') - try: - with open(fname, 'rb') as csvfile: - reader = csv.DictReader(csvfile) - # Transform the data into something NVD3 can digest - graph_data = munge_dict(reader, abstime) - except IOError: - print >>sys.stderr, "Could not open wtperf monitor file." - sys.exit(-1) - addPlotsToChart(chart, graph_data, 1) - -def main(): - # Parse the command line - import argparse - - parser = argparse.ArgumentParser(description='Create graphs from WiredTiger statistics.') - parser.add_argument('--abstime', action='store_true', - help='use absolute time on the x axis') - parser.add_argument('--output', '-o', metavar='file', - default='wtperf_stats.html', help='HTML output file') - parser.add_argument('files', metavar='file', nargs='+', - help='input monitor file generated by WiredTiger wtperf application') - args = parser.parse_args() - - output_file = open(args.output, 'w') - - if len(args.files) != 1: - print 'Script currently only supports a single monitor file' - exit (1) - - chart_extra = {} - # Add in the x axis if the user wants time. - if args.abstime: - chart_extra['x_axis_format'] = '%H:%M:%S' - - for f in args.files: - with open(f, 'rb') as csvfile: - reader = csv.DictReader(csvfile) - # Transform the data into something NVD3 can digest - graph_data = munge_dict(reader, args.abstime) - - chart = multiChart(name='wtperf', - height=450 + 10*len(graph_data[0].keys()), - resize=True, - x_is_date=args.abstime, - assets_directory='http://source.wiredtiger.com/graphs/', - **chart_extra) - - addPlotsToChart(chart, graph_data) - - chart.buildhtml() - output_file.write(chart.htmlcontent) - output_file.close() - -if __name__ == '__main__': - main() - diff --git a/src/third_party/wiredtiger/tools/wtstats.py b/src/third_party/wiredtiger/tools/wtstats.py index 740a2017d63..031b7cb546f 100644..100755 --- a/src/third_party/wiredtiger/tools/wtstats.py +++ b/src/third_party/wiredtiger/tools/wtstats.py @@ -27,53 +27,30 @@ # OTHER DEALINGS IN THE SOFTWARE. # -import fileinput, os, re, shutil, sys, textwrap +import os, re, sys from collections import defaultdict from glob import glob -from time import mktime -from subprocess import call - -# Make sure Python can find files in the tools directory -tool_dir = os.path.split(sys.argv[0])[0] - -# Make sure Python finds the NVD3 in our third party directory. -# To avoid compatability issues, prepend it to the system path. -sys.path = [ os.path.join(tool_dir, "3rdparty") ] + sys.path +import json +from datetime import datetime try: from stat_data \ import groups, no_scale_per_second_list, no_clear_list, prefix_list except ImportError: - print >>sys.stderr, "Could not import stat_data.py, it should be\ - in the same directory as %s" % sys.argv[0] + print >>sys.stderr, "Could not import stat_data.py, it should be" \ + "in the same directory as %s" % sys.argv[0] sys.exit(-1) -try: - from wtperf_stats import addPlotsToStatsChart -except ImportError: - print >>sys.stderr, "Could not import wtperf_stats.py, it should be\ - in the same directory as %s" % sys.argv[0] - sys.exit(-1) - -try: - from wt_nvd3_util import multiChart, parsetime -except ImportError: - print >>sys.stderr, "Could not import wt_nvd3_util.py, it should be\ - in the same directory as %s" % sys.argv[0] - sys.exit(-1) - -try: - from nvd3 import lineChart, lineWithFocusChart -except ImportError: - print >>sys.stderr, "Could not import nvd3 it should be installed locally" - sys.exit(-1) +thisyear = datetime.today().year +def parsetime(s): + return datetime.strptime(s, "%b %d %H:%M:%S").replace(year=thisyear) if sys.version_info<(2,7,0): print >>sys.stderr, "You need python 2.7 or later to run this script" sys.exit(-1) # Plot a set of entries for a title. -def munge(title, values): +def munge(args, title, values): t0, v0 = values[0] start_time = parsetime(t0) @@ -96,13 +73,6 @@ def munge(title, values): ydata = {} last_value = 0.0 for t, v in sorted(values): - if args.abstime: - # Build the time series, milliseconds since the epoch - x = int(mktime(parsetime(t).timetuple())) * 1000 - else: - # Build the time series as seconds since the start of the data - x = (parsetime(t) - start_time).seconds - float_v = float(v) if not stats_cleared: float_v = float_v - last_value @@ -111,191 +81,185 @@ def munge(title, values): if float_v < 0: float_v = 0.0 last_value = float(v) - ydata[x] = float_v / seconds + ydata[t] = float_v / seconds return ylabel, ydata # Parse the command line import argparse -parser = argparse.ArgumentParser(description='Create graphs from WiredTiger statistics.') -parser.add_argument('--abstime', action='store_true', - help='use absolute time on the x axis') -parser.add_argument('--all', '-A', action='store_true', - help='generate all series as separate HTML output files by category') -parser.add_argument('--clear', action='store_true', - help='WiredTiger stats gathered with clear set') -parser.add_argument('--focus', action='store_true', - help='generate a chart with focus slider') -parser.add_argument('--include', '-I', metavar='regexp', - type=re.compile, action='append', - help='include series with titles matching the specifed regexp') -parser.add_argument('--list', action='store_true', - help='list the series that would be displayed') -parser.add_argument('--output', '-o', metavar='file', default='wtstats', - help='HTML output file prefix') -parser.add_argument('--right', '-R', metavar='regexp', - type=re.compile, action='append', - help='use the right axis for series with titles matching the specifed regexp') -parser.add_argument('--wtperf', '-w', action='store_true', - help='Plot wtperf statistics on the same graph') -parser.add_argument('files', metavar='file', nargs='+', - help='input files or directories generated by WiredTiger statistics logging') -args = parser.parse_args() - -# Don't require users to specify regexps twice for right axis -if args.focus and args.right: - print >>sys.stderr, "focus charts cannot have a right-hand y-axis" - sys.exit(-1) +def main(): + parser = argparse.ArgumentParser(description='Create graphs from' \ + 'WiredTiger statistics.') + parser.add_argument('--all', '-A', action='store_true', + help='generate separate html files for each stats group') + parser.add_argument('--clear', action='store_true', + help='WiredTiger stats gathered with clear set') + parser.add_argument('--include', '-I', metavar='regexp', + type=re.compile, action='append', + help='only include series with titles matching regexp') + parser.add_argument('--list', action='store_true', + help='only list the parsed series, does not create html file') + parser.add_argument('--output', '-o', metavar='file', default='wtstats', + help='HTML output file prefix') + parser.add_argument('--json', action='store_true', + help='additionally output data series in json format') + parser.add_argument('files', metavar='file', nargs='+', + help='input files or directories generated by WiredTiger statistics' \ + 'logging') + args = parser.parse_args() + + # Read the input file(s) into a dictionary of lists. + def getfiles(l): + for f in l: + if os.path.isfile(f): + yield f + elif os.path.isdir(f): + for s in glob(os.path.join(f, 'WiredTigerStat*')): + print 'Processing ' + s + yield s + + d = defaultdict(list) + for f in getfiles(args.files): + for line in open(f, 'rU'): + month, day, time, v, title = line.strip('\n').split(" ", 4) + d[title].append((month + " " + day + " " + time, v)) + + # Process the series, eliminate constants + for title, values in sorted(d.iteritems()): + skip = True + t0, v0 = values[0] + for t, v in values: + if v != v0: + skip = False + break + if skip: + #print "Skipping", title + del d[title] + + # Common prefix / suffix elimination + prefix = suffix = None + + def common_prefix(a, b): + while not b.startswith(a): + a = a[:-1] + return a + + def common_suffix(a, b): + while not a.endswith(b): + b = b[1:] + return b + + def output_series(results, prefix=None, grouplist=[]): + # add .html ending if not present + filename, ext = os.path.splitext(args.output) + if ext == '': + ext = '.html' + + # open the output file based on prefix + if prefix == None: + outputname = filename + ext + elif len(grouplist) == 0: + outputname = filename +'.' + prefix + ext + else: + outputname = filename +'.group.' + prefix + ext -# Don't require users to specify regexps twice for right axis -if args.include and args.right: - args.include += args.right - -# Read the input file(s) into a dictionary of lists. -def getfiles(l): - for f in l: - if os.path.isfile(f): - yield f - elif os.path.isdir(f): - for s in glob(os.path.join(f, 'WiredTigerStat*')): - print 'Processing ' + s - yield s - -d = defaultdict(list) -for f in getfiles(args.files): - for line in open(f, 'rU'): - month, day, time, v, title = line.strip('\n').split(" ", 4) - d[title].append((month + " " + day + " " + time, v)) - -# Process the series, eliminate constants -for title, values in sorted(d.iteritems()): - skip = True - t0, v0 = values[0] - for t, v in values: - if v != v0: - skip = False - break - if skip: - #print "Skipping", title - del d[title] - -# Common prefix / suffix elimination -prefix = suffix = None - -def common_prefix(a, b): - while not b.startswith(a): - a = a[:-1] - return a - -def common_suffix(a, b): - while not a.endswith(b): - b = b[1:] - return b - -def output_series(results, prefix=None, grouplist=[]): - # open the output file based on prefix - if prefix == None: - outputname = args.output + '.html' - elif len(grouplist) == 0: - outputname = args.output +'.' + prefix + '.html' - else: - outputname = args.output +'.group.' + prefix + '.html' - - if prefix != None and len(grouplist) == 0: - this_series = [] - for title, yaxis, ydata in results: - if not prefix in title: - continue - #print 'Appending to dataset: ' + title - this_series.append((title, yaxis, ydata)) - elif prefix != None and len(grouplist) > 0: - this_series = [] - for title, yaxis, ydata in results: - for subgroup in grouplist: - if not subgroup in title: + if prefix != None and len(grouplist) == 0: + this_series = [] + for title, ydata in results: + if not prefix in title: continue - # print 'Appending to dataset: ' + title - this_series.append((title, yaxis, ydata)) - else: - this_series = results - - if len(this_series) == 0: - print 'Output: ' + outputname + ' has no data. Do not create.' - return - - #--------------------------------------- - if args.right: - charttype = multiChart - elif args.focus: - charttype = lineWithFocusChart - else: - charttype = lineChart - - chart_extra = {} - # Add in the x axis if the user wants time. - if args.abstime: - chart_extra['x_axis_format'] = '%H:%M:%S' - - # Create the chart, add the series - chart = charttype(name='statlog', height=450+10*len(this_series), resize=True, x_is_date=args.abstime, y_axis_format='g', assets_directory='http://source.wiredtiger.com/graphs/', **chart_extra) - - for title, yaxis, ydata in this_series: - chart.add_serie(x=xdata, y=(ydata.get(x, 0) for x in xdata), name=title, - type="line", yaxis="2" if yaxis else "1") - - if args.wtperf: - addPlotsToStatsChart(chart, os.path.dirname(args.files[0]), args.abstime) - - chart.buildhtml() - output_file = open(outputname, 'w') - output_file.write(chart.htmlcontent) - - #close Html file - output_file.close() - - -# Split out the data, convert timestamps -results = [] -for title, values in sorted(d.iteritems()): - title, ydata = munge(title, values) - # Ignore entries if a list of regular expressions was given - if args.include and not [r for r in args.include if r.search(title)]: + #print 'Appending to dataset: ' + title + this_series.append((title, ydata)) + elif prefix != None and len(grouplist) > 0: + this_series = [] + for title, ydata in results: + for subgroup in grouplist: + if not subgroup in title: + continue + # print 'Appending to dataset: ' + title + this_series.append((title, ydata)) + else: + this_series = results + + if len(this_series) == 0: + print 'Output: ' + outputname + ' has no data. Do not create.' + return + + + json_output = { "series": [] } + + for title, ydata in this_series: + json_output["series"].append({ + "key": title, + "values": ydata, + }); + + # load template + this_path = os.path.dirname(os.path.realpath(__file__)) + srcfile = os.path.join(this_path, 'wtstats.html.template') + try: + srcfile = open(srcfile) + contents = srcfile.read() + except IOError: + print >>sys.stderr, "Cannot find template file 'wtstats.html." \ + "template'. See ./template/README.md for more information." + sys.exit(-1) + + srcfile.close() + + # if --json write data to <filename>.json + if args.json: + jsonfile = filename + '.json' + with open(jsonfile, 'w') as f: + json.dump(json_output, f) + print "created %s" % jsonfile + + # write output file + dstfile = open(outputname, 'wt') + replaced_contents = contents.replace('"### INSERT DATA HERE ###"', + json.dumps(json_output)) + dstfile.write(replaced_contents) + dstfile.close() + print "created %s" % dstfile.name + + # Split out the data, convert timestamps + results = [] + for title, values in sorted(d.iteritems()): + title, ydata = munge(args, title, values) + # Ignore entries if a list of regular expressions was given + if args.include and not [r for r in args.include if r.search(title)]: continue - yaxis = args.right and [r for r in args.right if r.search(title)] - prefix = title if prefix is None else common_prefix(prefix, title) - suffix = title if suffix is None else common_suffix(title, suffix) - results.append((title, yaxis, ydata)) - -# Process titles, eliminate common prefixes and suffixes -if prefix or suffix: - new_results = [] - for title, yaxis, ydata in results: - title = title[len(prefix):] - if suffix: - title = title[:-len(suffix)] - new_results.append((title, yaxis, ydata)) - results = new_results - -# Dump the results as a CSV file -#print '"time", ' + ', '.join('"%s"' % title for title, values in ydata) -#for i in xrange(len(xdata)): -# print '%d, %s' % (xdata[i], ', '.join('%g' % values[i] for title, values in ydata)) - -# Are we just listing the results? -if args.list: - for title, yaxis, ydata in results: - print title - sys.exit(0) - -# Figure out the full set of x axis values -xdata = sorted(set(k for k in ydata.iterkeys() for ydata in results)) - -output_series(results) - -# If the user wants the stats split up by prefix type do so. -if args.all: - for prefix in prefix_list: - output_series(results, prefix) - for group in groups.keys(): - output_series(results, group, groups[group]) + prefix = title if prefix is None else common_prefix(prefix, title) + suffix = title if suffix is None else common_suffix(title, suffix) + results.append((title, ydata)) + + # Process titles, eliminate common prefixes and suffixes + if prefix or suffix: + new_results = [] + for title, ydata in results: + title = title[len(prefix):] + if suffix: + title = title[:-len(suffix)] + new_results.append((title, ydata)) + results = new_results + + # Are we just listing the results? + if args.list: + for title, ydata in results: + print title + sys.exit(0) + + output_series(results) + + # If the user wants the stats split up by prefix type do so. + if args.all: + for prefix in prefix_list: + output_series(results, prefix) + for group in groups.keys(): + output_series(results, group, groups[group]) + + +if __name__ == '__main__': + main() + |