summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMatt Kangas <matt.kangas@mongodb.com>2015-01-20 17:08:32 -0500
committerMatt Kangas <matt.kangas@mongodb.com>2015-01-20 18:23:10 -0500
commit876f5f475bf9242c098e6de641aa7ee0982803c2 (patch)
tree43997e6356af70bcf77d4805104c28b845295826
parentb6f25f8a9804e8c7efc995c8a40a3fa69da857e4 (diff)
downloadmongo-876f5f475bf9242c098e6de641aa7ee0982803c2.tar.gz
Import wiredtiger-wiredtiger-mongodb-2.8-rc5-168-gcbd3d5e.tar.gz from wiredtiger branch mongodb-2.8
-rw-r--r--src/third_party/wiredtiger/.hgtags2
-rw-r--r--src/third_party/wiredtiger/bench/wtperf/runners/overflow-10k-short.wtperf (renamed from src/third_party/wiredtiger/bench/wtperf/runners/voxer-10k-short.wtperf)0
-rw-r--r--src/third_party/wiredtiger/bench/wtperf/runners/overflow-10k.wtperf (renamed from src/third_party/wiredtiger/bench/wtperf/runners/voxer-10k.wtperf)0
-rw-r--r--src/third_party/wiredtiger/bench/wtperf/runners/overflow-130k-short.wtperf (renamed from src/third_party/wiredtiger/bench/wtperf/runners/voxer-130k-short.wtperf)0
-rw-r--r--src/third_party/wiredtiger/bench/wtperf/runners/overflow-130k.wtperf (renamed from src/third_party/wiredtiger/bench/wtperf/runners/voxer-130k.wtperf)0
-rw-r--r--src/third_party/wiredtiger/dist/api_data.py29
-rw-r--r--src/third_party/wiredtiger/dist/flags.py1
-rw-r--r--src/third_party/wiredtiger/dist/s_string.ok1
-rw-r--r--src/third_party/wiredtiger/dist/s_wtstats37
-rw-r--r--src/third_party/wiredtiger/lang/java/java_doc.i2
-rw-r--r--src/third_party/wiredtiger/lang/java/wiredtiger.i19
-rw-r--r--src/third_party/wiredtiger/lang/python/wiredtiger.i33
-rw-r--r--src/third_party/wiredtiger/src/block/block_compact.c10
-rw-r--r--src/third_party/wiredtiger/src/btree/bt_cursor.c50
-rw-r--r--src/third_party/wiredtiger/src/btree/bt_split.c141
-rw-r--r--src/third_party/wiredtiger/src/config/config_def.c10
-rw-r--r--src/third_party/wiredtiger/src/conn/conn_log.c108
-rw-r--r--src/third_party/wiredtiger/src/conn/conn_open.c3
-rw-r--r--src/third_party/wiredtiger/src/cursor/cur_backup.c2
-rw-r--r--src/third_party/wiredtiger/src/cursor/cur_config.c2
-rw-r--r--src/third_party/wiredtiger/src/cursor/cur_ds.c2
-rw-r--r--src/third_party/wiredtiger/src/cursor/cur_dump.c2
-rw-r--r--src/third_party/wiredtiger/src/cursor/cur_file.c41
-rw-r--r--src/third_party/wiredtiger/src/cursor/cur_index.c2
-rw-r--r--src/third_party/wiredtiger/src/cursor/cur_log.c2
-rw-r--r--src/third_party/wiredtiger/src/cursor/cur_metadata.c15
-rw-r--r--src/third_party/wiredtiger/src/cursor/cur_stat.c12
-rw-r--r--src/third_party/wiredtiger/src/cursor/cur_std.c115
-rw-r--r--src/third_party/wiredtiger/src/cursor/cur_table.c94
-rw-r--r--src/third_party/wiredtiger/src/evict/evict_page.c8
-rw-r--r--src/third_party/wiredtiger/src/include/config.h47
-rw-r--r--src/third_party/wiredtiger/src/include/connection.h12
-rw-r--r--src/third_party/wiredtiger/src/include/cursor.h4
-rw-r--r--src/third_party/wiredtiger/src/include/extern.h4
-rw-r--r--src/third_party/wiredtiger/src/include/flags.h19
-rw-r--r--src/third_party/wiredtiger/src/include/wiredtiger.in46
-rw-r--r--src/third_party/wiredtiger/src/log/log.c68
-rw-r--r--src/third_party/wiredtiger/src/lsm/lsm_cursor.c4
-rw-r--r--src/third_party/wiredtiger/src/reconcile/rec_write.c26
-rw-r--r--src/third_party/wiredtiger/src/session/session_compact.c10
-rw-r--r--src/third_party/wiredtiger/tools/statlog.py125
-rw-r--r--src/third_party/wiredtiger/tools/wt_nvd3_util.py58
-rw-r--r--src/third_party/wiredtiger/tools/wtperf_graph.py235
-rw-r--r--src/third_party/wiredtiger/tools/wtperf_stats.py178
-rwxr-xr-x[-rw-r--r--]src/third_party/wiredtiger/tools/wtstats.py400
45 files changed, 862 insertions, 1117 deletions
diff --git a/src/third_party/wiredtiger/.hgtags b/src/third_party/wiredtiger/.hgtags
index 21c8b360301..ca12821f6c8 100644
--- a/src/third_party/wiredtiger/.hgtags
+++ b/src/third_party/wiredtiger/.hgtags
@@ -23,3 +23,5 @@ aff8aabe571be6db68e8bf44bf7670df5d55d1ff 1.5.0
b5c9f28d72fe1f835d24fe427e211a539f8709fe 1.5.2
03ab950d31edfe1b77aa6e7259e64dbdd3e17fe2 1.6.5
828cbc550d7136a3ee67be3781cade8e96ec3af7 2.8-rc3
+5c9acd2584f2657dec2a44fd8b54211bf9c21193 mongodb-2.8-rc5
+f40795b146bd35a623ef57de5b875a817925b7c9 mongodb-2.8-rc5
diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/voxer-10k-short.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/overflow-10k-short.wtperf
index 47228079db8..47228079db8 100644
--- a/src/third_party/wiredtiger/bench/wtperf/runners/voxer-10k-short.wtperf
+++ b/src/third_party/wiredtiger/bench/wtperf/runners/overflow-10k-short.wtperf
diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/voxer-10k.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/overflow-10k.wtperf
index 9b4ed2acaee..9b4ed2acaee 100644
--- a/src/third_party/wiredtiger/bench/wtperf/runners/voxer-10k.wtperf
+++ b/src/third_party/wiredtiger/bench/wtperf/runners/overflow-10k.wtperf
diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/voxer-130k-short.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/overflow-130k-short.wtperf
index 83f67062bf8..83f67062bf8 100644
--- a/src/third_party/wiredtiger/bench/wtperf/runners/voxer-130k-short.wtperf
+++ b/src/third_party/wiredtiger/bench/wtperf/runners/overflow-130k-short.wtperf
diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/voxer-130k.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/overflow-130k.wtperf
index a3439f0c575..a3439f0c575 100644
--- a/src/third_party/wiredtiger/bench/wtperf/runners/voxer-130k.wtperf
+++ b/src/third_party/wiredtiger/bench/wtperf/runners/overflow-130k.wtperf
diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py
index 7e964252c83..0141526285c 100644
--- a/src/third_party/wiredtiger/dist/api_data.py
+++ b/src/third_party/wiredtiger/dist/api_data.py
@@ -559,6 +559,20 @@ common_wiredtiger_open = [
]),
]
+cursor_runtime_config = [
+ Config('append', 'false', r'''
+ append the value as a new record, creating a new record
+ number key; valid only for cursors with record number keys''',
+ type='boolean'),
+ Config('overwrite', 'true', r'''
+ configures whether the cursor's insert, update and remove
+ methods check the existing state of the record. If \c overwrite
+ is \c false, WT_CURSOR::insert fails with ::WT_DUPLICATE_KEY
+ if the record exists, WT_CURSOR::update and WT_CURSOR::remove
+ fail with ::WT_NOTFOUND if the record does not exist''',
+ type='boolean'),
+]
+
methods = {
'file.meta' : Method(file_meta),
@@ -570,6 +584,8 @@ methods = {
'cursor.close' : Method([]),
+'cursor.reconfigure' : Method(cursor_runtime_config),
+
'session.close' : Method([]),
'session.compact' : Method([
@@ -600,11 +616,7 @@ methods = {
'session.log_printf' : Method([]),
-'session.open_cursor' : Method([
- Config('append', 'false', r'''
- append the value as a new record, creating a new record
- number key; valid only for cursors with record number keys''',
- type='boolean'),
+'session.open_cursor' : Method(cursor_runtime_config + [
Config('bulk', 'false', r'''
configure the cursor for bulk-loading, a fast, initial load
path (see @ref tune_bulk_load for more information). Bulk-load
@@ -641,13 +653,6 @@ methods = {
WT_CURSOR::next and WT_CURSOR::close methods. See @ref
cursor_random for details''',
type='boolean'),
- Config('overwrite', 'true', r'''
- configures whether the cursor's insert, update and remove
- methods check the existing state of the record. If \c overwrite
- is \c false, WT_CURSOR::insert fails with ::WT_DUPLICATE_KEY
- if the record exists, WT_CURSOR::update and WT_CURSOR::remove
- fail with ::WT_NOTFOUND if the record does not exist''',
- type='boolean'),
Config('raw', 'false', r'''
ignore the encodings for the key and value, manage data as if
the formats were \c "u". See @ref cursor_raw for details''',
diff --git a/src/third_party/wiredtiger/dist/flags.py b/src/third_party/wiredtiger/dist/flags.py
index 2d68d932c02..a0e307debf6 100644
--- a/src/third_party/wiredtiger/dist/flags.py
+++ b/src/third_party/wiredtiger/dist/flags.py
@@ -90,6 +90,7 @@ flags = {
'CONN_CKPT_SYNC',
'CONN_EVICTION_RUN',
'CONN_LEAK_MEMORY',
+ 'CONN_LOG_SERVER_RUN',
'CONN_LSM_MERGE',
'CONN_PANIC',
'CONN_SERVER_RUN',
diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok
index d3717d27331..6c658df8bf0 100644
--- a/src/third_party/wiredtiger/dist/s_string.ok
+++ b/src/third_party/wiredtiger/dist/s_string.ok
@@ -581,6 +581,7 @@ enum's
env
eof
eop
+equalp
errhandler
errno
errv
diff --git a/src/third_party/wiredtiger/dist/s_wtstats b/src/third_party/wiredtiger/dist/s_wtstats
new file mode 100644
index 00000000000..e55157431f1
--- /dev/null
+++ b/src/third_party/wiredtiger/dist/s_wtstats
@@ -0,0 +1,37 @@
+#! /bin/sh
+
+# Create wtstats.template.html file
+
+t=__wt.$$
+trap 'rm -f $t; exit 0' 0 1 2 3 13 15
+
+out=wtstats.html.template
+# We require npm which may not be installed.
+type npm > /dev/null 2>&1 || {
+ echo 's_wtstats: npm not found' >&2
+ echo ' npm is part of node.js from http://nodejs.org' >&2
+ exit 1
+}
+
+cd ../tools/template || exit 1
+rm -f ./$out
+# Note: we don't do the npm install here, it downloads files
+test -d ./node_modules || {
+ echo 's_wtstats: missing node_modules directory, install by:' >&2
+ echo ' cd ../tools/template; npm install' >&2
+ exit 1
+}
+npm run build >$t 2>&1 || {
+ echo "s_wtstats: npm failed" >&2
+ cat $t
+ exit 1
+}
+test -f ./$out || {
+ echo "s_wtstats: $out not created" >&2
+ cat $t
+ exit 1
+}
+f=../$out
+cmp ./$out $f > /dev/null 2>&1 ||
+ (echo "Building tools/$out" && rm -f $f && cp ./$out $f)
+exit 0
diff --git a/src/third_party/wiredtiger/lang/java/java_doc.i b/src/third_party/wiredtiger/lang/java/java_doc.i
index 31bad525330..53785a3bab4 100644
--- a/src/third_party/wiredtiger/lang/java/java_doc.i
+++ b/src/third_party/wiredtiger/lang/java/java_doc.i
@@ -5,6 +5,7 @@ COPYDOC(__wt_cursor, WT_CURSOR, get_value)
COPYDOC(__wt_cursor, WT_CURSOR, set_key)
COPYDOC(__wt_cursor, WT_CURSOR, set_value)
COPYDOC(__wt_cursor, WT_CURSOR, compare)
+COPYDOC(__wt_cursor, WT_CURSOR, equals)
COPYDOC(__wt_cursor, WT_CURSOR, next)
COPYDOC(__wt_cursor, WT_CURSOR, prev)
COPYDOC(__wt_cursor, WT_CURSOR, reset)
@@ -14,6 +15,7 @@ COPYDOC(__wt_cursor, WT_CURSOR, insert)
COPYDOC(__wt_cursor, WT_CURSOR, update)
COPYDOC(__wt_cursor, WT_CURSOR, remove)
COPYDOC(__wt_cursor, WT_CURSOR, close)
+COPYDOC(__wt_cursor, WT_CURSOR, reconfigure)
COPYDOC(__wt_async_op, WT_ASYNC_OP, get_key)
COPYDOC(__wt_async_op, WT_ASYNC_OP, get_value)
COPYDOC(__wt_async_op, WT_ASYNC_OP, set_key)
diff --git a/src/third_party/wiredtiger/lang/java/wiredtiger.i b/src/third_party/wiredtiger/lang/java/wiredtiger.i
index 09290a70c67..a92247c7ebf 100644
--- a/src/third_party/wiredtiger/lang/java/wiredtiger.i
+++ b/src/third_party/wiredtiger/lang/java/wiredtiger.i
@@ -293,6 +293,8 @@ WT_CLASS(struct __wt_async_op, WT_ASYNC_OP, op)
%ignore __wt_cursor::compare(WT_CURSOR *, WT_CURSOR *, int *);
%rename (compare_wrap) __wt_cursor::compare;
+%ignore __wt_cursor::equals(WT_CURSOR *, WT_CURSOR *, int *);
+%rename (equals_wrap) __wt_cursor::equals;
%rename (AsyncOpType) WT_ASYNC_OPTYPE;
%rename (getKeyFormat) __wt_async_op::getKey_format;
%rename (getValueFormat) __wt_async_op::getValue_format;
@@ -1134,6 +1136,13 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler};
return cmp;
}
+ int equals_wrap(JNIEnv *jenv, WT_CURSOR *other) {
+ int cmp, ret = $self->equals($self, other, &cmp);
+ if (ret != 0)
+ throwWiredTigerException(jenv, ret);
+ return cmp;
+ }
+
%javamethodmodifiers java_init "protected";
int java_init(jobject jcursor) {
JAVA_CALLBACK *jcb = (JAVA_CALLBACK *)$self->lang_private;
@@ -1622,6 +1631,16 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler};
}
/**
+ * Compare this cursor's position to another Cursor.
+ *
+ * \return The result of the comparison.
+ */
+ public int equals(Cursor other)
+ throws WiredTigerException {
+ return equals_wrap(other);
+ }
+
+ /**
* Retrieve the next item in the table.
*
* \return The result of the comparison.
diff --git a/src/third_party/wiredtiger/lang/python/wiredtiger.i b/src/third_party/wiredtiger/lang/python/wiredtiger.i
index de5afb0a0fa..bf726ceac0a 100644
--- a/src/third_party/wiredtiger/lang/python/wiredtiger.i
+++ b/src/third_party/wiredtiger/lang/python/wiredtiger.i
@@ -390,6 +390,7 @@ NOTFOUND_OK(__wt_cursor::search)
NOTFOUND_OK(__wt_cursor::update)
COMPARE_OK(__wt_cursor::compare)
+COMPARE_OK(__wt_cursor::equals)
COMPARE_NOTFOUND_OK(__wt_cursor::search_near)
/* Lastly, some methods need no (additional) error checking. */
@@ -424,6 +425,7 @@ COMPARE_NOTFOUND_OK(__wt_cursor::search_near)
/* Next, override methods that return integers via arguments. */
%ignore __wt_cursor::compare(WT_CURSOR *, WT_CURSOR *, int *);
+%ignore __wt_cursor::equals(WT_CURSOR *, WT_CURSOR *, int *);
%ignore __wt_cursor::search_near(WT_CURSOR *, int *);
/* SWIG magic to turn Python byte strings into data / size. */
@@ -682,7 +684,7 @@ typedef int int_void;
return (ret);
}
- /* compare and search_near need special handling. */
+ /* compare: special handling. */
int compare(WT_CURSOR *other) {
int cmp = 0;
int ret = 0;
@@ -700,21 +702,40 @@ typedef int int_void;
* Map less-than-zero to -1 and greater-than-zero to 1
* to avoid colliding with other errors.
*/
- ret = ((ret != 0) ? ret :
- (cmp < 0) ? -1 : (cmp == 0) ? 0 : 1);
+ ret = (ret != 0) ? ret :
+ ((cmp < 0) ? -1 : (cmp == 0) ? 0 : 1);
}
return (ret);
}
+ /* equals: special handling. */
+ int equals(WT_CURSOR *other) {
+ int cmp = 0;
+ int ret = 0;
+ if (other == NULL) {
+ SWIG_Error(SWIG_NullReferenceError,
+ "in method 'Cursor_equals', "
+ "argument 1 of type 'struct __wt_cursor *' "
+ "is None");
+ ret = EINVAL; /* any non-zero value will do. */
+ }
+ else {
+ ret = $self->equals($self, other, &cmp);
+ if (ret == 0)
+ ret = cmp;
+ }
+ return (ret);
+ }
+
+ /* search_near: special handling. */
int search_near() {
int cmp = 0;
int ret = $self->search_near($self, &cmp);
/*
* Map less-than-zero to -1 and greater-than-zero to 1 to avoid
- * colliding with WT_NOTFOUND.
+ * colliding with other errors.
*/
- return ((ret != 0) ? ret :
- (cmp < 0) ? -1 : (cmp == 0) ? 0 : 1);
+ return ((ret != 0) ? ret : (cmp < 0) ? -1 : (cmp == 0) ? 0 : 1);
}
int _freecb() {
diff --git a/src/third_party/wiredtiger/src/block/block_compact.c b/src/third_party/wiredtiger/src/block/block_compact.c
index c7f2b09aa23..79494a274a9 100644
--- a/src/third_party/wiredtiger/src/block/block_compact.c
+++ b/src/third_party/wiredtiger/src/block/block_compact.c
@@ -137,8 +137,6 @@ __wt_block_compact_page_skip(WT_SESSION_IMPL *session,
/* Crack the cookie. */
WT_RET(__wt_block_buffer_to_addr(block, addr, &offset, &size, &cksum));
- __wt_spin_lock(session, &block->live_lock);
-
/*
* If this block is in the last 10% of the file and there's a block on
* the available list that's in the first 90% of the file, rewrite the
@@ -146,6 +144,7 @@ __wt_block_compact_page_skip(WT_SESSION_IMPL *session,
* the block would extend the file), but there's an obvious race if the
* file is sufficiently busy.
*/
+ __wt_spin_lock(session, &block->live_lock);
ninety = fh->size - fh->size / 10;
if (offset > ninety) {
el = &block->live.avail;
@@ -155,7 +154,6 @@ __wt_block_compact_page_skip(WT_SESSION_IMPL *session,
break;
}
}
-
__wt_spin_unlock(session, &block->live_lock);
return (ret);
@@ -176,7 +174,7 @@ __block_dump_avail(WT_SESSION_IMPL *session, WT_BLOCK *block)
el = &block->live.avail;
size = block->fh->size;
- WT_RET(__wt_verbose(session, WT_VERB_BLOCK,
+ WT_RET(__wt_verbose(session, WT_VERB_COMPACT,
"file size %" PRIuMAX "MB (%" PRIuMAX ") with %" PRIuMAX
"%% space available %" PRIuMAX "MB (%" PRIuMAX ")",
(uintmax_t)size / WT_MEGABYTE, (uintmax_t)size,
@@ -202,7 +200,7 @@ __block_dump_avail(WT_SESSION_IMPL *session, WT_BLOCK *block)
#ifdef __VERBOSE_OUTPUT_PERCENTILE
for (i = 0; i < WT_ELEMENTS(percentile); ++i) {
v = percentile[i] * 512;
- WT_RET(__wt_verbose(session, WT_VERB_BLOCK,
+ WT_RET(__wt_verbose(session, WT_VERB_COMPACT,
"%2u%%: %12" PRIuMAX "MB, (%" PRIuMAX "B, %"
PRIuMAX "%%)",
i, (uintmax_t)v / WT_MEGABYTE, (uintmax_t)v,
@@ -211,7 +209,7 @@ __block_dump_avail(WT_SESSION_IMPL *session, WT_BLOCK *block)
#endif
for (i = 0; i < WT_ELEMENTS(decile); ++i) {
v = decile[i] * 512;
- WT_RET(__wt_verbose(session, WT_VERB_BLOCK,
+ WT_RET(__wt_verbose(session, WT_VERB_COMPACT,
"%2u%%: %12" PRIuMAX "MB, (%" PRIuMAX "B, %"
PRIuMAX "%%)",
i * 10, (uintmax_t)v / WT_MEGABYTE, (uintmax_t)v,
diff --git a/src/third_party/wiredtiger/src/btree/bt_cursor.c b/src/third_party/wiredtiger/src/btree/bt_cursor.c
index 4aee1883ae0..704b258a7dd 100644
--- a/src/third_party/wiredtiger/src/btree/bt_cursor.c
+++ b/src/third_party/wiredtiger/src/btree/bt_cursor.c
@@ -800,16 +800,19 @@ err: if (ret != 0)
int
__wt_btcur_compare(WT_CURSOR_BTREE *a_arg, WT_CURSOR_BTREE *b_arg, int *cmpp)
{
- WT_BTREE *btree;
WT_CURSOR *a, *b;
WT_SESSION_IMPL *session;
a = (WT_CURSOR *)a_arg;
b = (WT_CURSOR *)b_arg;
- btree = a_arg->btree;
session = (WT_SESSION_IMPL *)a->session;
- switch (btree->type) {
+ /* Confirm both cursors reference the same object. */
+ if (a_arg->btree != b_arg->btree)
+ WT_RET_MSG(
+ session, EINVAL, "Cursors must reference the same object");
+
+ switch (a_arg->btree->type) {
case BTREE_COL_FIX:
case BTREE_COL_VAR:
/*
@@ -826,7 +829,7 @@ __wt_btcur_compare(WT_CURSOR_BTREE *a_arg, WT_CURSOR_BTREE *b_arg, int *cmpp)
break;
case BTREE_ROW:
WT_RET(__wt_compare(
- session, btree->collator, &a->key, &b->key, cmpp));
+ session, a_arg->btree->collator, &a->key, &b->key, cmpp));
break;
WT_ILLEGAL_VALUE(session);
}
@@ -837,7 +840,7 @@ __wt_btcur_compare(WT_CURSOR_BTREE *a_arg, WT_CURSOR_BTREE *b_arg, int *cmpp)
* __cursor_equals --
* Return if two cursors reference the same row.
*/
-static int
+static inline int
__cursor_equals(WT_CURSOR_BTREE *a, WT_CURSOR_BTREE *b)
{
switch (a->btree->type) {
@@ -867,6 +870,43 @@ __cursor_equals(WT_CURSOR_BTREE *a, WT_CURSOR_BTREE *b)
}
/*
+ * __wt_btcur_equals --
+ * Return an equality comparison between two cursors.
+ */
+int
+__wt_btcur_equals(
+ WT_CURSOR_BTREE *a_arg, WT_CURSOR_BTREE *b_arg, int *equalp)
+{
+ WT_CURSOR *a, *b;
+ WT_SESSION_IMPL *session;
+ int cmp;
+
+ a = (WT_CURSOR *)a_arg;
+ b = (WT_CURSOR *)b_arg;
+ session = (WT_SESSION_IMPL *)a->session;
+
+ /* Confirm both cursors reference the same object. */
+ if (a_arg->btree != b_arg->btree)
+ WT_RET_MSG(
+ session, EINVAL, "Cursors must reference the same object");
+
+ /*
+ * The reason for an equals method is because we can avoid doing
+ * a full key comparison in some cases. If both cursors point into the
+ * tree, take the fast path, otherwise fall back to the slower compare
+ * method; in both cases, return 1 if the cursors are equal, 0 if they
+ * are not.
+ */
+ if (F_ISSET(a, WT_CURSTD_KEY_INT) && F_ISSET(b, WT_CURSTD_KEY_INT))
+ *equalp = __cursor_equals(a_arg, b_arg);
+ else {
+ WT_RET(__wt_btcur_compare(a_arg, b_arg, &cmp));
+ *equalp = (cmp == 0) ? 1 : 0;
+ }
+ return (0);
+}
+
+/*
* __cursor_truncate --
* Discard a cursor range from row-store or variable-width column-store
* tree.
diff --git a/src/third_party/wiredtiger/src/btree/bt_split.c b/src/third_party/wiredtiger/src/btree/bt_split.c
index 4041bbb71b0..395ff447035 100644
--- a/src/third_party/wiredtiger/src/btree/bt_split.c
+++ b/src/third_party/wiredtiger/src/btree/bt_split.c
@@ -257,12 +257,13 @@ __split_ovfl_key_cleanup(WT_SESSION_IMPL *session, WT_PAGE *page, WT_REF *ref)
}
/*
- * __split_ref_instantiate --
- * Instantiate key/address pairs in memory in service of a split.
+ * __split_ref_deepen_move --
+ * Move a WT_REF from a parent to a child in service of a split to deepen
+ * the tree, including updating the accounting information.
*/
static int
-__split_ref_instantiate(WT_SESSION_IMPL *session,
- WT_PAGE *page, WT_REF *ref, size_t *parent_decrp, size_t *child_incrp)
+__split_ref_deepen_move(WT_SESSION_IMPL *session,
+ WT_PAGE *parent, WT_REF *ref, size_t *parent_decrp, size_t *child_incrp)
{
WT_ADDR *addr;
WT_CELL_UNPACK unpack;
@@ -279,8 +280,6 @@ __split_ref_instantiate(WT_SESSION_IMPL *session,
* of child pages, and so we can no longer reference the block image
* that remains with the page being split.
*
- * Track how much memory the parent is losing and the child gaining.
- *
* No locking is required to update the WT_REF structure because we're
* the only thread splitting the parent page, and there's no way for
* readers to race with our updates of single pointers. The changes
@@ -289,13 +288,13 @@ __split_ref_instantiate(WT_SESSION_IMPL *session,
*
* Row-store keys, first.
*/
- if (page->type == WT_PAGE_ROW_INT) {
+ if (parent->type == WT_PAGE_ROW_INT) {
if ((ikey = __wt_ref_key_instantiated(ref)) == NULL) {
- __wt_ref_key(page, ref, &key, &size);
+ __wt_ref_key(parent, ref, &key, &size);
WT_RET(__wt_row_ikey(session, 0, key, size, &ikey));
ref->key.ikey = ikey;
} else {
- WT_RET(__split_ovfl_key_cleanup(session, page, ref));
+ WT_RET(__split_ovfl_key_cleanup(session, parent, ref));
WT_MEMSIZE_ADD(*parent_decrp,
sizeof(WT_IKEY) + ikey->size);
}
@@ -307,12 +306,8 @@ __split_ref_instantiate(WT_SESSION_IMPL *session,
* address has been instantiated, there's no work to do. Otherwise,
* get the address from the on-page cell.
*/
- if ((addr = ref->addr) == NULL)
- return (0);
- if (__wt_off_page(page, addr))
- WT_MEMSIZE_TRANSFER(*parent_decrp, *child_incrp,
- sizeof(WT_ADDR) + addr->size);
- else {
+ addr = ref->addr;
+ if (addr != NULL && !__wt_off_page(parent, addr)) {
__wt_cell_unpack((WT_CELL *)ref->addr, &unpack);
WT_RET(__wt_calloc_one(session, &addr));
if ((ret = __wt_strndup(
@@ -324,8 +319,11 @@ __split_ref_instantiate(WT_SESSION_IMPL *session,
addr->type =
unpack.raw == WT_CELL_ADDR_INT ? WT_ADDR_INT : WT_ADDR_LEAF;
ref->addr = addr;
- WT_MEMSIZE_ADD(*child_incrp, sizeof(WT_ADDR) + addr->size);
}
+
+ /* And finally, the WT_REF itself. */
+ WT_MEMSIZE_TRANSFER(*parent_decrp, *child_incrp, sizeof(WT_REF));
+
return (0);
}
@@ -502,12 +500,9 @@ __split_deepen(WT_SESSION_IMPL *session, WT_PAGE *parent, uint32_t children)
child_incr = 0;
child_pindex = WT_INTL_INDEX_COPY(child);
for (child_refp = child_pindex->index, j = 0; j < slots; ++j) {
- WT_ERR(__split_ref_instantiate(session,
+ WT_ERR(__split_ref_deepen_move(session,
parent, *parent_refp, &parent_decr, &child_incr));
*child_refp++ = *parent_refp++;
-
- WT_MEMSIZE_TRANSFER(
- parent_decr, child_incr, sizeof(WT_REF));
}
__wt_cache_page_inmem_incr(session, child, child_incr);
}
@@ -600,9 +595,10 @@ __split_deepen(WT_SESSION_IMPL *session, WT_PAGE *parent, uint32_t children)
* be using the new index.
*/
size = sizeof(WT_PAGE_INDEX) + pindex->entries * sizeof(WT_REF *);
- WT_MEMSIZE_ADD(parent_decr, size);
WT_ERR(__split_safe_free(session, 0, pindex, size));
+ WT_MEMSIZE_ADD(parent_decr, size);
+#if 0
/*
* Adjust the parent's memory footprint. This may look odd, but we
* have already taken the allocation overhead into account, and an
@@ -611,6 +607,19 @@ __split_deepen(WT_SESSION_IMPL *session, WT_PAGE *parent, uint32_t children)
*/
__wt_cache_page_inmem_incr(session, parent, parent_incr);
__wt_cache_page_inmem_decr(session, parent, parent_decr);
+#else
+ /*
+ * XXX
+ * The code to track page sizes is fundamentally flawed in the face of
+ * splits: for example, we don't add in an overhead allocation constant
+ * when allocating WT_REF structures as pages are created, but the
+ * calculations during split assume that correction. For now, ignore
+ * our carefully calculated values and force the internal page size to
+ * 5% of its current value.
+ */
+ size = parent->memory_footprint - (parent->memory_footprint / 20);
+ __wt_cache_page_inmem_decr(session, parent, size);
+#endif
if (0) {
err: __wt_free_ref_index(session, parent, alloc_index, 1);
@@ -774,13 +783,11 @@ __wt_multi_to_ref(WT_SESSION_IMPL *session,
* the confusion.
*/
WT_RET(__wt_calloc_one(session, &addr));
- WT_MEMSIZE_ADD(incr, sizeof(WT_ADDR));
ref->addr = addr;
addr->size = multi->addr.size;
addr->type = multi->addr.type;
WT_RET(__wt_strndup(session,
multi->addr.addr, addr->size, &addr->addr));
- WT_MEMSIZE_ADD(incr, addr->size);
} else
WT_RET(__split_multi_inmem(session, page, ref, multi));
@@ -973,14 +980,17 @@ __split_parent(WT_SESSION_IMPL *session, WT_REF *ref, WT_REF **ref_new,
WT_TRET(__split_ovfl_key_cleanup(
session, parent, next_ref));
ikey = __wt_ref_key_instantiated(next_ref);
- if (ikey != NULL)
- WT_TRET(__split_safe_free(session, 0,
- ikey,
- sizeof(WT_IKEY) + ikey->size));
+ if (ikey != NULL) {
+ size = sizeof(WT_IKEY) + ikey->size;
+ WT_TRET(__split_safe_free(
+ session, 0, ikey, size));
+ WT_MEMSIZE_ADD(parent_decr, size);
+ }
}
WT_TRET(__split_safe_free(
session, 0, next_ref, sizeof(WT_REF)));
+ WT_MEMSIZE_ADD(parent_decr, sizeof(WT_REF));
}
}
@@ -1096,15 +1106,16 @@ __wt_split_insert(WT_SESSION_IMPL *session, WT_REF *ref, int *splitp)
WT_PAGE *page, *right;
WT_REF *child, *split_ref[2] = { NULL, NULL };
WT_UPDATE *upd;
- size_t page_decr, parent_incr, right_incr, size;
+ size_t page_decr, parent_decr, parent_incr, right_incr;
int i;
*splitp = 0;
btree = S2BT(session);
page = ref->page;
+ ikey = NULL;
right = NULL;
- page_decr = parent_incr = right_incr = 0;
+ page_decr = parent_decr = parent_incr = right_incr = 0;
/*
* Check for pages with append-only workloads. A common application
@@ -1205,9 +1216,19 @@ __wt_split_insert(WT_SESSION_IMPL *session, WT_REF *ref, int *splitp)
WT_ERR(__wt_row_ikey(session, 0,
WT_INSERT_KEY(moved_ins), WT_INSERT_KEY_SIZE(moved_ins),
&child->key.ikey));
+
+ /*
+ * We're swapping WT_REFs in the parent, adjust the accounting, and
+ * row store pages may have instantiated keys.
+ */
WT_MEMSIZE_ADD(parent_incr, sizeof(WT_REF));
- WT_MEMSIZE_ADD(parent_incr, sizeof(WT_IKEY));
- WT_MEMSIZE_ADD(parent_incr, WT_INSERT_KEY_SIZE(moved_ins));
+ WT_MEMSIZE_ADD(
+ parent_incr, sizeof(WT_IKEY) + WT_INSERT_KEY_SIZE(moved_ins));
+ WT_MEMSIZE_ADD(parent_decr, sizeof(WT_REF));
+ if (page->type == WT_PAGE_ROW_LEAF || page->type == WT_PAGE_ROW_INT)
+ if ((ikey = __wt_ref_key_instantiated(ref)) != NULL)
+ WT_MEMSIZE_ADD(
+ parent_decr, sizeof(WT_IKEY) + ikey->size);
/* The new page is dirty by definition. */
WT_ERR(__wt_page_modify_init(session, right));
@@ -1229,14 +1250,11 @@ __wt_split_insert(WT_SESSION_IMPL *session, WT_REF *ref, int *splitp)
*/
for (i = 0; i < WT_SKIP_MAXDEPTH && ins_head->tail[i] == moved_ins; ++i)
;
- size = ((size_t)i - 1) * sizeof(WT_INSERT *);
- size += sizeof(WT_INSERT) + WT_INSERT_KEY_SIZE(moved_ins);
+ WT_MEMSIZE_TRANSFER(page_decr, right_incr, sizeof(WT_INSERT) +
+ (size_t)i * sizeof(WT_INSERT *) + WT_INSERT_KEY_SIZE(moved_ins));
for (upd = moved_ins->upd; upd != NULL; upd = upd->next)
- size += sizeof(WT_UPDATE) + upd->size;
- WT_MEMSIZE_ADD(right_incr, size);
- WT_MEMSIZE_ADD(page_decr, size);
- __wt_cache_page_inmem_decr(session, page, page_decr);
- __wt_cache_page_inmem_incr(session, right, right_incr);
+ WT_MEMSIZE_TRANSFER(
+ page_decr, right_incr, sizeof(WT_UPDATE) + upd->size);
/*
* Allocation operations completed, move the last insert list item from
@@ -1323,10 +1341,23 @@ __wt_split_insert(WT_SESSION_IMPL *session, WT_REF *ref, int *splitp)
#endif
/*
- * Split into the parent.
+ * Save the transaction ID when the split happened. Application
+ * threads will not try to forcibly evict the page again until
+ * all concurrent transactions commit.
+ */
+ page->modify->inmem_split_txn = __wt_txn_new_id(session);
+
+ /* Update the page accounting. */
+ __wt_cache_page_inmem_decr(session, page, page_decr);
+ __wt_cache_page_inmem_incr(session, right, right_incr);
+
+ /*
+ * Split into the parent. After this, the original page is no
+ * longer locked, so we cannot safely look at it.
*/
+ page = NULL;
if ((ret = __split_parent(
- session, ref, split_ref, 2, 0, parent_incr, 0, 0)) != 0) {
+ session, ref, split_ref, 2, parent_decr, parent_incr, 0, 0)) != 0) {
/*
* Move the insert list element back to the original page list.
* For simplicity, the previous skip list pointers originally
@@ -1349,13 +1380,6 @@ __wt_split_insert(WT_SESSION_IMPL *session, WT_REF *ref, int *splitp)
WT_ERR(ret);
}
- /*
- * Save the transaction ID when the split happened. Application
- * threads will not try to forcibly evict the page again until
- * all concurrent transactions commit.
- */
- page->modify->inmem_split_txn = __wt_txn_new_id(session);
-
/* Let our caller know that we split. */
*splitp = 1;
@@ -1367,13 +1391,8 @@ __wt_split_insert(WT_SESSION_IMPL *session, WT_REF *ref, int *splitp)
* structure and instantiated key, there may be threads using them.
* Add them to the session discard list, to be freed once we know it's
* safe.
- *
- * After the split, we're going to discard the WT_REF, account for the
- * change in memory footprint. Row store pages have keys that may be
- * instantiated, check for that.
*/
- if ((page->type == WT_PAGE_ROW_LEAF || page->type == WT_PAGE_ROW_INT) &&
- (ikey = __wt_ref_key_instantiated(ref)) != NULL)
+ if (ikey != NULL)
WT_TRET(__split_safe_free(
session, 0, ikey, sizeof(WT_IKEY) + ikey->size));
WT_TRET(__split_safe_free(session, 0, ref, sizeof(WT_REF)));
@@ -1458,7 +1477,7 @@ __wt_split_multi(WT_SESSION_IMPL *session, WT_REF *ref, int exclusive)
WT_PAGE *page;
WT_PAGE_MODIFY *mod;
WT_REF **ref_new;
- size_t ikey_size, parent_decr, parent_incr;
+ size_t parent_decr, parent_incr;
uint32_t i, new_entries;
page = ref->page;
@@ -1466,7 +1485,7 @@ __wt_split_multi(WT_SESSION_IMPL *session, WT_REF *ref, int exclusive)
new_entries = mod->mod_multi_entries;
ikey = NULL;
- ikey_size = parent_decr = parent_incr = 0;
+ parent_decr = parent_incr = 0;
/*
* Convert the split page's multiblock reconciliation information into
@@ -1482,12 +1501,11 @@ __wt_split_multi(WT_SESSION_IMPL *session, WT_REF *ref, int exclusive)
* change in memory footprint. Row store pages have keys that may be
* instantiated, check for that.
*/
- if ((page->type == WT_PAGE_ROW_LEAF || page->type == WT_PAGE_ROW_INT) &&
- (ikey = __wt_ref_key_instantiated(ref)) != NULL) {
- ikey_size = sizeof(WT_IKEY) + ikey->size;
- WT_MEMSIZE_ADD(parent_decr, ikey_size);
- }
WT_MEMSIZE_ADD(parent_decr, sizeof(WT_REF));
+ if (page->type == WT_PAGE_ROW_LEAF || page->type == WT_PAGE_ROW_INT)
+ if ((ikey = __wt_ref_key_instantiated(ref)) != NULL)
+ WT_MEMSIZE_ADD(
+ parent_decr, sizeof(WT_IKEY) + ikey->size);
/* Split into the parent. */
WT_ERR(__split_parent(session,
@@ -1514,7 +1532,8 @@ __wt_split_multi(WT_SESSION_IMPL *session, WT_REF *ref, int exclusive)
* safe.
*/
if (ikey != NULL)
- WT_TRET(__split_safe_free(session, exclusive, ikey, ikey_size));
+ WT_TRET(__split_safe_free(
+ session, exclusive, ikey, sizeof(WT_IKEY) + ikey->size));
WT_TRET(__split_safe_free(session, exclusive, ref, sizeof(WT_REF)));
/*
diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c
index 9d2e5ba9c86..ec44e8839b0 100644
--- a/src/third_party/wiredtiger/src/config/config_def.c
+++ b/src/third_party/wiredtiger/src/config/config_def.c
@@ -110,6 +110,12 @@ static const WT_CONFIG_CHECK confchk_connection_reconfigure[] = {
{ NULL, NULL, NULL, NULL }
};
+static const WT_CONFIG_CHECK confchk_cursor_reconfigure[] = {
+ { "append", "boolean", NULL, NULL },
+ { "overwrite", "boolean", NULL, NULL },
+ { NULL, NULL, NULL, NULL }
+};
+
static const WT_CONFIG_CHECK confchk_file_meta[] = {
{ "allocation_size", "int", "min=512B,max=128MB", NULL },
{ "app_metadata", "string", NULL, NULL },
@@ -561,6 +567,10 @@ static const WT_CONFIG_ENTRY config_entries[] = {
"",
NULL
},
+ { "cursor.reconfigure",
+ "append=0,overwrite=",
+ confchk_cursor_reconfigure
+ },
{ "file.meta",
"allocation_size=4KB,app_metadata=,block_allocation=best,"
"block_compressor=,cache_resident=0,checkpoint=,checkpoint_lsn=,"
diff --git a/src/third_party/wiredtiger/src/conn/conn_log.c b/src/third_party/wiredtiger/src/conn/conn_log.c
index d15b27f9bc9..f70a9a4a60c 100644
--- a/src/third_party/wiredtiger/src/conn/conn_log.c
+++ b/src/third_party/wiredtiger/src/conn/conn_log.c
@@ -278,6 +278,72 @@ err:
}
/*
+ * __log_close_server --
+ * The log close server thread.
+ */
+static void *
+__log_close_server(void *arg)
+{
+ WT_CONNECTION_IMPL *conn;
+ WT_DECL_RET;
+ WT_FH *close_fh;
+ WT_LOG *log;
+ WT_LSN close_end_lsn, close_lsn;
+ WT_SESSION_IMPL *session;
+ int locked;
+
+ session = arg;
+ conn = S2C(session);
+ log = conn->log;
+ locked = 0;
+ while (F_ISSET(conn, WT_CONN_LOG_SERVER_RUN)) {
+ /*
+ * If there is a log file to close, make sure any outstanding
+ * write operations have completed, then fsync and close it.
+ */
+ if ((close_fh = log->log_close_fh) != NULL &&
+ (ret = __wt_log_extract_lognum(session, close_fh->name,
+ &close_lsn.file)) == 0 &&
+ close_lsn.file < log->write_lsn.file) {
+ /*
+ * We've copied the file handle, clear out the one in
+ * log structure to allow it to be set again.
+ */
+ log->log_close_fh = NULL;
+ /*
+ * Set the close_end_lsn to the LSN immediately after
+ * ours. That is, the beginning of the next log file.
+ * We need to know the LSN file number of our own close
+ * in case earlier calls are still in progress and the
+ * next one to move the sync_lsn into the next file for
+ * later syncs.
+ */
+ close_lsn.offset = 0;
+ close_end_lsn = close_lsn;
+ close_end_lsn.file++;
+ WT_ERR(__wt_fsync(session, close_fh));
+ __wt_spin_lock(session, &log->log_sync_lock);
+ locked = 1;
+ WT_ERR(__wt_close(session, close_fh));
+ log->sync_lsn = close_end_lsn;
+ WT_ERR(__wt_cond_signal(session, log->log_sync_cond));
+ locked = 0;
+ __wt_spin_unlock(session, &log->log_sync_lock);
+ } else
+ /* Wait until the next event. */
+ WT_ERR(__wt_cond_wait(session,
+ conn->log_close_cond, 10000));
+ }
+
+ if (0) {
+err: __wt_err(session, ret, "log close server error");
+ }
+ if (locked)
+ __wt_spin_unlock(session, &log->log_sync_lock);
+ return (NULL);
+}
+
+/*
* __log_server --
* The log server thread.
*/
@@ -294,7 +360,7 @@ __log_server(void *arg)
conn = S2C(session);
log = conn->log;
locked = 0;
- while (F_ISSET(conn, WT_CONN_SERVER_RUN)) {
+ while (F_ISSET(conn, WT_CONN_LOG_SERVER_RUN)) {
/*
* Perform log pre-allocation.
*/
@@ -322,7 +388,7 @@ __log_server(void *arg)
}
if (0) {
-err: __wt_err(session, ret, "log archive server error");
+err: __wt_err(session, ret, "log server error");
}
if (locked)
(void)__wt_writeunlock(session, log->log_archive_lock);
@@ -386,7 +452,7 @@ __wt_logmgr_create(WT_SESSION_IMPL *session, const char *cfg[])
/*
* __wt_logmgr_open --
- * Start the log subsystem and archive server thread.
+ * Start the log service threads.
*/
int
__wt_logmgr_open(WT_SESSION_IMPL *session)
@@ -396,14 +462,33 @@ __wt_logmgr_open(WT_SESSION_IMPL *session)
conn = S2C(session);
/* If no log thread services are configured, we're done. */
- if (!FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED) ||
- !FLD_ISSET(conn->log_flags,
+ if (!FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED))
+ return (0);
+
+ /*
+ * Start the log close thread. It is not configurable.
+ * If logging is enabled, this thread runs.
+ */
+ WT_RET(__wt_open_internal_session(
+ conn, "log-close-server", 0, 0, &conn->log_close_session));
+ WT_RET(__wt_cond_alloc(conn->log_close_session,
+ "log close server", 0, &conn->log_close_cond));
+
+ /*
+ * Start the thread.
+ */
+ WT_RET(__wt_thread_create(conn->log_close_session,
+ &conn->log_close_tid, __log_close_server, conn->log_close_session));
+ conn->log_close_tid_set = 1;
+
+ /* If no log thread services are configured, we're done. */
+ if (!FLD_ISSET(conn->log_flags,
(WT_CONN_LOG_ARCHIVE | WT_CONN_LOG_PREALLOC)))
return (0);
/*
* If a log server thread exists, the user may have reconfigured
- * archiving ore pre-allocation. Signal the thread. Otherwise the
+ * archiving or pre-allocation. Signal the thread. Otherwise the
* user wants archiving and/or allocation and we need to start up
* the thread.
*/
@@ -457,6 +542,17 @@ __wt_logmgr_destroy(WT_SESSION_IMPL *session)
conn->log_tid_set = 0;
}
WT_TRET(__wt_cond_destroy(session, &conn->log_cond));
+ if (conn->log_close_tid_set) {
+ WT_TRET(__wt_cond_signal(session, conn->log_close_cond));
+ WT_TRET(__wt_thread_join(session, conn->log_close_tid));
+ conn->log_close_tid_set = 0;
+ }
+ WT_TRET(__wt_cond_destroy(session, &conn->log_close_cond));
+ if (conn->log_close_session != NULL) {
+ wt_session = &conn->log_close_session->iface;
+ WT_TRET(wt_session->close(wt_session, NULL));
+ conn->log_close_session = NULL;
+ }
WT_TRET(__wt_log_close(session));
diff --git a/src/third_party/wiredtiger/src/conn/conn_open.c b/src/third_party/wiredtiger/src/conn/conn_open.c
index b425376d6ae..ab873cc36a9 100644
--- a/src/third_party/wiredtiger/src/conn/conn_open.c
+++ b/src/third_party/wiredtiger/src/conn/conn_open.c
@@ -25,7 +25,7 @@ __wt_connection_open(WT_CONNECTION_IMPL *conn, const char *cfg[])
* Tell internal server threads to run: this must be set before opening
* any sessions.
*/
- F_SET(conn, WT_CONN_SERVER_RUN);
+ F_SET(conn, WT_CONN_SERVER_RUN | WT_CONN_LOG_SERVER_RUN);
/* WT_SESSION_IMPL array. */
WT_RET(__wt_calloc(session,
@@ -130,6 +130,7 @@ __wt_connection_close(WT_CONNECTION_IMPL *conn)
if (FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED))
WT_TRET(__wt_txn_checkpoint_log(
session, 1, WT_TXN_LOG_CKPT_STOP, NULL));
+ F_CLR(conn, WT_CONN_LOG_SERVER_RUN);
WT_TRET(__wt_logmgr_destroy(session));
/* Free memory for collators, compressors, data sources. */
diff --git a/src/third_party/wiredtiger/src/cursor/cur_backup.c b/src/third_party/wiredtiger/src/cursor/cur_backup.c
index 3c4c5322534..c85e9e24fb2 100644
--- a/src/third_party/wiredtiger/src/cursor/cur_backup.c
+++ b/src/third_party/wiredtiger/src/cursor/cur_backup.c
@@ -109,6 +109,7 @@ __wt_curbackup_open(WT_SESSION_IMPL *session,
__wt_cursor_notsup, /* set-key */
__wt_cursor_notsup, /* set-value */
__wt_cursor_notsup, /* compare */
+ __wt_cursor_notsup, /* equals */
__curbackup_next, /* next */
__wt_cursor_notsup, /* prev */
__curbackup_reset, /* reset */
@@ -117,6 +118,7 @@ __wt_curbackup_open(WT_SESSION_IMPL *session,
__wt_cursor_notsup, /* insert */
__wt_cursor_notsup, /* update */
__wt_cursor_notsup, /* remove */
+ __wt_cursor_notsup, /* reconfigure */
__curbackup_close); /* close */
WT_CURSOR *cursor;
WT_CURSOR_BACKUP *cb;
diff --git a/src/third_party/wiredtiger/src/cursor/cur_config.c b/src/third_party/wiredtiger/src/cursor/cur_config.c
index f011f6db126..348cfbab1dd 100644
--- a/src/third_party/wiredtiger/src/cursor/cur_config.c
+++ b/src/third_party/wiredtiger/src/cursor/cur_config.c
@@ -32,6 +32,7 @@ __wt_curconfig_open(WT_SESSION_IMPL *session,
__wt_cursor_set_key, /* set-key */
__wt_cursor_set_value, /* set-value */
__wt_cursor_notsup, /* compare */
+ __wt_cursor_notsup, /* equals */
__wt_cursor_notsup, /* next */
__wt_cursor_notsup, /* prev */
__wt_cursor_noop, /* reset */
@@ -40,6 +41,7 @@ __wt_curconfig_open(WT_SESSION_IMPL *session,
__wt_cursor_notsup, /* insert */
__wt_cursor_notsup, /* update */
__wt_cursor_notsup, /* remove */
+ __wt_cursor_notsup, /* reconfigure */
__curconfig_close);
WT_CURSOR_CONFIG *cconfig;
WT_CURSOR *cursor;
diff --git a/src/third_party/wiredtiger/src/cursor/cur_ds.c b/src/third_party/wiredtiger/src/cursor/cur_ds.c
index 2cb791de85d..cc3e23570d5 100644
--- a/src/third_party/wiredtiger/src/cursor/cur_ds.c
+++ b/src/third_party/wiredtiger/src/cursor/cur_ds.c
@@ -454,6 +454,7 @@ __wt_curds_open(
__wt_cursor_set_key, /* set-key */
__wt_cursor_set_value, /* set-value */
__curds_compare, /* compare */
+ __wt_cursor_equal, /* equals */
__curds_next, /* next */
__curds_prev, /* prev */
__curds_reset, /* reset */
@@ -462,6 +463,7 @@ __wt_curds_open(
__curds_insert, /* insert */
__curds_update, /* update */
__curds_remove, /* remove */
+ __wt_cursor_notsup, /* reconfigure */
__curds_close); /* close */
WT_CONFIG_ITEM cval, metadata;
WT_CURSOR *cursor, *source;
diff --git a/src/third_party/wiredtiger/src/cursor/cur_dump.c b/src/third_party/wiredtiger/src/cursor/cur_dump.c
index 2be1d91f0f3..00281054d22 100644
--- a/src/third_party/wiredtiger/src/cursor/cur_dump.c
+++ b/src/third_party/wiredtiger/src/cursor/cur_dump.c
@@ -352,6 +352,7 @@ __wt_curdump_create(WT_CURSOR *child, WT_CURSOR *owner, WT_CURSOR **cursorp)
__curdump_set_key, /* set-key */
__curdump_set_value, /* set-value */
__wt_cursor_notsup, /* compare */
+ __wt_cursor_notsup, /* equals */
__curdump_next, /* next */
__curdump_prev, /* prev */
__curdump_reset, /* reset */
@@ -360,6 +361,7 @@ __wt_curdump_create(WT_CURSOR *child, WT_CURSOR *owner, WT_CURSOR **cursorp)
__curdump_insert, /* insert */
__curdump_update, /* update */
__curdump_remove, /* remove */
+ __wt_cursor_notsup, /* reconfigure */
__curdump_close); /* close */
WT_CURSOR *cursor;
WT_CURSOR_DUMP *cdump;
diff --git a/src/third_party/wiredtiger/src/cursor/cur_file.c b/src/third_party/wiredtiger/src/cursor/cur_file.c
index 10d1151c30c..2f24d8ed59a 100644
--- a/src/third_party/wiredtiger/src/cursor/cur_file.c
+++ b/src/third_party/wiredtiger/src/cursor/cur_file.c
@@ -46,10 +46,11 @@ __curfile_compare(WT_CURSOR *a, WT_CURSOR *b, int *cmpp)
CURSOR_API_CALL(a, session, compare, cbt->btree);
/*
- * Confirm both cursors refer to the same source and have keys, then
- * call the underlying object to compare them.
+ * Check both cursors are a "file:" type then call the underlying
+ * function, it can handle cursors pointing to different objects.
*/
- if (strcmp(a->internal_uri, b->internal_uri) != 0)
+ if (!WT_PREFIX_MATCH(a->internal_uri, "file:") ||
+ !WT_PREFIX_MATCH(b->internal_uri, "file:"))
WT_ERR_MSG(session, EINVAL,
"Cursors must reference the same object");
@@ -63,6 +64,38 @@ err: API_END_RET(session, ret);
}
/*
+ * __curfile_equals --
+ * WT_CURSOR->equals method for the btree cursor type.
+ */
+static int
+__curfile_equals(WT_CURSOR *a, WT_CURSOR *b, int *equalp)
+{
+ WT_CURSOR_BTREE *cbt;
+ WT_DECL_RET;
+ WT_SESSION_IMPL *session;
+
+ cbt = (WT_CURSOR_BTREE *)a;
+ CURSOR_API_CALL(a, session, equals, cbt->btree);
+
+ /*
+ * Check both cursors are a "file:" type then call the underlying
+ * function, it can handle cursors pointing to different objects.
+ */
+ if (!WT_PREFIX_MATCH(a->internal_uri, "file:") ||
+ !WT_PREFIX_MATCH(b->internal_uri, "file:"))
+ WT_ERR_MSG(session, EINVAL,
+ "Cursors must reference the same object");
+
+ WT_CURSOR_CHECKKEY(a);
+ WT_CURSOR_CHECKKEY(b);
+
+ ret = __wt_btcur_equals(
+ (WT_CURSOR_BTREE *)a, (WT_CURSOR_BTREE *)b, equalp);
+
+err: API_END_RET(session, ret);
+}
+
+/*
* __curfile_next --
* WT_CURSOR->next method for the btree cursor type.
*/
@@ -356,6 +389,7 @@ __wt_curfile_create(WT_SESSION_IMPL *session,
__wt_cursor_set_key, /* set-key */
__wt_cursor_set_value, /* set-value */
__curfile_compare, /* compare */
+ __curfile_equals, /* equals */
__curfile_next, /* next */
__curfile_prev, /* prev */
__curfile_reset, /* reset */
@@ -364,6 +398,7 @@ __wt_curfile_create(WT_SESSION_IMPL *session,
__curfile_insert, /* insert */
__curfile_update, /* update */
__curfile_remove, /* remove */
+ __wt_cursor_reconfigure, /* reconfigure */
__curfile_close); /* close */
WT_BTREE *btree;
WT_CONFIG_ITEM cval;
diff --git a/src/third_party/wiredtiger/src/cursor/cur_index.c b/src/third_party/wiredtiger/src/cursor/cur_index.c
index 007f3a5ae5b..abc6a106cc9 100644
--- a/src/third_party/wiredtiger/src/cursor/cur_index.c
+++ b/src/third_party/wiredtiger/src/cursor/cur_index.c
@@ -343,6 +343,7 @@ __wt_curindex_open(WT_SESSION_IMPL *session,
__wt_cursor_set_key, /* set-key */
__curindex_set_value, /* set-value */
__wt_cursor_notsup, /* compare */
+ __wt_cursor_notsup, /* equals */
__curindex_next, /* next */
__curindex_prev, /* prev */
__curindex_reset, /* reset */
@@ -351,6 +352,7 @@ __wt_curindex_open(WT_SESSION_IMPL *session,
__wt_cursor_notsup, /* insert */
__wt_cursor_notsup, /* update */
__wt_cursor_notsup, /* remove */
+ __wt_cursor_notsup, /* reconfigure */
__curindex_close); /* close */
WT_CURSOR_INDEX *cindex;
WT_CURSOR *cursor;
diff --git a/src/third_party/wiredtiger/src/cursor/cur_log.c b/src/third_party/wiredtiger/src/cursor/cur_log.c
index e3089e9fb83..2e2a2530df6 100644
--- a/src/third_party/wiredtiger/src/cursor/cur_log.c
+++ b/src/third_party/wiredtiger/src/cursor/cur_log.c
@@ -344,6 +344,7 @@ __wt_curlog_open(WT_SESSION_IMPL *session,
__wt_cursor_set_key, /* set-key */
__wt_cursor_set_value, /* set-value */
__curlog_compare, /* compare */
+ __wt_cursor_equal, /* equals */
__curlog_next, /* next */
__wt_cursor_notsup, /* prev */
__curlog_reset, /* reset */
@@ -352,6 +353,7 @@ __wt_curlog_open(WT_SESSION_IMPL *session,
__wt_cursor_notsup, /* insert */
__wt_cursor_notsup, /* update */
__wt_cursor_notsup, /* remove */
+ __wt_cursor_notsup, /* reconfigure */
__curlog_close); /* close */
WT_CURSOR *cursor;
WT_CURSOR_LOG *cl;
diff --git a/src/third_party/wiredtiger/src/cursor/cur_metadata.c b/src/third_party/wiredtiger/src/cursor/cur_metadata.c
index 618c678558a..31c96e3087a 100644
--- a/src/third_party/wiredtiger/src/cursor/cur_metadata.c
+++ b/src/third_party/wiredtiger/src/cursor/cur_metadata.c
@@ -409,6 +409,7 @@ __wt_curmetadata_open(WT_SESSION_IMPL *session,
__wt_cursor_set_key, /* set-key */
__wt_cursor_set_value, /* set-value */
__curmetadata_compare, /* compare */
+ __wt_cursor_equal, /* equals */
__curmetadata_next, /* next */
__curmetadata_prev, /* prev */
__curmetadata_reset, /* reset */
@@ -417,10 +418,12 @@ __wt_curmetadata_open(WT_SESSION_IMPL *session,
__curmetadata_insert, /* insert */
__curmetadata_update, /* update */
__curmetadata_remove, /* remove */
+ __wt_cursor_notsup, /* reconfigure */
__curmetadata_close); /* close */
WT_CURSOR *cursor;
WT_CURSOR_METADATA *mdc;
WT_DECL_RET;
+ WT_CONFIG_ITEM cval;
WT_RET(__wt_calloc_one(session, &mdc));
@@ -435,8 +438,16 @@ __wt_curmetadata_open(WT_SESSION_IMPL *session,
WT_ERR(__wt_cursor_init(cursor, uri, owner, cfg, cursorp));
- /* Metadata cursors default to read only. */
- WT_ERR(__wt_cursor_config_readonly(cursor, cfg, 1));
+ /*
+ * Metadata cursors default to readonly; if not set to not-readonly,
+ * they are permanently readonly and cannot be reconfigured.
+ */
+ WT_ERR(__wt_config_gets_def(session, cfg, "readonly", 1, &cval));
+ if (cval.val != 0) {
+ cursor->insert = __wt_cursor_notsup;
+ cursor->update = __wt_cursor_notsup;
+ cursor->remove = __wt_cursor_notsup;
+ }
if (0) {
err: if (mdc->file_cursor != NULL)
diff --git a/src/third_party/wiredtiger/src/cursor/cur_stat.c b/src/third_party/wiredtiger/src/cursor/cur_stat.c
index f7effb87a4f..e3364ad8009 100644
--- a/src/third_party/wiredtiger/src/cursor/cur_stat.c
+++ b/src/third_party/wiredtiger/src/cursor/cur_stat.c
@@ -408,11 +408,15 @@ __curstat_file_init(WT_SESSION_IMPL *session,
* We're likely holding the handle lock inside the statistics
* logging thread, not to mention calling __wt_conn_btree_apply
* from there as well. Save/restore the handle.
+ * Take the schema lock now, in case btree apply needs to
+ * get it later - that would violate lock ordering
+ * conventions and can lead to deadlocks.
*/
saved_dhandle = dhandle;
- WT_WITH_DHANDLE_LOCK(session,
- ret = __wt_conn_btree_apply(
- session, 1, dhandle->name, __curstat_checkpoint, cfg_arg));
+ WT_WITH_SCHEMA_LOCK(session,
+ WT_WITH_DHANDLE_LOCK(session,
+ ret = __wt_conn_btree_apply(session,
+ 1, dhandle->name, __curstat_checkpoint, cfg_arg)));
session->dhandle = saved_dhandle;
}
@@ -485,6 +489,7 @@ __wt_curstat_open(WT_SESSION_IMPL *session,
__curstat_set_key, /* set-key */
__curstat_set_value, /* set-value */
__wt_cursor_notsup, /* compare */
+ __wt_cursor_notsup, /* equals */
__curstat_next, /* next */
__curstat_prev, /* prev */
__curstat_reset, /* reset */
@@ -493,6 +498,7 @@ __wt_curstat_open(WT_SESSION_IMPL *session,
__wt_cursor_notsup, /* insert */
__wt_cursor_notsup, /* update */
__wt_cursor_notsup, /* remove */
+ __wt_cursor_notsup, /* reconfigure */
__curstat_close); /* close */
WT_CONFIG_ITEM cval, sval;
WT_CURSOR *cursor;
diff --git a/src/third_party/wiredtiger/src/cursor/cur_std.c b/src/third_party/wiredtiger/src/cursor/cur_std.c
index 1470cb37100..df38eb9e57d 100644
--- a/src/third_party/wiredtiger/src/cursor/cur_std.c
+++ b/src/third_party/wiredtiger/src/cursor/cur_std.c
@@ -59,28 +59,6 @@ __wt_cursor_set_notsup(WT_CURSOR *cursor)
}
/*
- * __wt_cursor_config_readonly --
- * Parse read only configuration and setup cursor appropriately.
- */
-int
-__wt_cursor_config_readonly(WT_CURSOR *cursor, const char *cfg[], int def)
-{
- WT_CONFIG_ITEM cval;
- WT_SESSION_IMPL *session;
-
- session = (WT_SESSION_IMPL *)cursor->session;
-
- WT_RET(__wt_config_gets_def(session, cfg, "readonly", def, &cval));
- if (cval.val != 0) {
- /* Reset all cursor methods that could modify data. */
- cursor->insert = __wt_cursor_notsup;
- cursor->update = __wt_cursor_notsup;
- cursor->remove = __wt_cursor_notsup;
- }
- return (0);
-}
-
-/*
* __wt_cursor_kv_not_set --
* Standard error message for key/values not set.
*/
@@ -502,29 +480,68 @@ __wt_cursor_close(WT_CURSOR *cursor)
}
/*
- * __cursor_runtime_config --
+ * __wt_cursor_equal --
+ * WT_CURSOR->equals default implementation.
+ */
+int
+__wt_cursor_equal(WT_CURSOR *cursor, WT_CURSOR *other, int *equalp)
+{
+ WT_DECL_RET;
+ WT_SESSION_IMPL *session;
+ int cmp;
+
+ session = (WT_SESSION_IMPL *)cursor->session;
+ CURSOR_API_CALL(cursor, session, equals, NULL);
+
+ WT_ERR(cursor->compare(cursor, other, &cmp));
+ *equalp = (cmp == 0) ? 1 : 0;
+
+err: API_END(session, ret);
+ return (ret);
+}
+
+/*
+ * __wt_cursor_reconfigure --
* Set runtime-configurable settings.
*/
-static int
-__cursor_runtime_config(WT_CURSOR *cursor, const char *cfg[])
+int
+__wt_cursor_reconfigure(WT_CURSOR *cursor, const char *config)
{
WT_CONFIG_ITEM cval;
+ WT_DECL_RET;
WT_SESSION_IMPL *session;
session = (WT_SESSION_IMPL *)cursor->session;
- /*
- * !!!
- * There's no way yet to reconfigure cursor flags at runtime; if, in
- * the future there is a way to do that, similar support needs to be
- * added for data-source cursors, or, this call needs to return an
- * error in the case of a data-source cursor.
+ /* Reconfiguration resets the cursor. */
+ WT_RET(cursor->reset(cursor));
+
+ /*
+ * append
+ * Only relevant to column stores.
*/
- WT_RET(__wt_config_gets_def(session, cfg, "overwrite", 1, &cval));
- if (cval.val)
- F_SET(cursor, WT_CURSTD_OVERWRITE);
- else
- F_CLR(cursor, WT_CURSTD_OVERWRITE);
+ if (WT_CURSOR_RECNO(cursor)) {
+ if ((ret = __wt_config_getones(
+ session, config, "append", &cval)) == 0) {
+ if (cval.val)
+ F_SET(cursor, WT_CURSTD_APPEND);
+ else
+ F_CLR(cursor, WT_CURSTD_APPEND);
+ } else
+ WT_RET_NOTFOUND_OK(ret);
+ }
+
+ /*
+ * overwrite
+ */
+ if ((ret = __wt_config_getones(
+ session, config, "overwrite", &cval)) == 0) {
+ if (cval.val)
+ F_SET(cursor, WT_CURSTD_OVERWRITE);
+ else
+ F_CLR(cursor, WT_CURSTD_OVERWRITE);
+ } else
+ WT_RET_NOTFOUND_OK(ret);
return (0);
}
@@ -587,9 +604,6 @@ __wt_cursor_init(WT_CURSOR *cursor,
if (cursor->internal_uri == NULL)
WT_RET(__wt_strdup(session, uri, &cursor->internal_uri));
- /* Set runtime-configurable settings. */
- WT_RET(__cursor_runtime_config(cursor, cfg));
-
/*
* append
* The append flag is only relevant to column stores.
@@ -601,14 +615,23 @@ __wt_cursor_init(WT_CURSOR *cursor,
}
/*
- * checkpoint
- * Checkpoint cursors are read-only.
+ * checkpoint, readonly
+ * Checkpoint cursors are permanently read-only, avoid the extra work
+ * of two configuration string checks.
*/
WT_RET(__wt_config_gets_def(session, cfg, "checkpoint", 0, &cval));
if (cval.len != 0) {
cursor->insert = __wt_cursor_notsup;
cursor->update = __wt_cursor_notsup;
cursor->remove = __wt_cursor_notsup;
+ } else {
+ WT_RET(
+ __wt_config_gets_def(session, cfg, "readonly", 0, &cval));
+ if (cval.val != 0) {
+ cursor->insert = __wt_cursor_notsup;
+ cursor->update = __wt_cursor_notsup;
+ cursor->remove = __wt_cursor_notsup;
+ }
}
/*
@@ -635,14 +658,18 @@ __wt_cursor_init(WT_CURSOR *cursor,
} else
cdump = NULL;
+ /* overwrite */
+ WT_RET(__wt_config_gets_def(session, cfg, "overwrite", 1, &cval));
+ if (cval.val)
+ F_SET(cursor, WT_CURSTD_OVERWRITE);
+ else
+ F_CLR(cursor, WT_CURSTD_OVERWRITE);
+
/* raw */
WT_RET(__wt_config_gets_def(session, cfg, "raw", 0, &cval));
if (cval.val != 0)
F_SET(cursor, WT_CURSTD_RAW);
- /* readonly */
- WT_RET(__wt_cursor_config_readonly(cursor, cfg, 0));
-
/*
* Cursors that are internal to some other cursor (such as file cursors
* inside a table cursor) should be closed after the containing cursor.
diff --git a/src/third_party/wiredtiger/src/cursor/cur_table.c b/src/third_party/wiredtiger/src/cursor/cur_table.c
index 4eb362cfcd4..f8e8625b0bd 100644
--- a/src/third_party/wiredtiger/src/cursor/cur_table.c
+++ b/src/third_party/wiredtiger/src/cursor/cur_table.c
@@ -77,20 +77,22 @@ __curextract_insert(WT_CURSOR *cursor) {
static int
__apply_idx(WT_CURSOR_TABLE *ctable, size_t func_off, int skip_immutable) {
WT_CURSOR_STATIC_INIT(iface,
- __wt_cursor_get_key, /* get-key */
- __wt_cursor_get_value, /* get-value */
- __wt_cursor_set_key, /* set-key */
- __wt_cursor_set_value, /* set-value */
- __wt_cursor_notsup, /* compare */
- __wt_cursor_notsup, /* next */
- __wt_cursor_notsup, /* prev */
- __wt_cursor_notsup, /* reset */
- __wt_cursor_notsup, /* search */
- __wt_cursor_notsup, /* search-near */
- __curextract_insert, /* insert */
- __wt_cursor_notsup, /* update */
- __wt_cursor_notsup, /* remove */
- __wt_cursor_notsup); /* close */
+ __wt_cursor_get_key, /* get-key */
+ __wt_cursor_get_value, /* get-value */
+ __wt_cursor_set_key, /* set-key */
+ __wt_cursor_set_value, /* set-value */
+ __wt_cursor_notsup, /* compare */
+ __wt_cursor_notsup, /* equals */
+ __wt_cursor_notsup, /* next */
+ __wt_cursor_notsup, /* prev */
+ __wt_cursor_notsup, /* reset */
+ __wt_cursor_notsup, /* search */
+ __wt_cursor_notsup, /* search-near */
+ __curextract_insert, /* insert */
+ __wt_cursor_notsup, /* update */
+ __wt_cursor_notsup, /* reconfigure */
+ __wt_cursor_notsup, /* remove */
+ __wt_cursor_notsup); /* close */
WT_CURSOR **cp;
WT_CURSOR_EXTRACTOR extract_cursor;
WT_DECL_RET;
@@ -755,11 +757,11 @@ __curtable_open_colgroups(WT_CURSOR_TABLE *ctable, const char *cfg_arg[])
WT_TABLE *table;
WT_CURSOR **cp;
/*
- * Underlying column groups are always opened without dump, and only
- * the primary is opened with next_random.
+ * Underlying column groups are always opened without dump or readonly,
+ * and only the primary is opened with next_random.
*/
const char *cfg[] = {
- cfg_arg[0], cfg_arg[1], "dump=\"\"", NULL, NULL
+ cfg_arg[0], cfg_arg[1], "dump=\"\",readonly=0", NULL, NULL
};
u_int i;
int complete;
@@ -832,20 +834,22 @@ __wt_curtable_open(WT_SESSION_IMPL *session,
const char *uri, const char *cfg[], WT_CURSOR **cursorp)
{
WT_CURSOR_STATIC_INIT(iface,
- __wt_curtable_get_key, /* get-key */
- __wt_curtable_get_value, /* get-value */
- __wt_curtable_set_key, /* set-key */
- __wt_curtable_set_value, /* set-value */
- __curtable_compare, /* compare */
- __curtable_next, /* next */
- __curtable_prev, /* prev */
- __curtable_reset, /* reset */
- __curtable_search, /* search */
- __curtable_search_near, /* search-near */
- __curtable_insert, /* insert */
- __curtable_update, /* update */
- __curtable_remove, /* remove */
- __curtable_close); /* close */
+ __wt_curtable_get_key, /* get-key */
+ __wt_curtable_get_value, /* get-value */
+ __wt_curtable_set_key, /* set-key */
+ __wt_curtable_set_value, /* set-value */
+ __curtable_compare, /* compare */
+ __wt_cursor_equal, /* equals */
+ __curtable_next, /* next */
+ __curtable_prev, /* prev */
+ __curtable_reset, /* reset */
+ __curtable_search, /* search */
+ __curtable_search_near, /* search-near */
+ __curtable_insert, /* insert */
+ __curtable_update, /* update */
+ __curtable_remove, /* remove */
+ __wt_cursor_reconfigure, /* reconfigure */
+ __curtable_close); /* close */
WT_CONFIG_ITEM cval;
WT_CURSOR *cursor;
WT_CURSOR_TABLE *ctable;
@@ -892,8 +896,8 @@ __wt_curtable_open(WT_SESSION_IMPL *session,
ctable->plan = table->plan;
/* Handle projections. */
+ WT_ERR(__wt_scr_alloc(session, 0, &tmp));
if (columns != NULL) {
- WT_ERR(__wt_scr_alloc(session, 0, &tmp));
WT_ERR(__wt_struct_reformat(session, table,
columns, strlen(columns), NULL, 1, tmp));
WT_ERR(__wt_strndup(
@@ -935,22 +939,28 @@ __wt_curtable_open(WT_SESSION_IMPL *session,
WT_ERR(__curtable_open_colgroups(ctable, cfg));
/*
- * We'll need to squirrel away a copy of the cursor configuration
- * for if/when we open indices.
+ * We'll need to squirrel away a copy of the cursor configuration for
+ * if/when we open indices.
*
* cfg[0] is the baseline configuration for the cursor open and we can
* acquire another copy from the configuration structures, so it would
* be reasonable not to copy it here: but I'd rather be safe than sorry.
*
- * Underlying indices are always opened without dump.
+ * cfg[1] is the application configuration.
+ *
+ * Underlying indices are always opened without dump or readonly; that
+ * information is appended to cfg[1] so later "fast" configuration calls
+ * (checking only cfg[0] and cfg[1]) work. I don't expect to see more
+ * than two configuration strings here, but it's written to compact into
+ * two configuration strings, a copy of cfg[0] and the rest in cfg[1].
*/
- for (cfg_cnt = 0; cfg[cfg_cnt] != NULL; ++cfg_cnt)
- ;
- WT_ERR(__wt_calloc_def(session, cfg_cnt + 2, &ctable->cfg));
- for (cfg_cnt = 0; cfg[cfg_cnt] != NULL; ++cfg_cnt)
- WT_ERR(
- __wt_strdup(session, cfg[cfg_cnt], &ctable->cfg[cfg_cnt]));
- WT_ERR(__wt_strdup(session, "dump=\"\"", &ctable->cfg[cfg_cnt]));
+ WT_ERR(__wt_calloc_def(session, 3, &ctable->cfg));
+ WT_ERR(__wt_strdup(session, cfg[0], &ctable->cfg[0]));
+ WT_ERR(__wt_buf_set(session, tmp, "", 0));
+ for (cfg_cnt = 1; cfg[cfg_cnt] != NULL; ++cfg_cnt)
+ WT_ERR(__wt_buf_catfmt(session, tmp, "%s,", cfg[cfg_cnt]));
+ WT_ERR(__wt_buf_catfmt(session, tmp, "dump=\"\",readonly=0"));
+ WT_ERR(__wt_strdup(session, tmp->data, &ctable->cfg[1]));
if (0) {
err: WT_TRET(__curtable_close(cursor));
diff --git a/src/third_party/wiredtiger/src/evict/evict_page.c b/src/third_party/wiredtiger/src/evict/evict_page.c
index b3a6f718ca2..99e2a6751be 100644
--- a/src/third_party/wiredtiger/src/evict/evict_page.c
+++ b/src/third_party/wiredtiger/src/evict/evict_page.c
@@ -355,6 +355,14 @@ __evict_review(WT_SESSION_IMPL *session, WT_REF *ref,
* pages after we've written them.
*/
if (WT_PAGE_IS_INTERNAL(page)) {
+ /*
+ * Quit if we're trying to push out a "tree", an internal page
+ * with live internal pages as children, it's not likely to
+ * succeed.
+ */
+ if (!top && !exclusive)
+ return (EBUSY);
+
WT_WITH_PAGE_INDEX(session, ret = __evict_review_subtree(
session, ref, exclusive, inmem_splitp, istreep));
WT_RET(ret);
diff --git a/src/third_party/wiredtiger/src/include/config.h b/src/third_party/wiredtiger/src/include/config.h
index 277a22063d1..65757c2ef6d 100644
--- a/src/third_party/wiredtiger/src/include/config.h
+++ b/src/third_party/wiredtiger/src/include/config.h
@@ -57,29 +57,30 @@ struct __wt_config_parser_impl {
#define WT_CONFIG_ENTRY_connection_open_session 8
#define WT_CONFIG_ENTRY_connection_reconfigure 9
#define WT_CONFIG_ENTRY_cursor_close 10
-#define WT_CONFIG_ENTRY_file_meta 11
-#define WT_CONFIG_ENTRY_index_meta 12
-#define WT_CONFIG_ENTRY_session_begin_transaction 13
-#define WT_CONFIG_ENTRY_session_checkpoint 14
-#define WT_CONFIG_ENTRY_session_close 15
-#define WT_CONFIG_ENTRY_session_commit_transaction 16
-#define WT_CONFIG_ENTRY_session_compact 17
-#define WT_CONFIG_ENTRY_session_create 18
-#define WT_CONFIG_ENTRY_session_drop 19
-#define WT_CONFIG_ENTRY_session_log_printf 20
-#define WT_CONFIG_ENTRY_session_open_cursor 21
-#define WT_CONFIG_ENTRY_session_reconfigure 22
-#define WT_CONFIG_ENTRY_session_rename 23
-#define WT_CONFIG_ENTRY_session_rollback_transaction 24
-#define WT_CONFIG_ENTRY_session_salvage 25
-#define WT_CONFIG_ENTRY_session_truncate 26
-#define WT_CONFIG_ENTRY_session_upgrade 27
-#define WT_CONFIG_ENTRY_session_verify 28
-#define WT_CONFIG_ENTRY_table_meta 29
-#define WT_CONFIG_ENTRY_wiredtiger_open 30
-#define WT_CONFIG_ENTRY_wiredtiger_open_all 31
-#define WT_CONFIG_ENTRY_wiredtiger_open_basecfg 32
-#define WT_CONFIG_ENTRY_wiredtiger_open_usercfg 33
+#define WT_CONFIG_ENTRY_cursor_reconfigure 11
+#define WT_CONFIG_ENTRY_file_meta 12
+#define WT_CONFIG_ENTRY_index_meta 13
+#define WT_CONFIG_ENTRY_session_begin_transaction 14
+#define WT_CONFIG_ENTRY_session_checkpoint 15
+#define WT_CONFIG_ENTRY_session_close 16
+#define WT_CONFIG_ENTRY_session_commit_transaction 17
+#define WT_CONFIG_ENTRY_session_compact 18
+#define WT_CONFIG_ENTRY_session_create 19
+#define WT_CONFIG_ENTRY_session_drop 20
+#define WT_CONFIG_ENTRY_session_log_printf 21
+#define WT_CONFIG_ENTRY_session_open_cursor 22
+#define WT_CONFIG_ENTRY_session_reconfigure 23
+#define WT_CONFIG_ENTRY_session_rename 24
+#define WT_CONFIG_ENTRY_session_rollback_transaction 25
+#define WT_CONFIG_ENTRY_session_salvage 26
+#define WT_CONFIG_ENTRY_session_truncate 27
+#define WT_CONFIG_ENTRY_session_upgrade 28
+#define WT_CONFIG_ENTRY_session_verify 29
+#define WT_CONFIG_ENTRY_table_meta 30
+#define WT_CONFIG_ENTRY_wiredtiger_open 31
+#define WT_CONFIG_ENTRY_wiredtiger_open_all 32
+#define WT_CONFIG_ENTRY_wiredtiger_open_basecfg 33
+#define WT_CONFIG_ENTRY_wiredtiger_open_usercfg 34
/*
* configuration section: END
* DO NOT EDIT: automatically built by dist/flags.py.
diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h
index c8a3ae6e291..c5723882489 100644
--- a/src/third_party/wiredtiger/src/include/connection.h
+++ b/src/third_party/wiredtiger/src/include/connection.h
@@ -279,10 +279,14 @@ struct __wt_connection_impl {
#define WT_CONN_LOG_EXISTED 0x04 /* Log files found */
#define WT_CONN_LOG_PREALLOC 0x08 /* Pre-allocation is enabled */
uint32_t log_flags; /* Global logging configuration */
- WT_CONDVAR *log_cond; /* Log archive wait mutex */
- WT_SESSION_IMPL *log_session; /* Log archive session */
- wt_thread_t log_tid; /* Log archive thread */
- int log_tid_set; /* Log archive thread set */
+ WT_CONDVAR *log_cond; /* Log server wait mutex */
+ WT_SESSION_IMPL *log_session; /* Log server session */
+ wt_thread_t log_tid; /* Log server thread */
+ int log_tid_set; /* Log server thread set */
+ WT_CONDVAR *log_close_cond;/* Log close thread wait mutex */
+ WT_SESSION_IMPL *log_close_session;/* Log close thread session */
+ wt_thread_t log_close_tid; /* Log close thread thread */
+ int log_close_tid_set;/* Log close thread set */
WT_LOG *log; /* Logging structure */
WT_COMPRESSOR *log_compressor;/* Logging compressor */
wt_off_t log_file_max; /* Log file max size */
diff --git a/src/third_party/wiredtiger/src/include/cursor.h b/src/third_party/wiredtiger/src/include/cursor.h
index e46c1f7de1b..ba53bd23c78 100644
--- a/src/third_party/wiredtiger/src/include/cursor.h
+++ b/src/third_party/wiredtiger/src/include/cursor.h
@@ -15,6 +15,7 @@
set_key, \
set_value, \
compare, \
+ equals, \
next, \
prev, \
reset, \
@@ -23,6 +24,7 @@
insert, \
update, \
remove, \
+ reconfigure, \
close) \
static const WT_CURSOR n = { \
NULL, /* session */ \
@@ -34,6 +36,7 @@
(void (*)(WT_CURSOR *, ...))(set_key), \
(void (*)(WT_CURSOR *, ...))(set_value), \
(int (*)(WT_CURSOR *, WT_CURSOR *, int *))(compare), \
+ (int (*)(WT_CURSOR *, WT_CURSOR *, int *))(equals), \
next, \
prev, \
reset, \
@@ -43,6 +46,7 @@
update, \
remove, \
close, \
+ (int (*)(WT_CURSOR *, const char *))(reconfigure), \
{ NULL, NULL }, /* TAILQ_ENTRY q */ \
0, /* recno key */ \
{ 0 }, /* recno raw buffer */ \
diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h
index d8ed3f5cef1..35b8dfc113e 100644
--- a/src/third_party/wiredtiger/src/include/extern.h
+++ b/src/third_party/wiredtiger/src/include/extern.h
@@ -96,6 +96,7 @@ extern int __wt_btcur_remove(WT_CURSOR_BTREE *cbt);
extern int __wt_btcur_update(WT_CURSOR_BTREE *cbt);
extern int __wt_btcur_next_random(WT_CURSOR_BTREE *cbt);
extern int __wt_btcur_compare(WT_CURSOR_BTREE *a_arg, WT_CURSOR_BTREE *b_arg, int *cmpp);
+extern int __wt_btcur_equals( WT_CURSOR_BTREE *a_arg, WT_CURSOR_BTREE *b_arg, int *equalp);
extern int __wt_btcur_range_truncate(WT_CURSOR_BTREE *start, WT_CURSOR_BTREE *stop);
extern int __wt_btcur_close(WT_CURSOR_BTREE *cbt);
extern int __wt_debug_set_verbose(WT_SESSION_IMPL *session, const char *v);
@@ -265,7 +266,6 @@ extern int __wt_curstat_open(WT_SESSION_IMPL *session, const char *uri, const ch
extern int __wt_cursor_notsup(WT_CURSOR *cursor);
extern int __wt_cursor_noop(WT_CURSOR *cursor);
extern void __wt_cursor_set_notsup(WT_CURSOR *cursor);
-extern int __wt_cursor_config_readonly(WT_CURSOR *cursor, const char *cfg[], int def);
extern int __wt_cursor_kv_not_set(WT_CURSOR *cursor, int key);
extern int __wt_cursor_get_key(WT_CURSOR *cursor, ...);
extern void __wt_cursor_set_key(WT_CURSOR *cursor, ...);
@@ -280,6 +280,8 @@ extern int __wt_cursor_get_valuev(WT_CURSOR *cursor, va_list ap);
extern void __wt_cursor_set_value(WT_CURSOR *cursor, ...);
extern void __wt_cursor_set_valuev(WT_CURSOR *cursor, va_list ap);
extern int __wt_cursor_close(WT_CURSOR *cursor);
+extern int __wt_cursor_equal(WT_CURSOR *cursor, WT_CURSOR *other, int *equalp);
+extern int __wt_cursor_reconfigure(WT_CURSOR *cursor, const char *config);
extern int __wt_cursor_dup_position(WT_CURSOR *to_dup, WT_CURSOR *cursor);
extern int __wt_cursor_init(WT_CURSOR *cursor, const char *uri, WT_CURSOR *owner, const char *cfg[], WT_CURSOR **cursorp);
extern int __wt_curtable_get_key(WT_CURSOR *cursor, ...);
diff --git a/src/third_party/wiredtiger/src/include/flags.h b/src/third_party/wiredtiger/src/include/flags.h
index c7e74885a35..9664fce3f9f 100644
--- a/src/third_party/wiredtiger/src/include/flags.h
+++ b/src/third_party/wiredtiger/src/include/flags.h
@@ -6,15 +6,16 @@
#define WT_CONN_CKPT_SYNC 0x00000002
#define WT_CONN_EVICTION_RUN 0x00000004
#define WT_CONN_LEAK_MEMORY 0x00000008
-#define WT_CONN_LSM_MERGE 0x00000010
-#define WT_CONN_PANIC 0x00000020
-#define WT_CONN_SERVER_ASYNC 0x00000040
-#define WT_CONN_SERVER_CHECKPOINT 0x00000080
-#define WT_CONN_SERVER_LSM 0x00000100
-#define WT_CONN_SERVER_RUN 0x00000200
-#define WT_CONN_SERVER_STATISTICS 0x00000400
-#define WT_CONN_SERVER_SWEEP 0x00000800
-#define WT_CONN_WAS_BACKUP 0x00001000
+#define WT_CONN_LOG_SERVER_RUN 0x00000010
+#define WT_CONN_LSM_MERGE 0x00000020
+#define WT_CONN_PANIC 0x00000040
+#define WT_CONN_SERVER_ASYNC 0x00000080
+#define WT_CONN_SERVER_CHECKPOINT 0x00000100
+#define WT_CONN_SERVER_LSM 0x00000200
+#define WT_CONN_SERVER_RUN 0x00000400
+#define WT_CONN_SERVER_STATISTICS 0x00000800
+#define WT_CONN_SERVER_SWEEP 0x00001000
+#define WT_CONN_WAS_BACKUP 0x00002000
#define WT_EVICTING 0x00000001
#define WT_FILE_TYPE_CHECKPOINT 0x00000001
#define WT_FILE_TYPE_DATA 0x00000002
diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in
index 91eb41af4f3..982e850241b 100644
--- a/src/third_party/wiredtiger/src/include/wiredtiger.in
+++ b/src/third_party/wiredtiger/src/include/wiredtiger.in
@@ -270,7 +270,8 @@ struct __wt_cursor {
*/
/*!
* Return the ordering relationship between two cursors: both cursors
- * must have the same data source and have valid keys.
+ * must have the same data source and have valid keys. (When testing
+ * only for equality, WT_CURSOR::equals may be faster.)
*
* @snippet ex_all.c Cursor comparison
*
@@ -286,6 +287,21 @@ struct __wt_cursor {
int __F(compare)(WT_CURSOR *cursor, WT_CURSOR *other, int *comparep);
/*!
+ * Return the ordering relationship between two cursors, testing only
+ * for equality: both cursors must have the same data source and have
+ * valid keys.
+ *
+ * @snippet ex_all.c Cursor equality
+ *
+ * @param cursor the cursor handle
+ * @param other another cursor handle
+ * @param[out] equalp the status of the comparison: 1 if the cursors
+ * refer to the same key, otherwise 0.
+ * @errors
+ */
+ int __F(equals)(WT_CURSOR *cursor, WT_CURSOR *other, int *equalp);
+
+ /*!
* Return the next record.
*
* @snippet ex_all.c Return the next record
@@ -495,6 +511,29 @@ struct __wt_cursor {
*/
int __F(close)(WT_HANDLE_CLOSED(WT_CURSOR) *cursor);
+ /*!
+ * Reconfigure the cursor.
+ *
+ * The cursor is reset.
+ *
+ * @snippet ex_all.c Reconfigure a cursor
+ *
+ * @param cursor the cursor handle
+ * @configstart{cursor.reconfigure, see dist/api_data.py}
+ * @config{append, append the value as a new record\, creating a new
+ * record number key; valid only for cursors with record number keys., a
+ * boolean flag; default \c false.}
+ * @config{overwrite, configures whether the cursor's insert\, update
+ * and remove methods check the existing state of the record. If \c
+ * overwrite is \c false\, WT_CURSOR::insert fails with
+ * ::WT_DUPLICATE_KEY if the record exists\, WT_CURSOR::update and
+ * WT_CURSOR::remove fail with ::WT_NOTFOUND if the record does not
+ * exist., a boolean flag; default \c true.}
+ * @configend
+ * @errors
+ */
+ int __F(reconfigure)(WT_CURSOR *cursor, const char *config);
+
/*
* Protected fields, only to be used by cursor implementations.
*/
@@ -522,6 +561,7 @@ struct __wt_cursor {
* user on open.
*/
const char *internal_uri;
+ /* Saved modification methods. */
#define WT_CURSTD_APPEND 0x0001
#define WT_CURSTD_BULK 0x0002
@@ -813,10 +853,6 @@ struct __wt_session {
* duplicated cursors are positioned at the same place in the data
* source as the original.
*
- * To reconfigure a cursor, duplicate it with a new configuration value:
- *
- * @snippet ex_all.c Reconfigure a cursor
- *
* Cursor handles should be discarded by calling WT_CURSOR::close.
*
* Cursors capable of supporting transactional operations operate in the
diff --git a/src/third_party/wiredtiger/src/log/log.c b/src/third_party/wiredtiger/src/log/log.c
index 181e77128a9..e75946e9885 100644
--- a/src/third_party/wiredtiger/src/log/log.c
+++ b/src/third_party/wiredtiger/src/log/log.c
@@ -240,6 +240,7 @@ __log_acquire(WT_SESSION_IMPL *session, uint64_t recsize, WT_LOGSLOT *slot)
if (log->log_close_fh != NULL)
F_SET(slot, SLOT_CLOSEFH);
}
+
/*
* Checkpoints can be configured based on amount of log written.
* Add in this log record to the sum and if needed, signal the
@@ -857,9 +858,8 @@ __log_release(WT_SESSION_IMPL *session, WT_LOGSLOT *slot)
{
WT_CONNECTION_IMPL *conn;
WT_DECL_RET;
- WT_FH *close_fh;
WT_LOG *log;
- WT_LSN close_end_lsn, close_lsn, sync_lsn;
+ WT_LSN sync_lsn;
size_t write_size;
int locked;
WT_DECL_SPINLOCK_ID(id); /* Must appear last */
@@ -868,31 +868,6 @@ __log_release(WT_SESSION_IMPL *session, WT_LOGSLOT *slot)
log = conn->log;
locked = 0;
- /*
- * If we're going to have to close our log file, make a local copy
- * of the file handle structure.
- */
- close_fh = NULL;
- WT_INIT_LSN(&close_lsn);
- WT_INIT_LSN(&close_end_lsn);
- if (F_ISSET(slot, SLOT_CLOSEFH)) {
- close_fh = log->log_close_fh;
- /*
- * Set the close_end_lsn to the LSN immediately after ours.
- * That is, the beginning of the next log file. We need to
- * know the LSN file number of our own close in case earlier
- * calls are still in progress and the next one to move the
- * sync_lsn into the next file for later syncs.
- */
- WT_ERR(__wt_log_extract_lognum(session, close_fh->name,
- &close_lsn.file));
- close_lsn.offset = 0;
- close_end_lsn = close_lsn;
- close_end_lsn.file++;
- log->log_close_fh = NULL;
- F_CLR(slot, SLOT_CLOSEFH);
- }
-
/* Write the buffered records */
if (F_ISSET(slot, SLOT_BUFFERED)) {
write_size = (size_t)
@@ -909,32 +884,8 @@ __log_release(WT_SESSION_IMPL *session, WT_LOGSLOT *slot)
__wt_yield();
log->write_lsn = slot->slot_end_lsn;
- /*
- * If we have a file to close, close it now. First fsync so
- * that a later sync will be assured all earlier transactions
- * in earlier log files are also on disk. We have to do this
- * before potentially syncing our own operation.
- */
- if (close_fh) {
- WT_ERR(__wt_fsync(session, close_fh));
- /*
- * This loop guarantees sync_lsn is updated in file order
- * so that when sync_lsn is updated, we know all earlier files
- * have already been fully processed.
- */
- while (log->sync_lsn.file < close_lsn.file ||
- __wt_spin_trylock(session, &log->log_sync_lock, &id) != 0) {
- WT_ERR(__wt_cond_wait(
- session, log->log_sync_cond, 10000));
- continue;
- }
- locked = 1;
- WT_ERR(__wt_close(session, close_fh));
- log->sync_lsn = close_end_lsn;
- WT_ERR(__wt_cond_signal(session, log->log_sync_cond));
- locked = 0;
- __wt_spin_unlock(session, &log->log_sync_lock);
- }
+ if (F_ISSET(slot, SLOT_CLOSEFH))
+ WT_ERR(__wt_cond_signal(session, conn->log_close_cond));
/*
* Try to consolidate calls to fsync to wait less. Acquire a spin lock
@@ -956,10 +907,10 @@ __log_release(WT_SESSION_IMPL *session, WT_LOGSLOT *slot)
locked = 1;
/*
- * Record the current end of log after we grabbed the lock.
+ * Record the current end of our update after the lock.
* That is how far our calls can guarantee.
*/
- sync_lsn = log->write_lsn;
+ sync_lsn = slot->slot_end_lsn;
/*
* Check if we have to sync the parent directory. Some
* combinations of sync flags may result in the log file
@@ -1506,6 +1457,13 @@ __wt_log_write(WT_SESSION_IMPL *session, WT_ITEM *record, WT_LSN *lsnp,
conn = S2C(session);
log = conn->log;
+ /*
+ * An error during opening the logging subsystem can result in it
+ * being enabled, but without an open log file. In that case,
+ * just return.
+ */
+ if (log->log_fh == NULL)
+ return (0);
ip = record;
if ((compressor = conn->log_compressor) != NULL &&
record->size < log->allocsize)
diff --git a/src/third_party/wiredtiger/src/lsm/lsm_cursor.c b/src/third_party/wiredtiger/src/lsm/lsm_cursor.c
index 0d44b16d85c..39b4b4508b7 100644
--- a/src/third_party/wiredtiger/src/lsm/lsm_cursor.c
+++ b/src/third_party/wiredtiger/src/lsm/lsm_cursor.c
@@ -1429,6 +1429,7 @@ __wt_clsm_open(WT_SESSION_IMPL *session,
__wt_cursor_set_key, /* set-key */
__wt_cursor_set_value, /* set-value */
__clsm_compare, /* compare */
+ __wt_cursor_equal, /* equals */
__clsm_next, /* next */
__clsm_prev, /* prev */
__clsm_reset, /* reset */
@@ -1437,6 +1438,7 @@ __wt_clsm_open(WT_SESSION_IMPL *session,
__clsm_insert, /* insert */
__clsm_update, /* update */
__clsm_remove, /* remove */
+ __wt_cursor_reconfigure, /* reconfigure */
__clsm_close); /* close */
WT_CURSOR *cursor;
WT_CURSOR_LSM *clsm;
@@ -1468,8 +1470,6 @@ __wt_clsm_open(WT_SESSION_IMPL *session,
cursor->key_format = lsm_tree->key_format;
cursor->value_format = lsm_tree->value_format;
- WT_ERR(__wt_cursor_config_readonly(cursor, cfg, 0));
-
clsm->lsm_tree = lsm_tree;
/*
diff --git a/src/third_party/wiredtiger/src/reconcile/rec_write.c b/src/third_party/wiredtiger/src/reconcile/rec_write.c
index 5c3ed614850..0300596f90b 100644
--- a/src/third_party/wiredtiger/src/reconcile/rec_write.c
+++ b/src/third_party/wiredtiger/src/reconcile/rec_write.c
@@ -2432,15 +2432,7 @@ no_slots:
* the "page" and try again after we accumulate some more rows.
*/
WT_STAT_FAST_DATA_INCR(session, compress_raw_fail_temporary);
-
-split_grow: /*
- * Double the page size and make sure we accommodate at least
- * one more record. The reason for the latter is that we may
- * be here because there's a large key/value pair that won't
- * fit in our initial page buffer, even at its expanded size.
- */
- r->page_size *= 2;
- return (__rec_split_grow(session, r, r->page_size + next_len));
+ goto split_grow;
}
/* We have a block, update the boundary counter. */
@@ -2462,6 +2454,22 @@ split_grow: /*
} else
WT_RET(
__rec_split_write(session, r, last, write_ref, last_block));
+
+ /*
+ * We got called because there wasn't enough room in the buffer for the
+ * next key and we might or might not have written a block. In any case,
+ * make sure the next key fits into the buffer.
+ */
+ if (r->space_avail < next_len) {
+split_grow: /*
+ * Double the page size and make sure we accommodate at least
+ * one more record. The reason for the latter is that we may
+ * be here because there's a large key/value pair that won't
+ * fit in our initial page buffer, even at its expanded size.
+ */
+ r->page_size *= 2;
+ return (__rec_split_grow(session, r, r->page_size + next_len));
+ }
return (0);
}
diff --git a/src/third_party/wiredtiger/src/session/session_compact.c b/src/third_party/wiredtiger/src/session/session_compact.c
index acb8b730a7d..2a18eb3be93 100644
--- a/src/third_party/wiredtiger/src/session/session_compact.c
+++ b/src/third_party/wiredtiger/src/session/session_compact.c
@@ -175,11 +175,13 @@ __compact_file(WT_SESSION_IMPL *session, const char *uri, const char *cfg[])
WT_ERR(__wt_epoch(session, &start_time));
/*
- * We compact 10% of the file on each pass, try 10 times (which is
- * probably overkill), and quit if we make no progress. Check for a
- * timeout each time through the loop.
+ * We compact 10% of the file on each pass (but the overall size of the
+ * file is decreasing each time, so we're not compacting 10% of the
+ * original file each time). Try 100 times (which is clearly more than
+ * we need); quit if we make no progress and check for a timeout each
+ * time through the loop.
*/
- for (i = 0; i < 10; ++i) {
+ for (i = 0; i < 100; ++i) {
WT_ERR(wt_session->checkpoint(wt_session, t->data));
session->compaction = 0;
diff --git a/src/third_party/wiredtiger/tools/statlog.py b/src/third_party/wiredtiger/tools/statlog.py
deleted file mode 100644
index a597eb060ae..00000000000
--- a/src/third_party/wiredtiger/tools/statlog.py
+++ /dev/null
@@ -1,125 +0,0 @@
-#!/usr/bin/env python
-#
-# Public Domain 2014-2015 MongoDB, Inc.
-# Public Domain 2008-2014 WiredTiger, Inc.
-#
-# This is free and unencumbered software released into the public domain.
-#
-# Anyone is free to copy, modify, publish, use, compile, sell, or
-# distribute this software, either in source code form or as a compiled
-# binary, for any purpose, commercial or non-commercial, and by any
-# means.
-#
-# In jurisdictions that recognize copyright laws, the author or authors
-# of this software dedicate any and all copyright interest in the
-# software to the public domain. We make this dedication for the benefit
-# of the public at large and to the detriment of our heirs and
-# successors. We intend this dedication to be an overt act of
-# relinquishment in perpetuity of all present and future rights to this
-# software under copyright law.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
-# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
-# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
-# OTHER DEALINGS IN THE SOFTWARE.
-#
-
-import fileinput, os, shutil, sys, textwrap
-from collections import defaultdict
-from datetime import datetime
-from subprocess import call
-
-# Import the data describing which statistics should not be scaled
-from stat_data import no_scale_per_second_list
-
-TIMEFMT = "%b %d %H:%M:%S"
-reportno = 0
-
-# Plot a set of entries for a title.
-def plot(title, values):
- global reportno
-
- # Ignore entries where the value never changes.
- skip = True
- t0, v0 = values[0]
- for t, v in values:
- if v != v0:
- skip = False
- break
- if skip:
- print 'skipping: ' + title
- return
-
- print 'building: ' + title
- reportno = reportno + 1
- num = "%03d" % reportno
-
- ylabel = 'Value'
- if title.split(' ')[1] != 'spinlock' and \
- title.split(' ', 1)[1] in no_scale_per_second_list:
- seconds = 1
- else:
- t1, v1 = values[1]
- seconds = (datetime.strptime(t1, TIMEFMT) -
- datetime.strptime(t0, TIMEFMT)).seconds
- if seconds == 0:
- seconds = 1
- ylabel += ' per second'
-
- # Write the raw data into a file for processing.
- of = open("reports/raw/report.%s.raw" % num, "w")
- for t, v in sorted(values):
- print >>of, "%s %g" % (t, float(v) / seconds)
- of.close()
-
- # Write a command file for gnuplot.
- of = open("gnuplot.cmd", "w")
- of.write('''
-set terminal png nocrop size 800,600
-set autoscale
-set grid
-set style data linespoints
-set title "%(title)s"
-set xlabel "Time"
-set xtics rotate by -45
-set xdata time
-set timefmt "%(timefmt)s"
-set format x "%(timefmt)s"
-set ylabel "%(ylabel)s"
-set yrange [0:]
-set output 'reports/report.%(num)s.png'
-plot "reports/raw/report.%(num)s.raw" using 1:4 notitle''' % {
- 'num' : num,
- 'timefmt' : TIMEFMT,
- 'title' : title,
- 'ylabel' : ylabel,
- })
- of.close()
-
- # Run gnuplot.
- call(["gnuplot", "gnuplot.cmd"])
-
- # Remove the command file.
- os.remove("gnuplot.cmd")
-
-# Read the input into a dictionary of lists.
-if sys.argv[1:] == []:
- print "usage: " + sys.argv[0] + " file ..."
- sys.exit(1)
-
-# Remove and re-create the reports folder.
-shutil.rmtree("reports", True)
-os.makedirs("reports/raw")
-
-d = defaultdict(list)
-for line in fileinput.input(sys.argv[1:]):
- month, day, time, v, desc = line.strip('\n').split(" ", 4)
- d[desc].append((month + " " + day + " " + time, v))
-
-# Plot each entry in the dictionary.
-for items in sorted(d.iteritems()):
- plot('\\n'.join(l for l in textwrap.wrap(items[0], 60)), items[1])
-
diff --git a/src/third_party/wiredtiger/tools/wt_nvd3_util.py b/src/third_party/wiredtiger/tools/wt_nvd3_util.py
deleted file mode 100644
index 435c2ce71e7..00000000000
--- a/src/third_party/wiredtiger/tools/wt_nvd3_util.py
+++ /dev/null
@@ -1,58 +0,0 @@
-#!/usr/bin/env python
-#
-# Public Domain 2014-2015 MongoDB, Inc.
-# Public Domain 2008-2014 WiredTiger, Inc.
-#
-# This is free and unencumbered software released into the public domain.
-#
-# Anyone is free to copy, modify, publish, use, compile, sell, or
-# distribute this software, either in source code form or as a compiled
-# binary, for any purpose, commercial or non-commercial, and by any
-# means.
-#
-# In jurisdictions that recognize copyright laws, the author or authors
-# of this software dedicate any and all copyright interest in the
-# software to the public domain. We make this dedication for the benefit
-# of the public at large and to the detriment of our heirs and
-# successors. We intend this dedication to be an overt act of
-# relinquishment in perpetuity of all present and future rights to this
-# software under copyright law.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
-# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
-# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
-# OTHER DEALINGS IN THE SOFTWARE.
-#
-import os, sys
-from datetime import datetime
-
-tool_dir = os.path.split(sys.argv[0])[0]
-# Make sure Python finds the NVD3 in our third party directory, to
-# avoid compatability issues
-sys.path.append(os.path.join(tool_dir, "3rdparty"))
-
-try:
- from nvd3 import lineChart
-except ImportError:
- print >>sys.stderr, "Could not import nvd3. It should be installed locally."
- sys.exit(-1)
-
-# Add a multiChart type so we can overlay line graphs
-class multiChart(lineChart):
- def __init__(self, **kwargs):
- lineChart.__init__(self, **kwargs)
-
- # Fix the axes
- del self.axislist['yAxis']
- self.create_y_axis('yAxis1', format=kwargs.get('y_axis_format', '.02f'))
- self.create_y_axis('yAxis2', format=kwargs.get('y_axis_format', '.02f'))
-
-TIMEFMT = "%b %d %H:%M:%S"
-
-thisyear = datetime.today().year
-def parsetime(s):
- return datetime.strptime(s, TIMEFMT).replace(year=thisyear)
-
diff --git a/src/third_party/wiredtiger/tools/wtperf_graph.py b/src/third_party/wiredtiger/tools/wtperf_graph.py
deleted file mode 100644
index d80eb0cbce5..00000000000
--- a/src/third_party/wiredtiger/tools/wtperf_graph.py
+++ /dev/null
@@ -1,235 +0,0 @@
-#!/usr/bin/env python
-#
-# Public Domain 2014-2015 MongoDB, Inc.
-# Public Domain 2008-2014 WiredTiger, Inc.
-#
-# This is free and unencumbered software released into the public domain.
-#
-# Anyone is free to copy, modify, publish, use, compile, sell, or
-# distribute this software, either in source code form or as a compiled
-# binary, for any purpose, commercial or non-commercial, and by any
-# means.
-#
-# In jurisdictions that recognize copyright laws, the author or authors
-# of this software dedicate any and all copyright interest in the
-# software to the public domain. We make this dedication for the benefit
-# of the public at large and to the detriment of our heirs and
-# successors. We intend this dedication to be an overt act of
-# relinquishment in perpetuity of all present and future rights to this
-# software under copyright law.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
-# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
-# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
-# OTHER DEALINGS IN THE SOFTWARE.
-#
-
-import csv, os, sys
-from subprocess import call
-# Python script to read wtperf monitor output and create a performance
-# graph.
-
-TIMEFMT = "%b %d %H:%M:%S"
-
-def process_monitor(fname, sfx, ckptlist, opdict):
- # Read the monitor file and figure out when a checkpoint was running.
- in_ckpt = 'N'
-
- ckptlist=[]
-
- ofname = 'monitor%s.png' % (sfx)
- # Monitor output format currently is:
- # time,totalsec,read,insert,update,ckpt,...latencies...
- ops = ('read', 'insert', 'update')
- csvcol = (2, 3, 4)
- with open(fname, 'r') as csvfile:
- reader = csv.reader(csvfile)
- for row in reader:
- if row[0].lstrip().startswith('#'):
- continue
- # Look for checkpoints and operations.
- if row[5] != in_ckpt:
- ckptlist.append(row[0])
- in_ckpt = row[5]
- for op, col in zip(ops, csvcol):
- if row[col] != '0' and opdict[op] == 0:
- opdict[op] = 1
-
- if in_ckpt == 'Y':
- ckptlist.append(row[0])
-
- # Graph time vs. read, insert and update operations per second.
- gcmd = "gnuplot.mon.cmd"
- of = open(gcmd, "w")
- of.write('''
-set autoscale
-set datafile sep ','
-set grid
-set style data lines
-set terminal png nocrop size 800,600
-set timefmt "%(TIMEFMT)s"
-set title "read, insert and update operations per second"
-set format x "%(TIMEFMT)s"
-set xlabel "Time"
-set xtics rotate by -45
-set xdata time
-set ylabel "Operations per second (thousands)"
-set yrange [0:]\n''' % {
- 'TIMEFMT' : TIMEFMT
- })
- it = iter(ckptlist)
- for start, stop in zip(it, it):
- of.write("set object rectangle from first '%s',\
- graph 0 to first '%s',\
- graph 1 fc rgb \"gray\" back\n" % (start, stop))
- of.write('set output "%s"\n' % (ofname))
- of.write("""plot "{name}" using 1:($3/1000) title "Reads", \\
- "{name}" using 1:($4/1000) title "Inserts",\\
- "{name}" using 1:($5/1000) title "Updates"
- """.format(name=fname))
- of.close()
- call(["gnuplot", gcmd])
- os.remove(gcmd)
-
-# Graph time vs. average, minimium, maximum latency for an operation.
-def plot_latency_operation(name, fname, sfx, ckptlist, col_avg, col_min, col_max):
- gcmd = "gnuplot." + name + ".l1.cmd"
- of = open(gcmd, "w")
- of.write('''
-set autoscale
-set datafile sep ','
-set grid
-set style data lines
-set terminal png nocrop size 800,600
-set timefmt "%(TIMEFMT)s"
-set title "%(NAME)s: average, minimum and maximum latency"
-set format x "%(TIMEFMT)s"
-set xlabel "Time"
-set xtics rotate by -45
-set xdata time
-set ylabel "Latency (us)"
-set logscale y
-set yrange [1:]\n''' % {
- 'NAME' : name,
- 'TIMEFMT' : TIMEFMT
- })
- it = iter(ckptlist)
- for start, stop in zip(it, it):
- of.write('set object rectangle from first \'' + start +\
- '\', graph 0 ' + ' to first \'' + stop +\
- '\', graph 1 fc rgb "gray" back\n')
- ofname = name + sfx + '.latency1.png'
- of.write('set output "' + ofname + '"\n')
- of.write('plot "' +\
- fname + '" using 1:($' + repr(col_avg) +\
- ') title "Average Latency", "' + fname +'" using 1:($' +\
- repr(col_min) + ') title "Minimum Latency", "' +\
- fname + '" using 1:($' + repr(col_max) +\
- ') title "Maximum Latency"\n')
- of.close()
- call(["gnuplot", gcmd])
- os.remove(gcmd)
-
-
-# Graph latency vs. % operations
-def plot_latency_percent(name, dirname, sfx, ckptlist):
- lfile = os.path.join(dirname, 'latency.' + name)
- if not os.path.exists(lfile):
- return
- gcmd = "gnuplot." + name + ".l2.cmd"
- of = open(gcmd, "w")
- of.write('''
-set autoscale
-set datafile sep ','
-set grid
-set style data points
-set terminal png nocrop size 800,600\n''')
- of.write('set title "' + name + ': latency distribution"\n')
- of.write('''
-set xlabel "Latency (us)"
-set xrange [1:]
-set xtics rotate by -45
-set logscale x
-set ylabel "%% operations"
-set yrange [0:]\n''')
- ofname = name + sfx + '.latency2.png'
- of.write('set output "' + ofname + '"\n')
- of.write('plot "' + lfile + sfx +\
- '" using (($2 * 100)/$4) title "' + name + '"\n')
- of.close()
- call(["gnuplot", gcmd])
- os.remove(gcmd)
-
-
-# Graph latency vs. % operations (cumulative)
-def plot_latency_cumulative_percent(name, dirname, sfx, ckptlist):
- lfile = os.path.join(dirname, 'latency.' + name)
- if not os.path.exists(lfile):
- return
- # Latency plot: cumulative operations vs. latency
- gcmd = "gnuplot." + name + ".l3.cmd"
- of = open(gcmd, "w")
- of.write('''
-set autoscale
-set datafile sep ','
-set grid
-set style data lines
-set terminal png nocrop size 800,600
-set title "%(NAME)s: cumulative latency distribution"
-set xlabel "Latency (us)"
-set xrange [1:]
-set xtics rotate by -45
-set logscale x
-set ylabel "%% operations"
-set yrange [0:]\n''' % {
- 'NAME' : name
- })
- ofname = name + sfx + '.latency3.png'
- of.write('set output "' + ofname + '"\n')
- of.write('plot "' + lfile + sfx +\
- '" using 1:(($3 * 100)/$4) title "' + name + '"\n')
- of.close()
- call(["gnuplot", gcmd])
- os.remove(gcmd)
-
-def process_file(fname):
- ckptlist = []
- # NOTE: The operations below must be in this exact order to match
- # the operation latency output in the monitor file.
- opdict={'read':0, 'insert':0, 'update':0}
-
- # This assumes the monitor file has the string "monitor"
- # and any other (optional) characters in the filename are a suffix.
- sfx = os.path.basename(fname).replace('monitor','')
- dirname = os.path.dirname(fname)
-
- process_monitor(fname, sfx, ckptlist, opdict)
- column = 7 # average, minimum, maximum start in column 7
- for k, v in opdict.items():
- if v != 0:
- plot_latency_operation(
- k, fname, sfx, ckptlist, column, column + 1, column + 2)
- plot_latency_percent(k, dirname, sfx, ckptlist)
- plot_latency_cumulative_percent(k, dirname, sfx, ckptlist)
- else:
- print fname + ': no ' + k + ' operations found. Skip.'
- column = column + 3
-
-def main():
- # This program takes a list of monitor files generated by
- # wtperf. If no args are given, it looks for a single file
- # named 'monitor'.
- numargs = len(sys.argv)
- if numargs < 2:
- process_file('monitor')
- else:
- d = 1
- while d < numargs:
- process_file(sys.argv[d])
- d += 1
-
-if __name__ == '__main__':
- main()
diff --git a/src/third_party/wiredtiger/tools/wtperf_stats.py b/src/third_party/wiredtiger/tools/wtperf_stats.py
deleted file mode 100644
index 5e6e1733fb7..00000000000
--- a/src/third_party/wiredtiger/tools/wtperf_stats.py
+++ /dev/null
@@ -1,178 +0,0 @@
-#!/usr/bin/env python
-#
-# Public Domain 2014-2015 MongoDB, Inc.
-# Public Domain 2008-2014 WiredTiger, Inc.
-#
-# This is free and unencumbered software released into the public domain.
-#
-# Anyone is free to copy, modify, publish, use, compile, sell, or
-# distribute this software, either in source code form or as a compiled
-# binary, for any purpose, commercial or non-commercial, and by any
-# means.
-#
-# In jurisdictions that recognize copyright laws, the author or authors
-# of this software dedicate any and all copyright interest in the
-# software to the public domain. We make this dedication for the benefit
-# of the public at large and to the detriment of our heirs and
-# successors. We intend this dedication to be an overt act of
-# relinquishment in perpetuity of all present and future rights to this
-# software under copyright law.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
-# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
-# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
-# OTHER DEALINGS IN THE SOFTWARE.
-#
-
-import os, csv, operator, sys
-from time import mktime
-
-tool_dir = os.path.split(sys.argv[0])[0]
-sys.path.append(tool_dir)
-
-try:
- from wt_nvd3_util import multiChart, parsetime
-except ImportError:
- print >>sys.stderr, "Could not import wt_nvd3_util.py, it should be\
- in the same directory as %s" % sys.argv[0]
- sys.exit(-1)
-
-def timesort(s):
- # Sort the timestr via its parsetime() value so that the year gets
- # added and it properly sorts. Times are only %b %d %H:%M:%S and
- # may improperly sort if the data crosses a month boundary.
- t = operator.itemgetter('#time')
- timestr = t(s)
- return parsetime(timestr)
-
-# Fixup the names and values in a dictionary read in from a csv file. One
-# field must be "#time" - which is used to calculate the interval.
-# Input is a dictionary, output is a list of dictionaries with a single entry.
-def munge_dict(values_dict, abstime):
- sorted_values = sorted(values_dict, key=timesort)
- start_time = parsetime(sorted_values[0]['#time'])
-
- ret = []
- for v in sorted_values:
- if abstime:
- # Build the time series, milliseconds since the epoch
- v['#time'] = int(mktime(parsetime(v['#time']).timetuple())) * 1000
- else:
- # Build the time series as seconds since the start of the data
- v['#time'] = (parsetime(v['#time']) - start_time).seconds
- next_val = {}
- for title, value in v.items():
- if title.find('uS') != -1:
- title = title.replace('uS', 'ms')
- value = float(value) / 1000
- if title == 'totalsec':
- value = 0
- if title == 'checkpoints' and value == 'N':
- value = 0
- elif title.find('time') != -1:
- title = 'time'
- elif title.find('latency') == -1 and \
- title.find('checkpoints') == -1:
- title = title + ' (thousands)'
- value = float(value) / 1000
- next_val[title] = value
- ret.append(next_val)
-
- # After building the series, eliminate constants
- d0 = ret[0]
- for t0, v0 in d0.items():
- skip = True
- for d in ret:
- v = d[t0]
- if v != v0:
- skip = False
- break
- if skip:
- for dicts in ret:
- del dicts[t0]
-
- return ret
-
-def addPlotsToChart(chart, graph_data, wtstat_chart = False):
- # Extract the times - they are the same for all lines.
- times = []
- for v in graph_data:
- times.append(v['time'])
-
- # Add a line to the graph for each field in the CSV file in alphabetical
- # order, so the key is sorted.
- for field in sorted(graph_data[0].keys()):
- if field == 'time':
- continue
- # Split the latency and non-latency measurements onto different scales
- axis = "1"
- if not wtstat_chart and field.find('latency') == -1:
- axis="2"
- ydata = []
- for v in graph_data:
- ydata.append(v[field])
- chart.add_serie(x=times, y=ydata, name=field, type="line", yaxis=axis)
-
-# Input parameters are a chart populated with WiredTiger statistics and
-# the directory where the wtperf monitor file can be found.
-def addPlotsToStatsChart(chart, dirname, abstime):
- fname = os.path.join(dirname, 'monitor')
- try:
- with open(fname, 'rb') as csvfile:
- reader = csv.DictReader(csvfile)
- # Transform the data into something NVD3 can digest
- graph_data = munge_dict(reader, abstime)
- except IOError:
- print >>sys.stderr, "Could not open wtperf monitor file."
- sys.exit(-1)
- addPlotsToChart(chart, graph_data, 1)
-
-def main():
- # Parse the command line
- import argparse
-
- parser = argparse.ArgumentParser(description='Create graphs from WiredTiger statistics.')
- parser.add_argument('--abstime', action='store_true',
- help='use absolute time on the x axis')
- parser.add_argument('--output', '-o', metavar='file',
- default='wtperf_stats.html', help='HTML output file')
- parser.add_argument('files', metavar='file', nargs='+',
- help='input monitor file generated by WiredTiger wtperf application')
- args = parser.parse_args()
-
- output_file = open(args.output, 'w')
-
- if len(args.files) != 1:
- print 'Script currently only supports a single monitor file'
- exit (1)
-
- chart_extra = {}
- # Add in the x axis if the user wants time.
- if args.abstime:
- chart_extra['x_axis_format'] = '%H:%M:%S'
-
- for f in args.files:
- with open(f, 'rb') as csvfile:
- reader = csv.DictReader(csvfile)
- # Transform the data into something NVD3 can digest
- graph_data = munge_dict(reader, args.abstime)
-
- chart = multiChart(name='wtperf',
- height=450 + 10*len(graph_data[0].keys()),
- resize=True,
- x_is_date=args.abstime,
- assets_directory='http://source.wiredtiger.com/graphs/',
- **chart_extra)
-
- addPlotsToChart(chart, graph_data)
-
- chart.buildhtml()
- output_file.write(chart.htmlcontent)
- output_file.close()
-
-if __name__ == '__main__':
- main()
-
diff --git a/src/third_party/wiredtiger/tools/wtstats.py b/src/third_party/wiredtiger/tools/wtstats.py
index 740a2017d63..031b7cb546f 100644..100755
--- a/src/third_party/wiredtiger/tools/wtstats.py
+++ b/src/third_party/wiredtiger/tools/wtstats.py
@@ -27,53 +27,30 @@
# OTHER DEALINGS IN THE SOFTWARE.
#
-import fileinput, os, re, shutil, sys, textwrap
+import os, re, sys
from collections import defaultdict
from glob import glob
-from time import mktime
-from subprocess import call
-
-# Make sure Python can find files in the tools directory
-tool_dir = os.path.split(sys.argv[0])[0]
-
-# Make sure Python finds the NVD3 in our third party directory.
-# To avoid compatability issues, prepend it to the system path.
-sys.path = [ os.path.join(tool_dir, "3rdparty") ] + sys.path
+import json
+from datetime import datetime
try:
from stat_data \
import groups, no_scale_per_second_list, no_clear_list, prefix_list
except ImportError:
- print >>sys.stderr, "Could not import stat_data.py, it should be\
- in the same directory as %s" % sys.argv[0]
+ print >>sys.stderr, "Could not import stat_data.py, it should be" \
+ "in the same directory as %s" % sys.argv[0]
sys.exit(-1)
-try:
- from wtperf_stats import addPlotsToStatsChart
-except ImportError:
- print >>sys.stderr, "Could not import wtperf_stats.py, it should be\
- in the same directory as %s" % sys.argv[0]
- sys.exit(-1)
-
-try:
- from wt_nvd3_util import multiChart, parsetime
-except ImportError:
- print >>sys.stderr, "Could not import wt_nvd3_util.py, it should be\
- in the same directory as %s" % sys.argv[0]
- sys.exit(-1)
-
-try:
- from nvd3 import lineChart, lineWithFocusChart
-except ImportError:
- print >>sys.stderr, "Could not import nvd3 it should be installed locally"
- sys.exit(-1)
+thisyear = datetime.today().year
+def parsetime(s):
+ return datetime.strptime(s, "%b %d %H:%M:%S").replace(year=thisyear)
if sys.version_info<(2,7,0):
print >>sys.stderr, "You need python 2.7 or later to run this script"
sys.exit(-1)
# Plot a set of entries for a title.
-def munge(title, values):
+def munge(args, title, values):
t0, v0 = values[0]
start_time = parsetime(t0)
@@ -96,13 +73,6 @@ def munge(title, values):
ydata = {}
last_value = 0.0
for t, v in sorted(values):
- if args.abstime:
- # Build the time series, milliseconds since the epoch
- x = int(mktime(parsetime(t).timetuple())) * 1000
- else:
- # Build the time series as seconds since the start of the data
- x = (parsetime(t) - start_time).seconds
-
float_v = float(v)
if not stats_cleared:
float_v = float_v - last_value
@@ -111,191 +81,185 @@ def munge(title, values):
if float_v < 0:
float_v = 0.0
last_value = float(v)
- ydata[x] = float_v / seconds
+ ydata[t] = float_v / seconds
return ylabel, ydata
# Parse the command line
import argparse
-parser = argparse.ArgumentParser(description='Create graphs from WiredTiger statistics.')
-parser.add_argument('--abstime', action='store_true',
- help='use absolute time on the x axis')
-parser.add_argument('--all', '-A', action='store_true',
- help='generate all series as separate HTML output files by category')
-parser.add_argument('--clear', action='store_true',
- help='WiredTiger stats gathered with clear set')
-parser.add_argument('--focus', action='store_true',
- help='generate a chart with focus slider')
-parser.add_argument('--include', '-I', metavar='regexp',
- type=re.compile, action='append',
- help='include series with titles matching the specifed regexp')
-parser.add_argument('--list', action='store_true',
- help='list the series that would be displayed')
-parser.add_argument('--output', '-o', metavar='file', default='wtstats',
- help='HTML output file prefix')
-parser.add_argument('--right', '-R', metavar='regexp',
- type=re.compile, action='append',
- help='use the right axis for series with titles matching the specifed regexp')
-parser.add_argument('--wtperf', '-w', action='store_true',
- help='Plot wtperf statistics on the same graph')
-parser.add_argument('files', metavar='file', nargs='+',
- help='input files or directories generated by WiredTiger statistics logging')
-args = parser.parse_args()
-
-# Don't require users to specify regexps twice for right axis
-if args.focus and args.right:
- print >>sys.stderr, "focus charts cannot have a right-hand y-axis"
- sys.exit(-1)
+def main():
+ parser = argparse.ArgumentParser(description='Create graphs from' \
+ 'WiredTiger statistics.')
+ parser.add_argument('--all', '-A', action='store_true',
+ help='generate separate html files for each stats group')
+ parser.add_argument('--clear', action='store_true',
+ help='WiredTiger stats gathered with clear set')
+ parser.add_argument('--include', '-I', metavar='regexp',
+ type=re.compile, action='append',
+ help='only include series with titles matching regexp')
+ parser.add_argument('--list', action='store_true',
+ help='only list the parsed series, does not create html file')
+ parser.add_argument('--output', '-o', metavar='file', default='wtstats',
+ help='HTML output file prefix')
+ parser.add_argument('--json', action='store_true',
+ help='additionally output data series in json format')
+ parser.add_argument('files', metavar='file', nargs='+',
+ help='input files or directories generated by WiredTiger statistics' \
+ 'logging')
+ args = parser.parse_args()
+
+ # Read the input file(s) into a dictionary of lists.
+ def getfiles(l):
+ for f in l:
+ if os.path.isfile(f):
+ yield f
+ elif os.path.isdir(f):
+ for s in glob(os.path.join(f, 'WiredTigerStat*')):
+ print 'Processing ' + s
+ yield s
+
+ d = defaultdict(list)
+ for f in getfiles(args.files):
+ for line in open(f, 'rU'):
+ month, day, time, v, title = line.strip('\n').split(" ", 4)
+ d[title].append((month + " " + day + " " + time, v))
+
+ # Process the series, eliminate constants
+ for title, values in sorted(d.iteritems()):
+ skip = True
+ t0, v0 = values[0]
+ for t, v in values:
+ if v != v0:
+ skip = False
+ break
+ if skip:
+ #print "Skipping", title
+ del d[title]
+
+ # Common prefix / suffix elimination
+ prefix = suffix = None
+
+ def common_prefix(a, b):
+ while not b.startswith(a):
+ a = a[:-1]
+ return a
+
+ def common_suffix(a, b):
+ while not a.endswith(b):
+ b = b[1:]
+ return b
+
+ def output_series(results, prefix=None, grouplist=[]):
+ # add .html ending if not present
+ filename, ext = os.path.splitext(args.output)
+ if ext == '':
+ ext = '.html'
+
+ # open the output file based on prefix
+ if prefix == None:
+ outputname = filename + ext
+ elif len(grouplist) == 0:
+ outputname = filename +'.' + prefix + ext
+ else:
+ outputname = filename +'.group.' + prefix + ext
-# Don't require users to specify regexps twice for right axis
-if args.include and args.right:
- args.include += args.right
-
-# Read the input file(s) into a dictionary of lists.
-def getfiles(l):
- for f in l:
- if os.path.isfile(f):
- yield f
- elif os.path.isdir(f):
- for s in glob(os.path.join(f, 'WiredTigerStat*')):
- print 'Processing ' + s
- yield s
-
-d = defaultdict(list)
-for f in getfiles(args.files):
- for line in open(f, 'rU'):
- month, day, time, v, title = line.strip('\n').split(" ", 4)
- d[title].append((month + " " + day + " " + time, v))
-
-# Process the series, eliminate constants
-for title, values in sorted(d.iteritems()):
- skip = True
- t0, v0 = values[0]
- for t, v in values:
- if v != v0:
- skip = False
- break
- if skip:
- #print "Skipping", title
- del d[title]
-
-# Common prefix / suffix elimination
-prefix = suffix = None
-
-def common_prefix(a, b):
- while not b.startswith(a):
- a = a[:-1]
- return a
-
-def common_suffix(a, b):
- while not a.endswith(b):
- b = b[1:]
- return b
-
-def output_series(results, prefix=None, grouplist=[]):
- # open the output file based on prefix
- if prefix == None:
- outputname = args.output + '.html'
- elif len(grouplist) == 0:
- outputname = args.output +'.' + prefix + '.html'
- else:
- outputname = args.output +'.group.' + prefix + '.html'
-
- if prefix != None and len(grouplist) == 0:
- this_series = []
- for title, yaxis, ydata in results:
- if not prefix in title:
- continue
- #print 'Appending to dataset: ' + title
- this_series.append((title, yaxis, ydata))
- elif prefix != None and len(grouplist) > 0:
- this_series = []
- for title, yaxis, ydata in results:
- for subgroup in grouplist:
- if not subgroup in title:
+ if prefix != None and len(grouplist) == 0:
+ this_series = []
+ for title, ydata in results:
+ if not prefix in title:
continue
- # print 'Appending to dataset: ' + title
- this_series.append((title, yaxis, ydata))
- else:
- this_series = results
-
- if len(this_series) == 0:
- print 'Output: ' + outputname + ' has no data. Do not create.'
- return
-
- #---------------------------------------
- if args.right:
- charttype = multiChart
- elif args.focus:
- charttype = lineWithFocusChart
- else:
- charttype = lineChart
-
- chart_extra = {}
- # Add in the x axis if the user wants time.
- if args.abstime:
- chart_extra['x_axis_format'] = '%H:%M:%S'
-
- # Create the chart, add the series
- chart = charttype(name='statlog', height=450+10*len(this_series), resize=True, x_is_date=args.abstime, y_axis_format='g', assets_directory='http://source.wiredtiger.com/graphs/', **chart_extra)
-
- for title, yaxis, ydata in this_series:
- chart.add_serie(x=xdata, y=(ydata.get(x, 0) for x in xdata), name=title,
- type="line", yaxis="2" if yaxis else "1")
-
- if args.wtperf:
- addPlotsToStatsChart(chart, os.path.dirname(args.files[0]), args.abstime)
-
- chart.buildhtml()
- output_file = open(outputname, 'w')
- output_file.write(chart.htmlcontent)
-
- #close Html file
- output_file.close()
-
-
-# Split out the data, convert timestamps
-results = []
-for title, values in sorted(d.iteritems()):
- title, ydata = munge(title, values)
- # Ignore entries if a list of regular expressions was given
- if args.include and not [r for r in args.include if r.search(title)]:
+ #print 'Appending to dataset: ' + title
+ this_series.append((title, ydata))
+ elif prefix != None and len(grouplist) > 0:
+ this_series = []
+ for title, ydata in results:
+ for subgroup in grouplist:
+ if not subgroup in title:
+ continue
+ # print 'Appending to dataset: ' + title
+ this_series.append((title, ydata))
+ else:
+ this_series = results
+
+ if len(this_series) == 0:
+ print 'Output: ' + outputname + ' has no data. Do not create.'
+ return
+
+
+ json_output = { "series": [] }
+
+ for title, ydata in this_series:
+ json_output["series"].append({
+ "key": title,
+ "values": ydata,
+ });
+
+ # load template
+ this_path = os.path.dirname(os.path.realpath(__file__))
+ srcfile = os.path.join(this_path, 'wtstats.html.template')
+ try:
+ srcfile = open(srcfile)
+ contents = srcfile.read()
+ except IOError:
+ print >>sys.stderr, "Cannot find template file 'wtstats.html." \
+ "template'. See ./template/README.md for more information."
+ sys.exit(-1)
+
+ srcfile.close()
+
+ # if --json write data to <filename>.json
+ if args.json:
+ jsonfile = filename + '.json'
+ with open(jsonfile, 'w') as f:
+ json.dump(json_output, f)
+ print "created %s" % jsonfile
+
+ # write output file
+ dstfile = open(outputname, 'wt')
+ replaced_contents = contents.replace('"### INSERT DATA HERE ###"',
+ json.dumps(json_output))
+ dstfile.write(replaced_contents)
+ dstfile.close()
+ print "created %s" % dstfile.name
+
+ # Split out the data, convert timestamps
+ results = []
+ for title, values in sorted(d.iteritems()):
+ title, ydata = munge(args, title, values)
+ # Ignore entries if a list of regular expressions was given
+ if args.include and not [r for r in args.include if r.search(title)]:
continue
- yaxis = args.right and [r for r in args.right if r.search(title)]
- prefix = title if prefix is None else common_prefix(prefix, title)
- suffix = title if suffix is None else common_suffix(title, suffix)
- results.append((title, yaxis, ydata))
-
-# Process titles, eliminate common prefixes and suffixes
-if prefix or suffix:
- new_results = []
- for title, yaxis, ydata in results:
- title = title[len(prefix):]
- if suffix:
- title = title[:-len(suffix)]
- new_results.append((title, yaxis, ydata))
- results = new_results
-
-# Dump the results as a CSV file
-#print '"time", ' + ', '.join('"%s"' % title for title, values in ydata)
-#for i in xrange(len(xdata)):
-# print '%d, %s' % (xdata[i], ', '.join('%g' % values[i] for title, values in ydata))
-
-# Are we just listing the results?
-if args.list:
- for title, yaxis, ydata in results:
- print title
- sys.exit(0)
-
-# Figure out the full set of x axis values
-xdata = sorted(set(k for k in ydata.iterkeys() for ydata in results))
-
-output_series(results)
-
-# If the user wants the stats split up by prefix type do so.
-if args.all:
- for prefix in prefix_list:
- output_series(results, prefix)
- for group in groups.keys():
- output_series(results, group, groups[group])
+ prefix = title if prefix is None else common_prefix(prefix, title)
+ suffix = title if suffix is None else common_suffix(title, suffix)
+ results.append((title, ydata))
+
+ # Process titles, eliminate common prefixes and suffixes
+ if prefix or suffix:
+ new_results = []
+ for title, ydata in results:
+ title = title[len(prefix):]
+ if suffix:
+ title = title[:-len(suffix)]
+ new_results.append((title, ydata))
+ results = new_results
+
+ # Are we just listing the results?
+ if args.list:
+ for title, ydata in results:
+ print title
+ sys.exit(0)
+
+ output_series(results)
+
+ # If the user wants the stats split up by prefix type do so.
+ if args.all:
+ for prefix in prefix_list:
+ output_series(results, prefix)
+ for group in groups.keys():
+ output_series(results, group, groups[group])
+
+
+if __name__ == '__main__':
+ main()
+