Import wiredtiger: f5c08e2b5f02805b062888d45c9eca19af175f7e from branch mongodb-3.2

ref: d48181f6f4..f5c08e2b5f
for: 3.2.13

SERVER-16796  Increase logging activity for journal recovery operations
SERVER-28168  Cannot start or repair mongodb after unexpected shutdown.
SERVER-28194  Missing WiredTiger.turtle file loses data
WT-2402       Misaligned structure accesses lead to undefined behavior
WT-2439       Enhance reconciliation page layout
WT-2771       Add a statistic to track per-btree dirty cache usage
WT-2790       Fix a text case false positive in test_sweep01
WT-2833       improvement: add projections to wt dump utility
WT-2898       Improve performance of eviction-heavy workloads by dynamically controlling the number of eviction threads
WT-2909       Create automatable test verifying checkpoint integrity after errors
WT-2978       Make WiredTiger python binding pip-compatible
WT-2990       checkpoint load live_open assertion failure
WT-2994       Create documentation describing page sizes and relationships
WT-3080       Python test suite: add timestamp or elapsed time for tests
WT-3082       Python test suite: shorten default run to avoid pull request timeouts.
WT-3083       Fix a bug in wtperf config dump
WT-3086       Add transaction state information to cache stuck diagnostic information
WT-3088       bug: Don't evict a page with refs visible to readers after a split
WT-3091       Add stats to test_perf0001
WT-3092       Quiet a warning from autogen.sh
WT-3093       Padding the WT_RWLOCK structure grew the WT_PAGE structure.
WT-3097       Race on reconfigure or shutdown can lead to waiting for statistics log server
WT-3099       lint: static function declarations, non-text characters in documentation
WT-3100       test bug: format is weighted to delete, insert, then write operations.
WT-3104       Fix wtperf configs for eviction tests
WT-3105       Fix a deadlock caused by allocating eviction thread sessions dynamically
WT-3106       Add truncate support to command line wt utility
WT-3108       Also dump disk page size as part of metadata information
WT-3109       wording fix in transaction doc
WT-3110       Add more test cases for the WT command line utility
WT-3111       util_create() doesnt free memory assigned to "uri"
WT-3112       Handle list lock statistic not incremented in eviction server
WT-3113       Add a verbose mode to dump the cache when eviction is stuck
WT-3114       Avoid archiving log files immediately after recovery
WT-3115       Change the dhandle lock to a read/write lock
WT-3116       Python style testing in s_all may not execute correctly
WT-3118       Protect random-abort test against unexpectedly slow child start
WT-3120       Fix ordering problem in connection_close for filesystem loaded in an extension
WT-3121       In test suite create standard way to load extensions
WT-3126       bug: dist/s_all script has misplaced quote causing bad error reporting
WT-3127       bug: CPU yield calls don't necessarily imply memory barriers
WT-3128       wt printlog returns operation-not-supported if it doesn't find any log files
WT-3130       Ensure extensions have access to database home directory
WT-3134       Coverity scan reports 1368529 and 1368528
WT-3135       search_near() for index with custom collator
WT-3136       bug fix: WiredTiger doesn't check sprintf calls for error return
WT-3137       Hang in __log_slot_join/__log_slot_switch_internal
WT-3139       Enhance wtperf to support periodic table scans
WT-3144       bug fix: random cursor returns not-found when descending to an empty page
WT-3148       Improve eviction efficiency with many small trees
WT-3149       Change eviction to start new walks from a random place in the tree
WT-3150       Reduce impact of checkpoints on eviction server
WT-3152       Convert table lock from a spinlock to a read write lock
WT-3155       Remove WT_CONN_SERVER_RUN flag
WT-3156       Assertion in log_write fires after write failure
WT-3157       checkpoint/transaction integrity issue when writes fail.
WT-3159       Incorrect key for index containing multiple variable sized entries
WT-3161       checkpoint hang after write failure injection.
WT-3164       Ensure all relevant btree fields are reset on checkpoint error
WT-3170       Clear the eviction walk point while populating from a tree
WT-3173       Add runtime detection for s390x CRC32 hardware support
WT-3174       Coverity/lint cleanup
WT-3175       New hang in internal page split
WT-3179       test bug: clang sanitizer failure in fail_fs
WT-3180       fault injection tests should only run as "long" tests and should not create core files
WT-3182       Switch make-check to run the short test suite by default
WT-3184       Problem duplicating index cursor with custom collator
WT-3186        Fix error path and panic detection in logging loops
WT-3187       Hang on shutdown with a busy cache pool
WT-3188       Fix error handling in logging where fatal errors could lead to a hang
WT-3189       Fix a segfault in the eviction server random positioning
WT-3190       Enhance eviction thread auto-tuning algorithm
WT-3191       lint
WT-3193       Close a race between verify opening a handle and eviction visiting it
WT-3196       Race with LSM and eviction when switching chunks
WT-3199       bug: eviction assertion failure
WT-3202       wtperf report an error on in_memory=true mode : No such file or directory
WT-3203       bulk-load state changes can race
WT-3204       eviction changes cost LSM performance
WT-3206       bug: core dump on NULL page index
WT-3207       Drops with checkpoint_wait=false should not wait for checkpoints
WT-3208       test format hung with 9mb cache
WT-3211       WT_CURSOR.remove cannot always retain its position.
WT-3212       'wt dump' crashes when given table with unknown collator
WT-3213       generated test/format CONFIG invalid on next run
WT-3216       add support for clang-tidy
WT-3218       unexpected checkpoint ordering failures
WT-3224       LSM assertion failure pindex->entries == 1
WT-3225       WiredTiger won't build with clang on CentOS 7.3.1611
WT-3227       Python test suite inserts unnecessary whitespace in error output.
WT-3228       Remove with overwrite shouldn't return WT_NOTFOUND
WT-3234       Update WiredTiger build for clang 4.0.
WT-3238       Java: Cursor.compare and Cursor.equals throw Exceptions for valid return values
WT-3240       Coverity reports
WT-3243       Reorder log slot release so joins don't wait on IO
WT-3244       metadata operations failing in in-memory configurations
WT-3249       Unit test test_readonly fails as it is unable to open WiredTiger.lock
WT-3250       Incorrect statistics incremented on Windows
WT-3254       test_reconfig02 uses incorrect configuration string
WT-3262       Schema operations shouldn't wait for cache
WT-3265       Verify hits assertion in eviction when transiting handle to exclusive mode
WT-3271       Eviction tuning stuck in a loop
WT-98         Update the current cursor value without a search

1 files changed, 288 insertions, 0 deletions

diff --git a/src/third_party/wiredtiger/test/csuite/scope/main.c b/src/third_party/wiredtiger/test/csuite/scope/main.c
new file mode 100644
index 00000000000..15dabd97c40
--- /dev/null
+++ b/src/third_party/wiredtiger/test/csuite/scope/main.c
@@ -0,0 +1,288 @@
+/*-
+ * Public Domain 2014-2016 MongoDB, Inc.
+ * Public Domain 2008-2014 WiredTiger, Inc.
+ *
+ * This is free and unencumbered software released into the public domain.
+ *
+ * Anyone is free to copy, modify, publish, use, compile, sell, or
+ * distribute this software, either in source code form or as a compiled
+ * binary, for any purpose, commercial or non-commercial, and by any
+ * means.
+ *
+ * In jurisdictions that recognize copyright laws, the author or authors
+ * of this software dedicate any and all copyright interest in the
+ * software to the public domain. We make this dedication for the benefit
+ * of the public at large and to the detriment of our heirs and
+ * successors. We intend this dedication to be an overt act of
+ * relinquishment in perpetuity of all present and future rights to this
+ * software under copyright law.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "test_util.h"
+
+#define	KEY	"key"
+#define	VALUE	"value"
+
+static int ignore_errors;
+
+static int
+handle_error(WT_EVENT_HANDLER *handler,
+    WT_SESSION *session, int error, const char *message)
+{
+	(void)(handler);
+
+	/* Skip the error messages we're expecting to see. */
+	if (ignore_errors > 0 &&
+	    (strstr(message, "requires key be set") != NULL ||
+	    strstr(message, "requires value be set") != NULL)) {
+		--ignore_errors;
+		return (0);
+	}
+
+	(void)fprintf(stderr, "%s: %s\n",
+	    message, session->strerror(session, error));
+	return (0);
+}
+
+static WT_EVENT_HANDLER event_handler = {
+	handle_error,
+	NULL,
+	NULL,
+	NULL
+};
+
+static void
+cursor_scope_ops(WT_SESSION *session, const char *uri)
+{
+	struct {
+		const char *op;
+		enum { INSERT, SEARCH, SEARCH_NEAR,
+		    REMOVE, REMOVE_POS, RESERVE, UPDATE } func;
+		const char *config;
+	} *op, ops[] = {
+		/*
+		 * The ops order is fixed and shouldn't change, that is, insert
+		 * has to happen first so search, update and remove operations
+		 * are possible, and remove has to be last.
+		 */
+		{ "insert", INSERT, NULL, },
+		{ "search", SEARCH, NULL, },
+		{ "search", SEARCH_NEAR, NULL, },
+#if 0
+		{ "reserve", RESERVE, NULL, },
+#endif
+		{ "update", UPDATE, NULL, },
+		{ "remove", REMOVE, NULL, },
+		{ "remove", REMOVE_POS, NULL, },
+		{ NULL, INSERT, NULL }
+	};
+	WT_CURSOR *cursor;
+	uint64_t keyr;
+	const char *key, *value;
+	char keybuf[100], valuebuf[100];
+	int exact;
+	bool recno;
+
+	/* Reserve requires a running transaction. */
+	testutil_check(session->begin_transaction(session, NULL));
+
+	cursor = NULL;
+	for (op = ops; op->op != NULL; op++) {
+		key = value = NULL;
+
+		/* Open a cursor. */
+		if (cursor != NULL)
+			testutil_check(cursor->close(cursor));
+		testutil_check(session->open_cursor(
+		    session, uri, NULL, op->config, &cursor));
+		recno = strcmp(cursor->key_format, "r") == 0;
+
+		/*
+		 * Set up application buffers so we can detect overwrites
+		 * or failure to copy application information into library
+		 * memory.
+		 */
+		if (recno)
+			cursor->set_key(cursor, (uint64_t)1);
+		else {
+			strcpy(keybuf, KEY);
+			cursor->set_key(cursor, keybuf);
+		}
+		strcpy(valuebuf, VALUE);
+		cursor->set_value(cursor, valuebuf);
+
+		/*
+		 * The application must keep key and value memory valid until
+		 * the next operation that positions the cursor, modifies the
+		 * data, or resets or closes the cursor.
+		 *
+		 * Modifying either the key or value buffers is not permitted.
+		 */
+		switch (op->func) {
+		case INSERT:
+			testutil_check(cursor->insert(cursor));
+			break;
+		case SEARCH:
+			testutil_check(cursor->search(cursor));
+			break;
+		case SEARCH_NEAR:
+			testutil_check(cursor->search_near(cursor, &exact));
+			break;
+		case REMOVE_POS:
+			/*
+			 * Remove has two modes, one where the remove is based
+			 * on a cursor position, the other where it's based on
+			 * a set key. The results are different, so test them
+			 * separately.
+			 */
+			testutil_check(cursor->search(cursor));
+			/* FALLTHROUGH */
+		case REMOVE:
+			testutil_check(cursor->remove(cursor));
+			break;
+		case RESERVE:
+#if 0
+			testutil_check(cursor->reserve(cursor));
+#endif
+			break;
+		case UPDATE:
+			testutil_check(cursor->update(cursor));
+			break;
+		}
+
+		/*
+		 * The cursor should no longer reference application memory,
+		 * and application buffers can be safely overwritten.
+		 */
+		memset(keybuf, 'K', sizeof(keybuf));
+		memset(valuebuf, 'V', sizeof(valuebuf));
+
+		/*
+		 * Check that get_key/get_value behave as expected after the
+		 * operation.
+		 */
+		switch (op->func) {
+		case INSERT:
+		case REMOVE:
+			/*
+			 * Insert and remove configured with a search key do
+			 * not position the cursor and have no key or value.
+			 *
+			 * There should be two error messages, ignore them.
+			 */
+			ignore_errors = 2;
+			if (recno)
+				testutil_assert(
+				    cursor->get_key(cursor, &keyr) != 0);
+			else
+				testutil_assert(
+				    cursor->get_key(cursor, &key) != 0);
+			testutil_assert(cursor->get_value(cursor, &value) != 0);
+			testutil_assert(ignore_errors == 0);
+			break;
+		case REMOVE_POS:
+			/*
+			 * Remove configured with a cursor position has a key,
+			 * but no value.
+			 *
+			 * There should be one error message, ignore it.
+			 */
+			if (recno) {
+				testutil_assert(
+				    cursor->get_key(cursor, &keyr) == 0);
+				testutil_assert(keyr == 1);
+			} else {
+				testutil_assert(
+				    cursor->get_key(cursor, &key) == 0);
+				testutil_assert(key != keybuf);
+				testutil_assert(strcmp(key, KEY) == 0);
+			}
+			ignore_errors = 1;
+			testutil_assert(cursor->get_value(cursor, &value) != 0);
+			testutil_assert(ignore_errors == 0);
+			break;
+		case RESERVE:
+		case SEARCH:
+		case SEARCH_NEAR:
+		case UPDATE:
+			/*
+			 * Reserve, search, search-near and update position the
+			 * cursor and have both a key and value.
+			 *
+			 * Any key/value should not reference application
+			 * memory.
+			 */
+			if (recno) {
+				testutil_assert(
+				    cursor->get_key(cursor, &keyr) == 0);
+				testutil_assert(keyr == 1);
+			} else {
+				testutil_assert(
+				    cursor->get_key(cursor, &key) == 0);
+				testutil_assert(key != keybuf);
+				testutil_assert(strcmp(key, KEY) == 0);
+			}
+			testutil_assert(cursor->get_value(cursor, &value) == 0);
+			testutil_assert(value != valuebuf);
+			testutil_assert(strcmp(value, VALUE) == 0);
+			break;
+		}
+
+		/*
+		 * We have more than one remove operation, add the key back
+		 * in.
+		 */
+		if (op->func == REMOVE || op->func == REMOVE_POS) {
+			if (recno)
+				cursor->set_key(cursor, (uint64_t)1);
+			else {
+				cursor->set_key(cursor, KEY);
+			}
+			cursor->set_value(cursor, VALUE);
+			testutil_check(cursor->insert(cursor));
+		}
+	}
+}
+
+static void
+run(WT_CONNECTION *conn, const char *uri, const char *config)
+{
+	WT_SESSION *session;
+
+	testutil_check(conn->open_session(conn, NULL, NULL, &session));
+	testutil_check(session->create(session, uri, config));
+	cursor_scope_ops(session, uri);
+	testutil_check(session->close(session, NULL));
+}
+
+int
+main(int argc, char *argv[])
+{
+	TEST_OPTS *opts, _opts;
+
+	opts = &_opts;
+	memset(opts, 0, sizeof(*opts));
+	testutil_check(testutil_parse_opts(argc, argv, opts));
+	testutil_make_work_dir(opts->home);
+
+	testutil_check(
+	    wiredtiger_open(opts->home, &event_handler, "create", &opts->conn));
+
+	run(opts->conn, "file:file.SS", "key_format=S,value_format=S");
+	run(opts->conn, "file:file.rS", "key_format=r,value_format=S");
+	run(opts->conn, "lsm:lsm.SS", "key_format=S,value_format=S");
+	run(opts->conn, "lsm:lsm.rS", "key_format=r,value_format=S");
+	run(opts->conn, "table:table.SS", "key_format=S,value_format=S");
+	run(opts->conn, "table:table.rS", "key_format=r,value_format=S");
+
+	testutil_cleanup(opts);
+
+	return (EXIT_SUCCESS);
+}