Import wiredtiger: 911c940adab547d36ac305fc627a79e637fa3c40 from branch mongodb-3.2r3.2.10-rc0

ref: dddca65..911c940ada
for: 3.2.10

SERVER-24971  Excessive memory held by sessions when application threads do evictions
SERVER-25843  Coverity analysis defect 99856: Redundant test
SERVER-25845  Coverity analysis defect 99859: Explicit null dereferenced
SERVER-25846  Coverity analysis defect 99861: Dereference after null check
WT-1162       Add latency to Jenkins wtperf tests and plots
WT-2026       Maximum pages size at eviction too large
WT-2221       Document which statistics are available via a "fast" configuration vs. an "all" configuration
WT-2233       Investigate changing when the eviction server switches to aggressive mode.
WT-2239       Make sure LSM cursors read up to date dsk_gen, it was racing with compact
WT-2323       Allocate a transaction id at the beginning of join cursor iteration
WT-2353       Failure to create async threads as part of a wiredtiger_open call will cause a hang
WT-2380       Make scripts fail if code doesn't match style
WT-2486       Update make check so that it runs faster
WT-2555       make format run on Windows
WT-2578       remove write barriers from the TAILQ_INSERT_XXX macros
WT-2631       nullptr is passed for parameters marked with attribute non-null
WT-2638       ftruncate may not be supported
WT-2645       wt dump: push the complexity of collecting metadata into a dump cursor
WT-2648       cache-line alignment for new ports
WT-2665       Limit allocator fragmentation in WiredTiger
WT-2678       The metadata should not imply that an empty value is true
WT-2688       configure --enable-python doesn't check for availability of swig
WT-2693       Check open_cursor error paths for consistent handling
WT-2695       Integrate s390x accelerated crc32c support
WT-2708       split child-update race with reconciliation/eviction
WT-2711       Change statistics log configuration options
WT-2719       add fuzz testing for WiredTiger options and reconfiguration.
WT-2728       Don't re-read log file headers during log_flush
WT-2729       Focus eviction walks in largest trees
WT-2730       cursor next/prev can return the wrong key/value pair when crossing a page boundary
WT-2731       Raw compression can create pages that are larger than expected
WT-2732       Coverity analysis defect 99665: Redundant test
WT-2734       Improve documentation of eviction behavior
WT-2737       Scrub dirty pages rather than evicting them
WT-2738       Remove the ability to change the default checkpoint name
WT-2739       pluggable file systems documentation cleanups
WT-2743       Thread count statistics always report 0
WT-2744       partial line even with line buffering set
WT-2746       track checkpoint I/O separately from eviction I/O
WT-2751       column-store statistics incorrectly calculates the number of entries
WT-2752       Fixes to zipfian wtperf workload config
WT-2755       flexelint configuration treats size_t as 4B type
WT-2756       Upgrade the autoconf archive package to check for swig 3.0
WT-2757       Column tables behave differently when column names are provided
WT-2759       Releasing the hot-backup lock doesn't require the schema lock.
WT-2760       Fix a bug in backup related to directory sync. Change the filesystem API to make durable the default
WT-2762       wtstats tool fails if checkpoint runs
WT-2763       Unit test test_intpack failing on OSX
WT-2764       Optimize checkpoints to reduce throughput disruption
WT-2765       wt dump: indices need to be shown in the dump output
WT-2766       Don't count eviction of lookaside file pages for the purpose of checking stuck cache
WT-2767       test suite needs way to run an individual scenario
WT-2769       Update documentation to reflect correct limits of memory_page_max
WT-2770       Add statistics tracking schema operations
WT-2772       Investigate log performance testing weirdness
WT-2773       search_near in indexes does not find exact matches
WT-2774       minor cleanups/improvements
WT-2778       Python test suite: make scenario initialization consistent
WT-2779       Raw compression created unexpectedly large pages on disk
WT-2781       Enhance bulk cursor option with an option to return immediately on contention
WT-2782       Missing a fs_directory_list_free in ex_file_system.c
WT-2783       wtperf multi-btree.wtperf dumps core on Mac
WT-2785       Scrub dirty pages rather than evicting them: single-page reconciliation
WT-2787       Include src/include/wiredtiger_ext.h is problematic
WT-2788       Java: freed memory overwrite during handle close can cause JNI crash
WT-2791       Enhance OS X Evergreen unit test
WT-2793       wtperf config improvements
WT-2795       Update documentation around read-only configuration
WT-2796       Memory leak in reconciliation uncovered by stress testing
WT-2798       Crash vulnerability with nojournal after create during checkpoint
WT-2800       Illegal file format in test/format on PPC
WT-2801       Crash vulnerability from eviction of metadata during checkpoint
WT-2802       Transaction commit causes heap-use-after free
WT-2803       Add verbose functionality to WT Evergreen tests
WT-2804       Don't read values in a tree without a snapshot
WT-2805       Infinite recursion if error streams fail
WT-2806       wtperf allocation size off-by-one
WT-2807       Switch Jenkins performance tests to tcmalloc
WT-2811       Reconciliation asserts that transaction time has gone backwards
WT-2812       Error when reconfiguring cache targets
WT-2813       small cache usage stuck even with large cache
WT-2814       Enhance wtperf to support single-op truncate mode
WT-2816       Improve WiredTiger eviction performance
WT-2817       Investigate performance regression in develop, add workload to wtperf/runners
WT-2818       The page visibility check when queuing pages for eviction is overly restrictive
WT-2820       add gcc warn_unused_result attribute
WT-2822       panic mutex and other functions that cannot fail
WT-2823       support file handles without a truncate method
WT-2824       wtperf displays connection and table create configurations twice
WT-2826       clang38 false positive on uninitialized variable.
WT-2827       checkpoint log_size configuration improvements
WT-2828       Make long wtperf tests reflect mongoDB usage
WT-2829       Switch automated testing to use enable-strict configure option
WT-2832       Python test uses hard-coded temporary directory
WT-2834       Join cursor: discrepancy with bloom filters
WT-2835       WT_CONNECTION.leak-memory can skip memory map and cache cleanup
WT-2838       Don't free session handles on close if leak memory is configured
WT-2839       lint: Ignoring return value of function
WT-2840       clang analysis: garbage values
WT-2841       Jenkins Valgrind runner is reporting errors in test wt2719_reconfig
WT-2842       split wtperf's configuration into per-database and per-run parts
WT-2843       Fix a bug in recovery if there is no filesystem truncate support
WT-2846       Several bugs related to reconfiguring eviction server at runtime
WT-2847       Merge fair locks into read/write locks.
WT-2850       clang 4.1 attribute warnings when building
WT-2853       Multi threaded reader writer example shows temporary slowdown or lockup
WT-2857       POSIX ftruncate calls should be #ifdef'd HAVE_FTRUNCATE
WT-2862       Fix lint error in test case for forced eviction with multiple cursors
WT-2863       Support UTF-8 paths on Windows
WT-2865       eviction thread error failure
WT-2866       Eviction server algorithm tuning
WT-2867       Review and fix barrier usage in __lsm_tree_close
WT-2868       Add sample_interval to checkpoint-stress wtperf config
WT-2869       Performance regression on secondaries
WT-2870       Rename wtperf checkpoint schema jobs
WT-2871       __wt_verbose has the wrong GCC format attributes
WT-2872       Recent stuck cache test/stress failures.
WT-2873       Refactor CRC32 code
WT-2875       Test test_wt2853_perf can run too long under valgrind
WT-2876       Extend wtperf to support a log like table
WT-2878       Verbose changes affected performance
WT-2881       Add -Wpedantic to clang compiler warning flags
WT-2883       wiredtiger_open with verbose=handleops recursive loop
WT-2885       __wt_checkpoint_signal lint
WT-2886       Decide how in-memory configuration and eviction_dirty_target interact
WT-2888       Switch functions to return void where possible
WT-2892       hot backup can race with block truncate
WT-2896       Coverity #1362535: resource leak
WT-2897       Checkpoints can become corrupted on failure
WT-2901       Add option to disable checkpoint dirty stepdown phase
WT-2903       Reduce the impact of checkpoint scrubbing on applications

1 files changed, 402 insertions, 0 deletions

diff --git a/src/third_party/wiredtiger/test/csuite/wt2323_join_visibility/main.c b/src/third_party/wiredtiger/test/csuite/wt2323_join_visibility/main.c
new file mode 100644
index 00000000000..bbf1626fe82
--- /dev/null
+++ b/src/third_party/wiredtiger/test/csuite/wt2323_join_visibility/main.c
@@ -0,0 +1,402 @@
+/*-
+ * Public Domain 2014-2016 MongoDB, Inc.
+ * Public Domain 2008-2014 WiredTiger, Inc.
+ *
+ * This is free and unencumbered software released into the public domain.
+ *
+ * Anyone is free to copy, modify, publish, use, compile, sell, or
+ * distribute this software, either in source code form or as a compiled
+ * binary, for any purpose, commercial or non-commercial, and by any
+ * means.
+ *
+ * In jurisdictions that recognize copyright laws, the author or authors
+ * of this software dedicate any and all copyright interest in the
+ * software to the public domain. We make this dedication for the benefit
+ * of the public at large and to the detriment of our heirs and
+ * successors. We intend this dedication to be an overt act of
+ * relinquishment in perpetuity of all present and future rights to this
+ * software under copyright law.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "test_util.h"
+
+/*
+ * JIRA ticket reference: WT-2323
+ *
+ * Test case description: We create two kinds of threads that race: One kind
+ * is populating/updating records in a table with a few indices, the other
+ * is reading from a join of that table. The hope in constructing this test
+ * was to have the updates interleaved between reads of multiple indices by
+ * the join, yielding an inconsistent view of the data.  In the main table,
+ * we insert account records, with a positive or negative balance.  The
+ * negative balance accounts always have a flag set to non-zero, positive
+ * balances have the flag set to zero.  The join we do is:
+ *
+ *   select (*) from account where account.postal_code = '54321' and
+ *                      account.balance < 0 and account.flags == 0
+ *
+ * which should always yield no results.
+ *
+ * Failure mode: This test never actually failed with any combination of
+ * parameters, with N_INSERT up to 50000000.  It seems that a snapshot is
+ * implicitly allocated in the session used by a join by the set_key calls
+ * that occur before the first 'next' of the join cursor is done.  Despite
+ * that, the test seems interesting enough to keep around, with the number
+ * of inserts set low as a default.
+ */
+
+void (*custom_die)(void) = NULL;
+
+#define	N_RECORDS	10000
+#define	N_INSERT	500000
+#define	N_INSERT_THREAD	2
+#define	N_JOIN_THREAD	2
+#define	S64 "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789::"
+#define	S1024 (S64 S64 S64 S64 S64 S64 S64 S64 S64 S64 S64 S64 S64 S64 S64 S64)
+
+typedef struct {
+	char posturi[256];
+	char baluri[256];
+	char flaguri[256];
+	char joinuri[256];
+	bool bloom;
+	bool remove;
+} SHARED_OPTS;
+
+typedef struct {
+	TEST_OPTS *testopts;
+	SHARED_OPTS *sharedopts;
+	int threadnum;
+	int nthread;
+	int done;
+	int joins;
+	int removes;
+	int inserts;
+	int notfounds;
+	int rollbacks;
+} THREAD_ARGS;
+
+static void *thread_insert(void *);
+static void *thread_join(void *);
+static void test_join(TEST_OPTS *, SHARED_OPTS *, bool, bool);
+
+int
+main(int argc, char *argv[])
+{
+	SHARED_OPTS *sharedopts, _sharedopts;
+	TEST_OPTS *opts, _opts;
+	const char *tablename;
+
+	opts = &_opts;
+	sharedopts = &_sharedopts;
+	memset(opts, 0, sizeof(*opts));
+	memset(sharedopts, 0, sizeof(*sharedopts));
+
+	testutil_check(testutil_parse_opts(argc, argv, opts));
+	testutil_make_work_dir(opts->home);
+
+	tablename = strchr(opts->uri, ':');
+	testutil_assert(tablename != NULL);
+	tablename++;
+	snprintf(sharedopts->posturi, sizeof(sharedopts->posturi),
+	    "index:%s:post", tablename);
+	snprintf(sharedopts->baluri, sizeof(sharedopts->baluri),
+	    "index:%s:bal", tablename);
+	snprintf(sharedopts->flaguri, sizeof(sharedopts->flaguri),
+	    "index:%s:flag", tablename);
+	snprintf(sharedopts->joinuri, sizeof(sharedopts->joinuri),
+	    "join:%s", opts->uri);
+
+	testutil_check(wiredtiger_open(opts->home, NULL,
+	    "create,cache_size=1G", &opts->conn));
+
+	test_join(opts, sharedopts, true, true);
+	test_join(opts, sharedopts, true, false);
+	test_join(opts, sharedopts, false, true);
+	test_join(opts, sharedopts, false, false);
+
+	testutil_cleanup(opts);
+
+	return (0);
+}
+
+static void
+test_join(TEST_OPTS *opts, SHARED_OPTS *sharedopts, bool bloom,
+    bool sometimes_remove)
+{
+	THREAD_ARGS insert_args[N_INSERT_THREAD], join_args[N_JOIN_THREAD];
+	WT_CURSOR *maincur;
+	WT_SESSION *session;
+	pthread_t insert_tid[N_INSERT_THREAD], join_tid[N_JOIN_THREAD];
+	int i;
+
+	memset(insert_args, 0, sizeof(insert_args));
+	memset(join_args, 0, sizeof(join_args));
+
+	sharedopts->bloom = bloom;
+	sharedopts->remove = sometimes_remove;
+
+	fprintf(stderr, "Running with bloom=%d, remove=%d\n",
+	    (int)bloom, (int)sometimes_remove);
+
+	testutil_check(
+	    opts->conn->open_session(opts->conn, NULL, NULL, &session));
+
+	/*
+	 * Note: id is repeated as id2.  This makes it easier to
+	 * identify the primary key in dumps of the index files.
+	 */
+	testutil_check(session->create(session, opts->uri,
+	    "key_format=i,value_format=iiSii,"
+	    "columns=(id,post,bal,extra,flag,id2)"));
+
+	testutil_check(session->create(session, sharedopts->posturi,
+	    "columns=(post)"));
+	testutil_check(session->create(session, sharedopts->baluri,
+	    "columns=(bal)"));
+	testutil_check(session->create(session, sharedopts->flaguri,
+	    "columns=(flag)"));
+
+	/*
+	 * Insert a single record with all items we need to
+	 * call search() on, this makes our join logic easier.
+	 */
+	testutil_check(session->open_cursor(session, opts->uri, NULL, NULL,
+	    &maincur));
+	maincur->set_key(maincur, N_RECORDS);
+	maincur->set_value(maincur, 54321, 0, "", 0, N_RECORDS);
+	testutil_check(maincur->insert(maincur));
+	testutil_check(maincur->close(maincur));
+
+	for (i = 0; i < N_INSERT_THREAD; ++i) {
+		insert_args[i].threadnum = i;
+		insert_args[i].nthread = N_INSERT_THREAD;
+		insert_args[i].testopts = opts;
+		insert_args[i].sharedopts = sharedopts;
+		testutil_check(pthread_create(&insert_tid[i], NULL,
+		    thread_insert, (void *)&insert_args[i]));
+	}
+
+	for (i = 0; i < N_JOIN_THREAD; ++i) {
+		join_args[i].threadnum = i;
+		join_args[i].nthread = N_JOIN_THREAD;
+		join_args[i].testopts = opts;
+		join_args[i].sharedopts = sharedopts;
+		testutil_check(pthread_create(&join_tid[i], NULL,
+		    thread_join, (void *)&join_args[i]));
+	}
+
+	/*
+	 * Wait for insert threads to finish.  When they
+	 * are done, signal join threads to complete.
+	 */
+	for (i = 0; i < N_INSERT_THREAD; ++i)
+		testutil_check(pthread_join(insert_tid[i], NULL));
+
+	for (i = 0; i < N_JOIN_THREAD; ++i)
+		join_args[i].done = 1;
+
+	for (i = 0; i < N_JOIN_THREAD; ++i)
+		testutil_check(pthread_join(join_tid[i], NULL));
+
+	fprintf(stderr, "\n");
+	for (i = 0; i < N_JOIN_THREAD; ++i) {
+		fprintf(stderr, "  join thread %d did %d joins\n",
+		    i, join_args[i].joins);
+	}
+	for (i = 0; i < N_INSERT_THREAD; ++i)
+		fprintf(stderr,
+		    "  insert thread %d did "
+		    "%d inserts, %d removes, %d notfound, %d rollbacks\n",
+		    i, insert_args[i].inserts, insert_args[i].removes,
+		    insert_args[i].notfounds, insert_args[i].rollbacks);
+
+	testutil_check(session->drop(session, sharedopts->posturi, NULL));
+	testutil_check(session->drop(session, sharedopts->baluri, NULL));
+	testutil_check(session->drop(session, sharedopts->flaguri, NULL));
+	testutil_check(session->drop(session, opts->uri, NULL));
+	testutil_check(session->close(session, NULL));
+}
+
+static void *thread_insert(void *arg)
+{
+	SHARED_OPTS *sharedopts;
+	TEST_OPTS *opts;
+	THREAD_ARGS *threadargs;
+	WT_CURSOR *maincur;
+	WT_RAND_STATE rnd;
+	WT_SESSION *session;
+	int bal, i, flag, key, post, ret;
+	const char *extra = S1024;
+
+	threadargs = (THREAD_ARGS *)arg;
+	opts = threadargs->testopts;
+	sharedopts = threadargs->sharedopts;
+	testutil_check(__wt_random_init_seed(NULL, &rnd));
+
+	testutil_check(opts->conn->open_session(
+	    opts->conn, NULL, NULL, &session));
+
+	testutil_check(session->open_cursor(session, opts->uri, NULL, NULL,
+	    &maincur));
+
+	for (i = 0; i < N_INSERT; i++) {
+		/*
+		 * Insert threads may stomp on each other's records;
+		 * that's okay.
+		 */
+		key = (int)(__wt_random(&rnd) % N_RECORDS);
+		maincur->set_key(maincur, key);
+		if (sharedopts->remove)
+			testutil_check(session->begin_transaction(session,
+			    "isolation=snapshot"));
+		if (sharedopts->remove && __wt_random(&rnd) % 5 == 0 &&
+		    maincur->search(maincur) == 0) {
+			/*
+			 * Another thread can be removing at the
+			 * same time.
+			 */
+			ret = maincur->remove(maincur);
+			testutil_assert(ret == 0 ||
+			    (N_INSERT_THREAD > 1 &&
+			    (ret == WT_NOTFOUND || ret == WT_ROLLBACK)));
+			if (ret == 0)
+				threadargs->removes++;
+			else if (ret == WT_NOTFOUND)
+				threadargs->notfounds++;
+			else if (ret == WT_ROLLBACK)
+				threadargs->rollbacks++;
+		} else {
+			if (__wt_random(&rnd) % 2 == 0)
+				post = 54321;
+			else
+				post = i % 100000;
+			if (__wt_random(&rnd) % 2 == 0) {
+				bal = -100;
+				flag = 1;
+			} else {
+				bal = 1 + (i % 1000) * 100;
+				flag = 0;
+			}
+			maincur->set_value(maincur, post, bal, extra, flag,
+			    key);
+			ret = maincur->insert(maincur);
+			testutil_assert(ret == 0 ||
+			    (N_INSERT_THREAD > 1 && ret == WT_ROLLBACK));
+			testutil_check(maincur->reset(maincur));
+			if (ret == 0)
+				threadargs->inserts++;
+			else if (ret == WT_ROLLBACK)
+				threadargs->rollbacks++;
+		}
+		if (sharedopts->remove)
+			testutil_check(session->commit_transaction(session,
+			    NULL));
+		if (i % 1000 == 0 && i != 0) {
+			if (i % 10000 == 0)
+				fprintf(stderr, "*");
+			else
+				fprintf(stderr, ".");
+		}
+	}
+	testutil_check(maincur->close(maincur));
+	testutil_check(session->close(session, NULL));
+	return (NULL);
+}
+
+static void *thread_join(void *arg)
+{
+	SHARED_OPTS *sharedopts;
+	TEST_OPTS *opts;
+	THREAD_ARGS *threadargs;
+	WT_CURSOR *postcur, *balcur, *flagcur, *joincur;
+	WT_SESSION *session;
+	int bal, flag, key, key2, post, ret;
+	char cfg[128];
+	char *extra;
+
+	threadargs = (THREAD_ARGS *)arg;
+	opts = threadargs->testopts;
+	sharedopts = threadargs->sharedopts;
+
+	testutil_check(opts->conn->open_session(
+	    opts->conn, NULL, NULL, &session));
+
+	testutil_check(session->open_cursor(
+	    session, sharedopts->posturi, NULL, NULL, &postcur));
+	testutil_check(session->open_cursor(
+	    session, sharedopts->baluri, NULL, NULL, &balcur));
+	testutil_check(session->open_cursor(
+	    session, sharedopts->flaguri, NULL, NULL, &flagcur));
+
+	for (threadargs->joins = 0; threadargs->done == 0;
+	     threadargs->joins++) {
+		testutil_check(session->open_cursor(
+		    session, sharedopts->joinuri, NULL, NULL, &joincur));
+		postcur->set_key(postcur, 54321);
+		testutil_check(postcur->search(postcur));
+		testutil_check(session->join(session, joincur, postcur,
+		    "compare=eq"));
+
+		balcur->set_key(balcur, 0);
+		testutil_check(balcur->search(balcur));
+		if (sharedopts->bloom)
+			sprintf(cfg, "compare=lt,strategy=bloom,count=%d",
+			    N_RECORDS);
+		else
+			sprintf(cfg, "compare=lt");
+		testutil_check(session->join(session, joincur, balcur, cfg));
+
+		flagcur->set_key(flagcur, 0);
+		testutil_check(flagcur->search(flagcur));
+		if (sharedopts->bloom)
+			sprintf(cfg, "compare=eq,strategy=bloom,count=%d",
+			    N_RECORDS);
+		else
+			sprintf(cfg, "compare=eq");
+		testutil_check(session->join(session, joincur, flagcur, cfg));
+
+		/* Expect no values returned */
+		ret = joincur->next(joincur);
+		if (ret == 0) {
+			/*
+			 * The values may already have been changed, but
+			 * print them for informational purposes.
+			 */
+			testutil_check(joincur->get_key(joincur, &key));
+			testutil_check(joincur->get_value(joincur, &post,
+			    &bal, &extra, &flag, &key2));
+			fprintf(stderr, "FAIL: iteration %d: "
+			    "key=%d/%d, postal_code=%d, balance=%d, flag=%d\n",
+			    threadargs->joins, key, key2, post, bal, flag);
+			/* Save the results. */
+			testutil_check(opts->conn->close(opts->conn, NULL));
+			opts->conn = NULL;
+			return (NULL);
+		}
+		testutil_assert(ret == WT_NOTFOUND);
+		testutil_check(joincur->close(joincur));
+
+		/*
+		 * Reset the cursors, potentially allowing the insert
+		 * threads to proceed.
+		 */
+		testutil_check(postcur->reset(postcur));
+		testutil_check(balcur->reset(balcur));
+		testutil_check(flagcur->reset(flagcur));
+		if (threadargs->joins % 100 == 0)
+			fprintf(stderr, "J");
+	}
+	testutil_check(postcur->close(postcur));
+	testutil_check(balcur->close(balcur));
+	testutil_check(flagcur->close(flagcur));
+	testutil_check(session->close(session, NULL));
+	return (NULL);
+}