Import wiredtiger-wiredtiger-2.7.0-559-g07966a4.tar.gz from wiredtiger branch mongodb-3.2

ref: 3c2ad56..07966a4

WT-1517       schema format edge cases
WT-1801       Add a directory sync after rollback of a WT_SESSION::rename operation
WT-2060       Simplify aggregation of statistics
WT-2073       metadata cleanups
WT-2099       Seeing memory underflow messages
WT-2113       truncate01 sometimes fails
WT-2142       Connection cleanup in Python tests
WT-2177       Add an optional per-thread seed to random number generator
WT-2198       bulk load and column store appends
WT-2216       simplify row-store search loop slightly
WT-2225       New split code performance impact
WT-2231       pinned page cursor searches could check parent keys
WT-2235       wt printlog option without unicode
WT-2242       WiredTiger treats dead trees the same as other trees in eviction
WT-2244       Trigger in-memory splits sooner
WT-2245       WTPERF Truncate has no ability to catch up when it falls behind
WT-2246       column-store append searches the leaf page; the maximum record number fails CRUD operations
WT-2247       variable-length column-store in-memory page splits
WT-2256       WTPERFs throttle option fires in bursts
WT-2257       wtperf doesn't handle overriding workload config
WT-2258       WiredTiger preloads pages even when direct-IO is configured.
WT-2259       __wt_evict_file_exclusive_on() should clear WT_BTREE_NO_EVICTION on error
WT-2260       Workloads evict internal pages unexpectedly
WT-2262       Random sampling is skewed by tree shape
WT-2265       Wiredtiger related change in ppc64le specific code block in gcc.h
WT-2266       Add wtperf config to set if perf thresholds are fatal
WT-2267       Improve wtperf throttling implementation to provide steady load
WT-2269       wtperf should dump its config everytime it runs
WT-2272       Stress test assertion in the sweep server
WT-2275       broken DB after application crash
WT-2276       tool to decode checkpoint addr
WT-2277       Remove WT check against big-endian systems
WT-2279       Define WT_PAUSE(), WT_FULL_BARRIER(), etc when s390x is defined
WT-2281       wtperf smoke.sh fails on ppc64le
WT-2282       error in wt_txn_update_oldest verbose message test
WT-2283       retry in txn_update_oldest results in a hang
WT-2284       Repeated macro definition
WT-2285       configure should set BUFFER_ALIGNMENT_DEFAULT to 4kb on linux
WT-2287       WT_SESSION.rebalance
WT-2289       failure in fast key check
WT-2290       WT_SESSION.compact could be more effective.
WT-2291       Random cursor walk inefficient in skip list only trees
WT-2295       WT_SESSION.create does a full-scan of the main table
WT-2296       New log algorithm needs improving for sync/flush settings
WT-2297       Fix off-by-one error in Huffman config file parsing
WT-2299       upper-level WiredTiger code is reaching into the block manager
WT-2301       Add reading a range to wtperf
WT-2303       Build warning in wtperf
WT-2304       wtperf crash dumping config
WT-2305       Fix coverity scan issues on 23/12/2015
WT-2307       Internal page splits can corrupt cursor iteration
WT-2308       custom extractor for ref_cursors in join cursor
WT-2311       Support Sparc
WT-2312       re-creating a deleted column-store page can corrupt the in-memory tree
WT-2313       sweep-server: conn_dhandle.c, 610: dhandle != conn->cache->evict_file_next
WT-2314       page-swap error handling is inconsistent
WT-2316       stress test failure: WT_CURSOR.prev out-of-order returns
WT-2320       Only check copyright when cutting releases
WT-2321       WT-2321: race between eviction and worker threads on the eviction queue
WT-2326       Change WTPERF to use new memory allocation functions instead of the standard
WT-2328       schema drop does direct unlink, it should use a block manager interface.
WT-2331       Checking of search() result for reference cursors before join()
WT-2332       Bug in logging write-no-sync mode
WT-2333       Add a flag so drop doesn't block
WT-2335       NULL pointer crash in config_check_search with invalid configuration string
WT-2338       Disable using pre-allocated log files when backup cursor is open
WT-2339       format post-rebalance verify failure (stress run #11586)
WT-2340       Add logging guarantee assertions, whitespace
WT-2342       Enhance wtperf to support background create and drop operations
WT-2344       OS X compiler warning
WT-2347       Java: schema format edge cases
WT-2348       xargs -P isn't portable
WT-2355       Fix minor scratch buffer usage in logging
SERVER-21833  Compact does not release space to the system with WiredTiger
SERVER-21887  $sample takes disproportionately long time on newly created collection
SERVER-22064  Coverity analysis defect 77699: Unchecked return value
SERVER-21944  WiredTiger changes for 3.2.2

1 files changed, 356 insertions, 0 deletions

diff --git a/src/third_party/wiredtiger/test/thread/rw.c b/src/third_party/wiredtiger/test/thread/rw.c
new file mode 100644
index 00000000000..913fa6e6c25
--- /dev/null
+++ b/src/third_party/wiredtiger/test/thread/rw.c
@@ -0,0 +1,356 @@
+/*-
+ * Public Domain 2014-2016 MongoDB, Inc.
+ * Public Domain 2008-2014 WiredTiger, Inc.
+ *
+ * This is free and unencumbered software released into the public domain.
+ *
+ * Anyone is free to copy, modify, publish, use, compile, sell, or
+ * distribute this software, either in source code form or as a compiled
+ * binary, for any purpose, commercial or non-commercial, and by any
+ * means.
+ *
+ * In jurisdictions that recognize copyright laws, the author or authors
+ * of this software dedicate any and all copyright interest in the
+ * software to the public domain. We make this dedication for the benefit
+ * of the public at large and to the detriment of our heirs and
+ * successors. We intend this dedication to be an overt act of
+ * relinquishment in perpetuity of all present and future rights to this
+ * software under copyright law.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "thread.h"
+
+static void  print_stats(u_int);
+static void *reader(void *);
+static void *writer(void *);
+
+typedef struct {
+	char *name;				/* object name */
+	u_int nops;				/* Thread op count */
+
+	WT_RAND_STATE rnd;			/* RNG */
+
+	int remove;				/* cursor.remove */
+	int update;				/* cursor.update */
+	int reads;				/* cursor.search */
+} INFO;
+
+static INFO *run_info;
+
+int
+rw_start(u_int readers, u_int writers)
+{
+	struct timeval start, stop;
+	double seconds;
+	pthread_t *tids;
+	u_int i, name_index, offset, total_nops;
+	int ret;
+	void *thread_ret;
+
+	tids = NULL;	/* Keep GCC 4.1 happy. */
+	total_nops = 0;
+
+	/* Create per-thread structures. */
+	if ((run_info = calloc(
+	    (size_t)(readers + writers), sizeof(*run_info))) == NULL ||
+	    (tids = calloc((size_t)(readers + writers), sizeof(*tids))) == NULL)
+		testutil_die(errno, "calloc");
+
+	/* Create the files and load the initial records. */
+	for (i = 0; i < writers; ++i) {
+		if (i == 0 || multiple_files) {
+			if ((run_info[i].name = malloc(64)) == NULL)
+				testutil_die(errno, "malloc");
+			snprintf(run_info[i].name, 64, FNAME, i);
+
+			/* Vary by orders of magnitude */
+			if (vary_nops)
+				run_info[i].nops = WT_MAX(1000, max_nops >> i);
+			load(run_info[i].name);
+		} else
+			run_info[i].name = run_info[0].name;
+
+		/* Setup op count if not varying ops. */
+		if (run_info[i].nops == 0)
+			run_info[i].nops = max_nops;
+		total_nops += run_info[i].nops;
+	}
+
+	/* Setup the reader configurations */
+	for (i = 0; i < readers; ++i) {
+		offset = i + writers;
+		if (multiple_files) {
+			if ((run_info[offset].name = malloc(64)) == NULL)
+				testutil_die(errno, "malloc");
+			/* Have readers read from tables with writes. */
+			name_index = i % writers;
+			snprintf(
+			    run_info[offset].name, 64, FNAME, name_index);
+
+			/* Vary by orders of magnitude */
+			if (vary_nops)
+				run_info[offset].nops =
+				    WT_MAX(1000, max_nops >> name_index);
+		} else
+			run_info[offset].name = run_info[0].name;
+
+		/* Setup op count if not varying ops. */
+		if (run_info[offset].nops == 0)
+			run_info[offset].nops = max_nops;
+		total_nops += run_info[offset].nops;
+	}
+
+	(void)gettimeofday(&start, NULL);
+
+	/* Create threads. */
+	for (i = 0; i < readers; ++i)
+		if ((ret = pthread_create(
+		    &tids[i], NULL, reader, (void *)(uintptr_t)i)) != 0)
+			testutil_die(ret, "pthread_create");
+	for (; i < readers + writers; ++i) {
+		if ((ret = pthread_create(
+		    &tids[i], NULL, writer, (void *)(uintptr_t)i)) != 0)
+			testutil_die(ret, "pthread_create");
+	}
+
+	/* Wait for the threads. */
+	for (i = 0; i < readers + writers; ++i)
+		(void)pthread_join(tids[i], &thread_ret);
+
+	(void)gettimeofday(&stop, NULL);
+	seconds = (stop.tv_sec - start.tv_sec) +
+	    (stop.tv_usec - start.tv_usec) * 1e-6;
+	fprintf(stderr, "timer: %.2lf seconds (%d ops/second)\n",
+	    seconds, (int)(((readers + writers) * total_nops) / seconds));
+
+	/* Verify the files. */
+	for (i = 0; i < readers + writers; ++i) {
+		verify(run_info[i].name);
+		if (!multiple_files)
+			break;
+	}
+
+	/* Output run statistics. */
+	print_stats(readers + writers);
+
+	/* Free allocated memory. */
+	for (i = 0; i < readers + writers; ++i) {
+		free(run_info[i].name);
+		if (!multiple_files)
+			break;
+	}
+
+	free(run_info);
+	free(tids);
+
+	return (0);
+}
+
+/*
+ * reader_op --
+ *	Read operation.
+ */
+static inline void
+reader_op(WT_SESSION *session, WT_CURSOR *cursor, INFO *s)
+{
+	WT_ITEM *key, _key;
+	u_int keyno;
+	int ret;
+	char keybuf[64];
+
+	key = &_key;
+
+	keyno = __wt_random(&s->rnd) % nkeys + 1;
+	if (ftype == ROW) {
+		key->data = keybuf;
+		key->size = (uint32_t)
+		    snprintf(keybuf, sizeof(keybuf), "%017u", keyno);
+		cursor->set_key(cursor, key);
+	} else
+		cursor->set_key(cursor, (uint32_t)keyno);
+	if ((ret = cursor->search(cursor)) != 0 && ret != WT_NOTFOUND)
+		testutil_die(ret, "cursor.search");
+	if (log_print)
+		(void)session->log_printf(session,
+		    "Reader Thread %p key %017u", pthread_self(), keyno);
+}
+
+/*
+ * reader --
+ *	Reader thread start function.
+ */
+static void *
+reader(void *arg)
+{
+	INFO *s;
+	WT_CURSOR *cursor;
+	WT_SESSION *session;
+	u_int i;
+	int id, ret;
+	char tid[128];
+
+	id = (int)(uintptr_t)arg;
+	s = &run_info[id];
+	__wt_thread_id(tid, sizeof(tid));
+	__wt_random_init(&s->rnd);
+
+	printf(" read thread %2d starting: tid: %s, file: %s\n",
+	    id, tid, s->name);
+
+	__wt_yield();		/* Get all the threads created. */
+
+	if (session_per_op) {
+		for (i = 0; i < s->nops; ++i, ++s->reads, __wt_yield()) {
+			if ((ret = conn->open_session(
+			    conn, NULL, NULL, &session)) != 0)
+				testutil_die(ret, "conn.open_session");
+			if ((ret = session->open_cursor(
+			    session, s->name, NULL, NULL, &cursor)) != 0)
+				testutil_die(ret, "session.open_cursor");
+			reader_op(session, cursor, s);
+			if ((ret = session->close(session, NULL)) != 0)
+				testutil_die(ret, "session.close");
+		}
+	} else {
+		if ((ret = conn->open_session(
+		    conn, NULL, NULL, &session)) != 0)
+			testutil_die(ret, "conn.open_session");
+		if ((ret = session->open_cursor(
+		    session, s->name, NULL, NULL, &cursor)) != 0)
+			testutil_die(ret, "session.open_cursor");
+		for (i = 0; i < s->nops; ++i, ++s->reads, __wt_yield())
+			reader_op(session, cursor, s);
+		if ((ret = session->close(session, NULL)) != 0)
+			testutil_die(ret, "session.close");
+	}
+
+	printf(" read thread %2d stopping: tid: %s, file: %s\n",
+	    id, tid, s->name);
+
+	return (NULL);
+}
+
+/*
+ * writer_op --
+ *	Write operation.
+ */
+static inline void
+writer_op(WT_SESSION *session, WT_CURSOR *cursor, INFO *s)
+{
+	WT_ITEM *key, _key, *value, _value;
+	u_int keyno;
+	int ret;
+	char keybuf[64], valuebuf[64];
+
+	key = &_key;
+	value = &_value;
+
+	keyno = __wt_random(&s->rnd) % nkeys + 1;
+	if (ftype == ROW) {
+		key->data = keybuf;
+		key->size = (uint32_t)
+		    snprintf(keybuf, sizeof(keybuf), "%017u", keyno);
+		cursor->set_key(cursor, key);
+	} else
+		cursor->set_key(cursor, (uint32_t)keyno);
+	if (keyno % 5 == 0) {
+		++s->remove;
+		if ((ret =
+		    cursor->remove(cursor)) != 0 && ret != WT_NOTFOUND)
+			testutil_die(ret, "cursor.remove");
+	} else {
+		++s->update;
+		value->data = valuebuf;
+		if (ftype == FIX)
+			cursor->set_value(cursor, 0x10);
+		else {
+			value->size = (uint32_t)snprintf(
+			    valuebuf, sizeof(valuebuf), "XXX %37u", keyno);
+			cursor->set_value(cursor, value);
+		}
+		if ((ret = cursor->update(cursor)) != 0)
+			testutil_die(ret, "cursor.update");
+	}
+	if (log_print)
+		(void)session->log_printf(session,
+		    "Writer Thread %p key %017u", pthread_self(), keyno);
+}
+
+/*
+ * writer --
+ *	Writer thread start function.
+ */
+static void *
+writer(void *arg)
+{
+	INFO *s;
+	WT_CURSOR *cursor;
+	WT_SESSION *session;
+	u_int i;
+	int id, ret;
+	char tid[128];
+
+	id = (int)(uintptr_t)arg;
+	s = &run_info[id];
+	__wt_thread_id(tid, sizeof(tid));
+	__wt_random_init(&s->rnd);
+
+	printf("write thread %2d starting: tid: %s, file: %s\n",
+	    id, tid, s->name);
+
+	__wt_yield();		/* Get all the threads created. */
+
+	if (session_per_op) {
+		for (i = 0; i < s->nops; ++i, __wt_yield()) {
+			if ((ret = conn->open_session(
+			    conn, NULL, NULL, &session)) != 0)
+				testutil_die(ret, "conn.open_session");
+			if ((ret = session->open_cursor(
+			    session, s->name, NULL, NULL, &cursor)) != 0)
+				testutil_die(ret, "session.open_cursor");
+			writer_op(session, cursor, s);
+			if ((ret = session->close(session, NULL)) != 0)
+				testutil_die(ret, "session.close");
+		}
+	} else {
+		if ((ret = conn->open_session(
+		    conn, NULL, NULL, &session)) != 0)
+			testutil_die(ret, "conn.open_session");
+		if ((ret = session->open_cursor(
+		    session, s->name, NULL, NULL, &cursor)) != 0)
+			testutil_die(ret, "session.open_cursor");
+		for (i = 0; i < s->nops; ++i, __wt_yield())
+			writer_op(session, cursor, s);
+		if ((ret = session->close(session, NULL)) != 0)
+			testutil_die(ret, "session.close");
+	}
+
+	printf("write thread %2d stopping: tid: %s, file: %s\n",
+	    id, tid, s->name);
+
+	return (NULL);
+}
+
+/*
+ * print_stats --
+ *	Display reader/writer thread stats.
+ */
+static void
+print_stats(u_int nthreads)
+{
+	INFO *s;
+	u_int id;
+
+	s = run_info;
+	for (id = 0; id < nthreads; ++id, ++s)
+		printf("%3d: read %6d, remove %6d, update %6d\n",
+		    id, s->reads, s->remove, s->update);
+}