Import wiredtiger-wiredtiger-2.7.0-559-g07966a4.tar.gz from wiredtiger branch mongodb-3.2

ref: 3c2ad56..07966a4

WT-1517       schema format edge cases
WT-1801       Add a directory sync after rollback of a WT_SESSION::rename operation
WT-2060       Simplify aggregation of statistics
WT-2073       metadata cleanups
WT-2099       Seeing memory underflow messages
WT-2113       truncate01 sometimes fails
WT-2142       Connection cleanup in Python tests
WT-2177       Add an optional per-thread seed to random number generator
WT-2198       bulk load and column store appends
WT-2216       simplify row-store search loop slightly
WT-2225       New split code performance impact
WT-2231       pinned page cursor searches could check parent keys
WT-2235       wt printlog option without unicode
WT-2242       WiredTiger treats dead trees the same as other trees in eviction
WT-2244       Trigger in-memory splits sooner
WT-2245       WTPERF Truncate has no ability to catch up when it falls behind
WT-2246       column-store append searches the leaf page; the maximum record number fails CRUD operations
WT-2247       variable-length column-store in-memory page splits
WT-2256       WTPERFs throttle option fires in bursts
WT-2257       wtperf doesn't handle overriding workload config
WT-2258       WiredTiger preloads pages even when direct-IO is configured.
WT-2259       __wt_evict_file_exclusive_on() should clear WT_BTREE_NO_EVICTION on error
WT-2260       Workloads evict internal pages unexpectedly
WT-2262       Random sampling is skewed by tree shape
WT-2265       Wiredtiger related change in ppc64le specific code block in gcc.h
WT-2266       Add wtperf config to set if perf thresholds are fatal
WT-2267       Improve wtperf throttling implementation to provide steady load
WT-2269       wtperf should dump its config everytime it runs
WT-2272       Stress test assertion in the sweep server
WT-2275       broken DB after application crash
WT-2276       tool to decode checkpoint addr
WT-2277       Remove WT check against big-endian systems
WT-2279       Define WT_PAUSE(), WT_FULL_BARRIER(), etc when s390x is defined
WT-2281       wtperf smoke.sh fails on ppc64le
WT-2282       error in wt_txn_update_oldest verbose message test
WT-2283       retry in txn_update_oldest results in a hang
WT-2284       Repeated macro definition
WT-2285       configure should set BUFFER_ALIGNMENT_DEFAULT to 4kb on linux
WT-2287       WT_SESSION.rebalance
WT-2289       failure in fast key check
WT-2290       WT_SESSION.compact could be more effective.
WT-2291       Random cursor walk inefficient in skip list only trees
WT-2295       WT_SESSION.create does a full-scan of the main table
WT-2296       New log algorithm needs improving for sync/flush settings
WT-2297       Fix off-by-one error in Huffman config file parsing
WT-2299       upper-level WiredTiger code is reaching into the block manager
WT-2301       Add reading a range to wtperf
WT-2303       Build warning in wtperf
WT-2304       wtperf crash dumping config
WT-2305       Fix coverity scan issues on 23/12/2015
WT-2307       Internal page splits can corrupt cursor iteration
WT-2308       custom extractor for ref_cursors in join cursor
WT-2311       Support Sparc
WT-2312       re-creating a deleted column-store page can corrupt the in-memory tree
WT-2313       sweep-server: conn_dhandle.c, 610: dhandle != conn->cache->evict_file_next
WT-2314       page-swap error handling is inconsistent
WT-2316       stress test failure: WT_CURSOR.prev out-of-order returns
WT-2320       Only check copyright when cutting releases
WT-2321       WT-2321: race between eviction and worker threads on the eviction queue
WT-2326       Change WTPERF to use new memory allocation functions instead of the standard
WT-2328       schema drop does direct unlink, it should use a block manager interface.
WT-2331       Checking of search() result for reference cursors before join()
WT-2332       Bug in logging write-no-sync mode
WT-2333       Add a flag so drop doesn't block
WT-2335       NULL pointer crash in config_check_search with invalid configuration string
WT-2338       Disable using pre-allocated log files when backup cursor is open
WT-2339       format post-rebalance verify failure (stress run #11586)
WT-2340       Add logging guarantee assertions, whitespace
WT-2342       Enhance wtperf to support background create and drop operations
WT-2344       OS X compiler warning
WT-2347       Java: schema format edge cases
WT-2348       xargs -P isn't portable
WT-2355       Fix minor scratch buffer usage in logging
SERVER-21833  Compact does not release space to the system with WiredTiger
SERVER-21887  $sample takes disproportionately long time on newly created collection
SERVER-22064  Coverity analysis defect 77699: Unchecked return value
SERVER-21944  WiredTiger changes for 3.2.2

1 files changed, 284 insertions, 0 deletions

diff --git a/src/third_party/wiredtiger/examples/c/ex_extractor.c b/src/third_party/wiredtiger/examples/c/ex_extractor.c
new file mode 100644
index 00000000000..fff9c79f8e0
--- /dev/null
+++ b/src/third_party/wiredtiger/examples/c/ex_extractor.c
@@ -0,0 +1,284 @@
+/*-
+ * Public Domain 2014-2016 MongoDB, Inc.
+ * Public Domain 2008-2014 WiredTiger, Inc.
+ *
+ * This is free and unencumbered software released into the public domain.
+ *
+ * Anyone is free to copy, modify, publish, use, compile, sell, or
+ * distribute this software, either in source code form or as a compiled
+ * binary, for any purpose, commercial or non-commercial, and by any
+ * means.
+ *
+ * In jurisdictions that recognize copyright laws, the author or authors
+ * of this software dedicate any and all copyright interest in the
+ * software to the public domain. We make this dedication for the benefit
+ * of the public at large and to the detriment of our heirs and
+ * successors. We intend this dedication to be an overt act of
+ * relinquishment in perpetuity of all present and future rights to this
+ * software under copyright law.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * ex_extractor.c
+ *	Example of how to use a WiredTiger custom index extractor extension.
+ */
+
+#include <inttypes.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include <wiredtiger.h>
+
+#define	RET_OK(ret)	((ret) == 0 || (ret) == WT_NOTFOUND)
+
+int add_extractor(WT_CONNECTION *conn);
+
+static const char *home;
+
+struct president_data {
+	int		id;
+	const char	*last_name;
+	const char	*first_name;
+	uint16_t	term_start;
+	uint16_t	term_end;
+};
+
+static const struct president_data example_data[] = {
+	{ 0, "Obama", "Barack", 2009, 2014 },
+	{ 1, "Bush", "George W", 2001, 2009 },
+	{ 2, "Clinton", "Bill", 1993, 2001 },
+	{ 3, "Bush", "George H", 1989, 1993 },
+	{ 4, "Reagan", "Ronald", 1981, 1989 },
+	{ 0, NULL, NULL, 0, 0 }
+};
+/*
+ * Number of years this data spans
+ */
+#define	YEAR_BASE	1981
+#define	YEAR_SPAN	(2014-1981)
+
+/*
+ * A custom index extractor function that adds an index entry for each year of
+ * the given president's term.
+ */
+static int
+my_extract(WT_EXTRACTOR *extractor, WT_SESSION *session,
+    const WT_ITEM *key, const WT_ITEM *value, WT_CURSOR *result_cursor)
+{
+	char *last_name, *first_name;
+	uint16_t term_end, term_start, year;
+	int ret;
+
+	/* Unused parameters */
+	(void)extractor;
+	(void)key;
+
+	/* Unpack the value. */
+	if ((ret = wiredtiger_struct_unpack(
+	    session, value->data, value->size, "SSHH",
+	    &last_name, &first_name, &term_start, &term_end)) != 0)
+		return (ret);
+
+	/*
+	 * We have overlapping years, so multiple records may share the same
+	 * index key.
+	 */
+	for (year = term_start; year <= term_end; ++year) {
+		/*
+		 * Note that the extract callback is called for all operations
+		 * that update the table, not just inserts.  The user sets the
+		 * key and uses the cursor->insert() method to return the index
+		 * key(s).  WiredTiger will perform the required operation
+		 * (such as a remove()).
+		 */
+		fprintf(stderr, "EXTRACTOR: index op for year %d: %s %s\n",
+		    year, first_name, last_name);
+		result_cursor->set_key(result_cursor, year);
+		if ((ret = result_cursor->insert(result_cursor)) != 0) {
+			fprintf(stderr, "EXTRACTOR: op year %d: error %d\n",
+			    year, ret);
+			return (ret);
+		}
+	}
+	return (0);
+}
+
+/*
+ * The terminate method is called to release any allocated resources when the
+ * table is closed.  In this example, no cleanup is required.
+ */
+static int
+my_extract_terminate(WT_EXTRACTOR *extractor, WT_SESSION *session)
+{
+	(void)extractor;
+	(void)session;
+
+	return (0);
+}
+
+int
+add_extractor(WT_CONNECTION *conn)
+{
+	int ret;
+
+	static WT_EXTRACTOR my_extractor = {
+	    my_extract, NULL, my_extract_terminate
+	};
+	ret = conn->add_extractor(conn, "my_extractor", &my_extractor, NULL);
+
+	return (ret);
+}
+
+/*
+ * Read the index by year and print out who was in office that year.
+ */
+static int
+read_index(WT_SESSION *session)
+{
+	WT_CURSOR *cursor;
+	int i, ret;
+	char *first_name, *last_name;
+	uint16_t rec_year, term_end, term_start, year;
+
+	year = 0;
+	srand((unsigned int)getpid());
+	ret = session->open_cursor(
+	    session, "index:presidents:term", NULL, NULL, &cursor);
+	/*
+	 * Pick 10 random years and read the data.
+	 */
+	for (i = 0; i < 10 && RET_OK(ret); i++) {
+		year = (uint16_t)((rand() % YEAR_SPAN) + YEAR_BASE);
+		printf("Year %d:\n", year);
+		cursor->set_key(cursor, year);
+		if ((ret = cursor->search(cursor)) != 0)
+			break;
+		if ((ret = cursor->get_key(cursor, &rec_year)) != 0)
+			break;
+		if ((ret = cursor->get_value(cursor,
+		    &last_name, &first_name, &term_start, &term_end)) != 0)
+			break;
+
+		/* Report all presidents that served during the chosen year */
+		while (term_start <= year &&
+		    year <= term_end && year == rec_year) {
+			printf("\t%s %s\n", first_name, last_name);
+			if ((ret = cursor->next(cursor)) != 0)
+				break;
+			if ((ret = cursor->get_key(cursor, &rec_year)) != 0)
+				break;
+			if ((ret = cursor->get_value(cursor, &last_name,
+			    &first_name, &term_start, &term_end)) != 0)
+				break;
+		}
+	}
+	if (!RET_OK(ret))
+		fprintf(stderr, "Error %d for year %d\n", ret, year);
+
+	ret = cursor->close(cursor);
+	return (ret);
+}
+
+/*
+ * Remove some items from the primary table.
+ */
+static int
+remove_items(WT_SESSION *session)
+{
+	WT_CURSOR *cursor;
+	struct president_data p;
+	int i, ret;
+
+	/*
+	 * Removing items from the primary table will call the extractor
+	 * for the index and allow our custom extractor code to handle
+	 * each custom key.
+	 */
+	ret = session->open_cursor(
+	    session, "table:presidents", NULL, NULL, &cursor);
+	/*
+	 * Just remove the first few items.
+	 */
+	for (i = 0; example_data[i].last_name != NULL && i < 2; i++) {
+		p = example_data[i];
+		cursor->set_key(cursor, p.id);
+		ret = cursor->remove(cursor);
+	}
+	return (ret);
+}
+
+/*
+ * Set up the table and index of the data.
+ */
+static int
+setup_table(WT_SESSION *session)
+{
+	WT_CURSOR *cursor;
+	struct president_data p;
+	int i, ret;
+
+	/* Create the primary table. It has a key of the unique ID. */
+	ret = session->create(session, "table:presidents",
+	    "key_format=I,value_format=SSHH,"
+	    "columns=(ID,last_name,first_name,term_begin,term_end)");
+
+	/*
+	 * Create the index that is generated with an extractor. The index
+	 * will generate an entry in the index for each year a president
+	 * was in office.
+	 */
+	ret = session->create(session, "index:presidents:term",
+	    "key_format=H,columns=(term),extractor=my_extractor");
+
+	ret = session->open_cursor(
+	    session, "table:presidents", NULL, NULL, &cursor);
+	for (i = 0; example_data[i].last_name != NULL; i++) {
+		p = example_data[i];
+		cursor->set_key(cursor, p.id);
+		cursor->set_value(cursor,
+		    p.last_name, p.first_name, p.term_start, p.term_end);
+		fprintf(stderr, "SETUP: table insert %d-%d: %s %s\n",
+		    p.term_start, p.term_end,
+		    p.first_name, p.last_name);
+		ret = cursor->insert(cursor);
+	}
+	return (ret);
+}
+
+int
+main(void)
+{
+	WT_CONNECTION *conn;
+	WT_SESSION *session;
+	int ret;
+
+	/*
+	 * Create a clean test directory for this run of the test program if the
+	 * environment variable isn't already set (as is done by make check).
+	 */
+	if (getenv("WIREDTIGER_HOME") == NULL) {
+		home = "WT_HOME";
+		ret = system("rm -rf WT_HOME && mkdir WT_HOME");
+	} else
+		home = NULL;
+
+	ret = wiredtiger_open(home, NULL, "create,cache_size=500M", &conn);
+	ret = add_extractor(conn);
+	ret = conn->open_session(conn, NULL, NULL, &session);
+
+	ret = setup_table(session);
+	ret = read_index(session);
+	ret = remove_items(session);
+
+	ret = conn->close(conn, NULL);
+
+	return (ret);
+}