Import wiredtiger-wiredtiger-2.8.0-410-g636a7b2.tar.gz from wiredtiger branch mongodb-3.4

ref: eaa7b5f..636a7b2

WT-2103       add incremental backup testing to format
WT-2223       Add stress testing for in-memory
WT-2343       Assert we don't remove or rename when backup cursor is open
WT-2349       Add ability to open databases read-only
WT-2359       WiredTiger with Python will hang if a calloc failure occurs during __wt_connection_close
WT-2360       Allow disjunctions and combinations of operations in join cursors
WT-2446       Estimate WT cache hit ratio
WT-2450       salvage releases pages, then explicitly evicts them.
WT-2453       Throughput drop in wtperf evict Jenkins tests
WT-2479       dump utility discards table config (JSON)
WT-2504       Should READONLY always read basecfg file?
WT-2505       Review clang analyzer warnings
WT-2508       test programs should remove test directories on the "clean" target
WT-2518       LSM checkpoint handle acquisition optimization
WT-2520       WT_SESSION::verify should not alter tables
WT-2526       mixing and matching readonly and read/write handles
WT-2535       Extend test/format to test for transactions reading their writes
WT-2537       cannot open DB written by WT2.6.1 with WT2.8.0 due to WT_NOTFOUND on recovery
WT-2539       Implement file streaming above pluggable filesystems
WT-2540       Separate stream and file handle methods
WT-2542       fixed-length column store reconciliation overwrites original values
WT-2544       Investigate any thread populating eviction queue
WT-2546       Eviction server not help evict pages sometimes
WT-2547       Add 1-eviction-worker jobs to Jenkins
WT-2548       Cap the amount of data handed to raw compression.
WT-2549       joins using recno keys return no values
WT-2550       java ex_schema example fails
WT-2552       Public API for pluggable filesystems
WT-2553       Document in-memory configuration and WT_CACHE_FULL error return
WT-2556       typo in the Java example code
WT-2557       format test program should discard log files after incremental backup
WT-2558       WT_PAGE structure reorganization
WT-2559       Jenkins Windows segfault in logging code
WT-2560       test/format workload stuck trying to update oldest transaction ID
WT-2562       reconfig02 test failing sometimes on PPC
WT-2565       item 3573 on page at [write-check] is a corrupted cell
WT-2566       All lock operations should be barriers
WT-2567       segfault in test/format log truncate
WT-2568       Java PackTest.java compilation error
WT-2569       win_handle_read should always call GetLastError on error
WT-2570       Minor lint cleanups.
WT-2571       join code cleanup
WT-2572       don't select an in-memory format run if incompatible options configured
WT-2573       free of stack-allocated WT_REF
WT-2574       format doesn't free all allocated configure memory
WT-2576       variable-length column-store out-of-order return
WT-2577       core dump discarding non-existent addresses
WT-2579       in-memory configurations break debugging support
WT-2580       potential SWIG naming conflict in Java
WT-2581       assert multi->disk_image == NULL
WT-2582       cache eviction server error: WT_RESTART
WT-2583       incremental backup can prevent future recovery
WT-2584       don't use periods in error messages
WT-2586       Remove ex_config.c until config cursors are supported
WT-2592       Joins using non-recno key types not working
WT-2593       disk full with pre-allocated log files
WT-2595       Fix compiler warning in packing tests
WT-2598       in-memory FS needs fast lookup on file names
WT-2599       split out the checksum code from the support directory
WT-2600       clean up test program #includes
WT-2602       LSM stress hangs with very large uncompressed pages
WT-2609       Incorrect "skips API_END call" error.
WT-2612       The dist/s_prototypes script is creating a debugging file xxx.
WT-2613       WT Compile windows Alt is returning a C4100 error
WT-2615       Enabling checkpoints in test/format leads to reduced concurrency
WT-2616       In-memory deadlock getting size
WT-2621       WiredTiger fails to compile on MSVC 2013
SERVER-23661  $sample takes disproportionately long time on newly created collection
SERVER-23904  WiredTiger changes for MongoDB 3.3.6

1 files changed, 108 insertions, 54 deletions

diff --git a/src/third_party/wiredtiger/src/os_common/os_fhandle.c b/src/third_party/wiredtiger/src/os_common/os_fhandle.c
index b16b2e24bfa..818829203e0 100644
--- a/src/third_party/wiredtiger/src/os_common/os_fhandle.c
+++ b/src/third_party/wiredtiger/src/os_common/os_fhandle.c
@@ -9,20 +9,88 @@
 #include "wt_internal.h"
 
 /*
- * __wt_handle_search --
- *	Search for a matching handle.
+ * __fhandle_method_finalize --
+ *	Initialize any NULL WT_FH structure methods to not-supported. Doing
+ *	this means that custom file systems with incomplete implementations
+ *	won't dereference NULL pointers.
+ */
+static int
+__fhandle_method_finalize(
+    WT_SESSION_IMPL *session, WT_FILE_HANDLE *handle, bool readonly)
+{
+#define	WT_HANDLE_METHOD_REQ(name)					\
+	if (handle->name == NULL)					\
+		WT_RET_MSG(session, EINVAL,				\
+		    "a WT_FILE_HANDLE.%s method must be configured", #name)
+
+	WT_HANDLE_METHOD_REQ(close);
+	/* not required: fadvise */
+	/* not required: fallocate */
+	/* not required: fallocate_nolock */
+	/* not required: lock */
+	/* not required: map */
+	/* not required: map_discard */
+	/* not required: map_preload */
+	/* not required: map_unmap */
+	WT_HANDLE_METHOD_REQ(read);
+	WT_HANDLE_METHOD_REQ(size);
+	/* not required: sync */
+	/* not required: sync_nowait */
+	if (!readonly) {
+		WT_HANDLE_METHOD_REQ(truncate);
+		WT_HANDLE_METHOD_REQ(write);
+	}
+
+	return (0);
+}
+
+#ifdef HAVE_DIAGNOSTIC
+/*
+ * __wt_handle_is_open --
+ *	Return if there's an open handle matching a name.
  */
 bool
-__wt_handle_search(WT_SESSION_IMPL *session,
-    const char *name, bool increment_ref, WT_FH *newfh, WT_FH **fhp)
+__wt_handle_is_open(WT_SESSION_IMPL *session, const char *name)
+{
+	WT_CONNECTION_IMPL *conn;
+	WT_FH *fh;
+	uint64_t bucket, hash;
+	bool found;
+
+	conn = S2C(session);
+	found = false;
+
+	hash = __wt_hash_city64(name, strlen(name));
+	bucket = hash % WT_HASH_ARRAY_SIZE;
+
+	__wt_spin_lock(session, &conn->fh_lock);
+
+	TAILQ_FOREACH(fh, &conn->fhhash[bucket], hashq)
+		if (strcmp(name, fh->name) == 0) {
+			found = true;
+			break;
+		}
+
+	__wt_spin_unlock(session, &conn->fh_lock);
+
+	return (found);
+}
+#endif
+
+/*
+ * __handle_search --
+ *	Search for a matching handle.
+ */
+static bool
+__handle_search(
+    WT_SESSION_IMPL *session, const char *name, WT_FH *newfh, WT_FH **fhp)
 {
 	WT_CONNECTION_IMPL *conn;
 	WT_FH *fh;
 	uint64_t bucket, hash;
 	bool found;
 
-	if (fhp != NULL)
-		*fhp = NULL;
+	*fhp = NULL;
 
 	conn = S2C(session);
 	found = false;
@@ -33,15 +101,13 @@ __wt_handle_search(WT_SESSION_IMPL *session,
 	__wt_spin_lock(session, &conn->fh_lock);
 
 	/*
-	 * If we already have the file open, optionally increment the reference
-	 * count and return a pointer.
+	 * If we already have the file open, increment the reference count and
+	 * return a pointer.
 	 */
 	TAILQ_FOREACH(fh, &conn->fhhash[bucket], hashq)
 		if (strcmp(name, fh->name) == 0) {
-			if (increment_ref)
-				++fh->ref;
-			if (fhp != NULL)
-				*fhp = fh;
+			++fh->ref;
+			*fhp = fh;
 			found = true;
 			break;
 		}
@@ -49,13 +115,11 @@ __wt_handle_search(WT_SESSION_IMPL *session,
 	/* If we don't find a match, optionally add a new entry. */
 	if (!found && newfh != NULL) {
 		newfh->name_hash = hash;
-		WT_CONN_FILE_INSERT(conn, newfh, bucket);
+		WT_FILE_HANDLE_INSERT(conn, newfh, bucket);
 		(void)__wt_atomic_add32(&conn->open_file_count, 1);
 
-		if (increment_ref)
-			++newfh->ref;
-		if (fhp != NULL)
-			*fhp = newfh;
+		++newfh->ref;
+		*fhp = newfh;
 	}
 
 	__wt_spin_unlock(session, &conn->fh_lock);
@@ -68,8 +132,8 @@ __wt_handle_search(WT_SESSION_IMPL *session,
  *	Optionally output a verbose message on handle open.
  */
 static inline int
-__open_verbose(WT_SESSION_IMPL *session,
-    const char *name, uint32_t file_type, uint32_t flags)
+__open_verbose(
+    WT_SESSION_IMPL *session, const char *name, int file_type, u_int flags)
 {
 #ifdef HAVE_VERBOSE
 	WT_DECL_RET;
@@ -85,19 +149,19 @@ __open_verbose(WT_SESSION_IMPL *session,
 	 */
 
 	switch (file_type) {
-	case WT_FILE_TYPE_CHECKPOINT:
+	case WT_OPEN_FILE_TYPE_CHECKPOINT:
 		file_type_tag = "checkpoint";
 		break;
-	case WT_FILE_TYPE_DATA:
+	case WT_OPEN_FILE_TYPE_DATA:
 		file_type_tag = "data";
 		break;
-	case WT_FILE_TYPE_DIRECTORY:
+	case WT_OPEN_FILE_TYPE_DIRECTORY:
 		file_type_tag = "directory";
 		break;
-	case WT_FILE_TYPE_LOG:
+	case WT_OPEN_FILE_TYPE_LOG:
 		file_type_tag = "log";
 		break;
-	case WT_FILE_TYPE_REGULAR:
+	case WT_OPEN_FILE_TYPE_REGULAR:
 		file_type_tag = "regular";
 		break;
 	default:
@@ -115,18 +179,16 @@ __open_verbose(WT_SESSION_IMPL *session,
 	}
 
 	WT_OPEN_VERBOSE_FLAG(WT_OPEN_CREATE, "create");
+	WT_OPEN_VERBOSE_FLAG(WT_OPEN_DIRECTIO, "direct-IO");
 	WT_OPEN_VERBOSE_FLAG(WT_OPEN_EXCLUSIVE, "exclusive");
 	WT_OPEN_VERBOSE_FLAG(WT_OPEN_FIXED, "fixed");
 	WT_OPEN_VERBOSE_FLAG(WT_OPEN_READONLY, "readonly");
-	WT_OPEN_VERBOSE_FLAG(WT_STREAM_APPEND, "stream-append");
-	WT_OPEN_VERBOSE_FLAG(WT_STREAM_READ, "stream-read");
-	WT_OPEN_VERBOSE_FLAG(WT_STREAM_WRITE, "stream-write");
 
 	if (tmp->size != 0)
 		WT_ERR(__wt_buf_catfmt(session, tmp, ")"));
 
 	ret = __wt_verbose(session, WT_VERB_FILEOPS,
-	    "%s: handle-open: type %s%s",
+	    "%s: file-open: type %s%s",
 	    name, file_type_tag, tmp->size == 0 ? "" : (char *)tmp->data);
 
 err:	__wt_scr_free(session, &tmp);
@@ -146,17 +208,19 @@ err:	__wt_scr_free(session, &tmp);
  */
 int
 __wt_open(WT_SESSION_IMPL *session,
-    const char *name, uint32_t file_type, uint32_t flags, WT_FH **fhp)
+    const char *name, WT_OPEN_FILE_TYPE file_type, u_int flags, WT_FH **fhp)
 {
 	WT_CONNECTION_IMPL *conn;
 	WT_DECL_RET;
 	WT_FH *fh;
+	WT_FILE_SYSTEM *file_system;
 	bool lock_file, open_called;
 	char *path;
 
 	WT_ASSERT(session, file_type != 0);	/* A file type is required. */
 
 	conn = S2C(session);
+	file_system = conn->file_system;
 	fh = NULL;
 	open_called = false;
 	path = NULL;
@@ -164,21 +228,12 @@ __wt_open(WT_SESSION_IMPL *session,
 	WT_RET(__open_verbose(session, name, file_type, flags));
 
 	/* Check if the handle is already open. */
-	if (__wt_handle_search(session, name, true, NULL, &fh)) {
-		/*
-		 * XXX
-		 * The in-memory implementation has to reset the file offset
-		 * when a file is re-opened (which obviously also depends on
-		 * in-memory configurations never opening a file in more than
-		 * one thread at a time). This needs to be fixed.
-		 */
-		if (F_ISSET(fh, WT_FH_IN_MEMORY) && fh->ref == 1)
-			fh->off = 0;
+	if (__handle_search(session, name, NULL, &fh)) {
 		*fhp = fh;
 		return (0);
 	}
 
-	/* Allocate a structure and set the name. */
+	/* Allocate and initialize the handle. */
 	WT_ERR(__wt_calloc_one(session, &fh));
 	WT_ERR(__wt_strdup(session, name, &fh->name));
 
@@ -200,17 +255,21 @@ __wt_open(WT_SESSION_IMPL *session,
 		WT_ERR(__wt_filename(session, name, &path));
 
 	/* Call the underlying open function. */
-	WT_ERR(conn->handle_open(
-	    session, fh, path == NULL ? name : path, file_type, flags));
+	WT_ERR(file_system->open_file(file_system, &session->iface,
+	    path == NULL ? name : path, file_type, flags, &fh->handle));
 	open_called = true;
 
+	WT_ERR(__fhandle_method_finalize(
+	    session, fh->handle, LF_ISSET(WT_OPEN_READONLY)));
+
 	/*
 	 * Repeat the check for a match: if there's no match, link our newly
 	 * created handle onto the database's list of files.
 	 */
-	if (__wt_handle_search(session, name, true, fh, fhp)) {
+	if (__handle_search(session, name, fh, fhp)) {
 err:		if (open_called)
-			WT_TRET(fh->fh_close(session, fh));
+			WT_TRET(fh->handle->close(
+			    fh->handle, (WT_SESSION *)session));
 		if (fh != NULL) {
 			__wt_free(session, fh->name);
 			__wt_free(session, fh);
@@ -242,7 +301,7 @@ __wt_close(WT_SESSION_IMPL *session, WT_FH **fhp)
 
 	/* Track handle-close as a file operation, so open and close match. */
 	WT_RET(__wt_verbose(
-	    session, WT_VERB_FILEOPS, "%s: handle-close", fh->name));
+	    session, WT_VERB_FILEOPS, "%s: file-close", fh->name));
 
 	/*
 	 * If the reference count hasn't gone to 0, or if it's an in-memory
@@ -252,20 +311,20 @@ __wt_close(WT_SESSION_IMPL *session, WT_FH **fhp)
 	 */
 	__wt_spin_lock(session, &conn->fh_lock);
 	WT_ASSERT(session, fh->ref > 0);
-	if ((fh->ref > 0 && --fh->ref > 0) || F_ISSET(fh, WT_FH_IN_MEMORY)) {
+	if ((fh->ref > 0 && --fh->ref > 0)) {
 		__wt_spin_unlock(session, &conn->fh_lock);
 		return (0);
 	}
 
 	/* Remove from the list. */
 	bucket = fh->name_hash % WT_HASH_ARRAY_SIZE;
-	WT_CONN_FILE_REMOVE(conn, fh, bucket);
+	WT_FILE_HANDLE_REMOVE(conn, fh, bucket);
 	(void)__wt_atomic_sub32(&conn->open_file_count, 1);
 
 	__wt_spin_unlock(session, &conn->fh_lock);
 
 	/* Discard underlying resources. */
-	ret = fh->fh_close(session, fh);
+	ret = fh->handle->close(fh->handle, (WT_SESSION *)session);
 
 	__wt_free(session, fh->name);
 	__wt_free(session, fh);
@@ -287,18 +346,13 @@ __wt_close_connection_close(WT_SESSION_IMPL *session)
 	conn = S2C(session);
 
 	while ((fh = TAILQ_FIRST(&conn->fhqh)) != NULL) {
-		/*
-		 * In-memory configurations will have open files, but the ref
-		 * counts should be zero.
-		 */
-		if (!F_ISSET(conn, WT_CONN_IN_MEMORY) || fh->ref != 0) {
+		if (fh->ref != 0) {
 			ret = EBUSY;
 			__wt_errx(session,
 			    "Connection has open file handles: %s", fh->name);
 		}
 
 		fh->ref = 1;
-		F_CLR(fh, WT_FH_IN_MEMORY);
 
 		WT_TRET(__wt_close(session, &fh));
 	}