Merge branch 'develop' into mongodb-3.4mongodb-3.3.6

1 files changed, 108 insertions, 54 deletions

diff --git a/src/os_common/os_fhandle.c b/src/os_common/os_fhandle.c
index b16b2e24bfa..818829203e0 100644
--- a/src/os_common/os_fhandle.c
+++ b/src/os_common/os_fhandle.c
@@ -9,20 +9,88 @@
 #include "wt_internal.h"
 
 /*
- * __wt_handle_search --
- *	Search for a matching handle.
+ * __fhandle_method_finalize --
+ *	Initialize any NULL WT_FH structure methods to not-supported. Doing
+ *	this means that custom file systems with incomplete implementations
+ *	won't dereference NULL pointers.
+ */
+static int
+__fhandle_method_finalize(
+    WT_SESSION_IMPL *session, WT_FILE_HANDLE *handle, bool readonly)
+{
+#define	WT_HANDLE_METHOD_REQ(name)					\
+	if (handle->name == NULL)					\
+		WT_RET_MSG(session, EINVAL,				\
+		    "a WT_FILE_HANDLE.%s method must be configured", #name)
+
+	WT_HANDLE_METHOD_REQ(close);
+	/* not required: fadvise */
+	/* not required: fallocate */
+	/* not required: fallocate_nolock */
+	/* not required: lock */
+	/* not required: map */
+	/* not required: map_discard */
+	/* not required: map_preload */
+	/* not required: map_unmap */
+	WT_HANDLE_METHOD_REQ(read);
+	WT_HANDLE_METHOD_REQ(size);
+	/* not required: sync */
+	/* not required: sync_nowait */
+	if (!readonly) {
+		WT_HANDLE_METHOD_REQ(truncate);
+		WT_HANDLE_METHOD_REQ(write);
+	}
+
+	return (0);
+}
+
+#ifdef HAVE_DIAGNOSTIC
+/*
+ * __wt_handle_is_open --
+ *	Return if there's an open handle matching a name.
  */
 bool
-__wt_handle_search(WT_SESSION_IMPL *session,
-    const char *name, bool increment_ref, WT_FH *newfh, WT_FH **fhp)
+__wt_handle_is_open(WT_SESSION_IMPL *session, const char *name)
+{
+	WT_CONNECTION_IMPL *conn;
+	WT_FH *fh;
+	uint64_t bucket, hash;
+	bool found;
+
+	conn = S2C(session);
+	found = false;
+
+	hash = __wt_hash_city64(name, strlen(name));
+	bucket = hash % WT_HASH_ARRAY_SIZE;
+
+	__wt_spin_lock(session, &conn->fh_lock);
+
+	TAILQ_FOREACH(fh, &conn->fhhash[bucket], hashq)
+		if (strcmp(name, fh->name) == 0) {
+			found = true;
+			break;
+		}
+
+	__wt_spin_unlock(session, &conn->fh_lock);
+
+	return (found);
+}
+#endif
+
+/*
+ * __handle_search --
+ *	Search for a matching handle.
+ */
+static bool
+__handle_search(
+    WT_SESSION_IMPL *session, const char *name, WT_FH *newfh, WT_FH **fhp)
 {
 	WT_CONNECTION_IMPL *conn;
 	WT_FH *fh;
 	uint64_t bucket, hash;
 	bool found;
 
-	if (fhp != NULL)
-		*fhp = NULL;
+	*fhp = NULL;
 
 	conn = S2C(session);
 	found = false;
@@ -33,15 +101,13 @@ __wt_handle_search(WT_SESSION_IMPL *session,
 	__wt_spin_lock(session, &conn->fh_lock);
 
 	/*
-	 * If we already have the file open, optionally increment the reference
-	 * count and return a pointer.
+	 * If we already have the file open, increment the reference count and
+	 * return a pointer.
 	 */
 	TAILQ_FOREACH(fh, &conn->fhhash[bucket], hashq)
 		if (strcmp(name, fh->name) == 0) {
-			if (increment_ref)
-				++fh->ref;
-			if (fhp != NULL)
-				*fhp = fh;
+			++fh->ref;
+			*fhp = fh;
 			found = true;
 			break;
 		}
@@ -49,13 +115,11 @@ __wt_handle_search(WT_SESSION_IMPL *session,
 	/* If we don't find a match, optionally add a new entry. */
 	if (!found && newfh != NULL) {
 		newfh->name_hash = hash;
-		WT_CONN_FILE_INSERT(conn, newfh, bucket);
+		WT_FILE_HANDLE_INSERT(conn, newfh, bucket);
 		(void)__wt_atomic_add32(&conn->open_file_count, 1);
 
-		if (increment_ref)
-			++newfh->ref;
-		if (fhp != NULL)
-			*fhp = newfh;
+		++newfh->ref;
+		*fhp = newfh;
 	}
 
 	__wt_spin_unlock(session, &conn->fh_lock);
@@ -68,8 +132,8 @@ __wt_handle_search(WT_SESSION_IMPL *session,
  *	Optionally output a verbose message on handle open.
  */
 static inline int
-__open_verbose(WT_SESSION_IMPL *session,
-    const char *name, uint32_t file_type, uint32_t flags)
+__open_verbose(
+    WT_SESSION_IMPL *session, const char *name, int file_type, u_int flags)
 {
 #ifdef HAVE_VERBOSE
 	WT_DECL_RET;
@@ -85,19 +149,19 @@ __open_verbose(WT_SESSION_IMPL *session,
 	 */
 
 	switch (file_type) {
-	case WT_FILE_TYPE_CHECKPOINT:
+	case WT_OPEN_FILE_TYPE_CHECKPOINT:
 		file_type_tag = "checkpoint";
 		break;
-	case WT_FILE_TYPE_DATA:
+	case WT_OPEN_FILE_TYPE_DATA:
 		file_type_tag = "data";
 		break;
-	case WT_FILE_TYPE_DIRECTORY:
+	case WT_OPEN_FILE_TYPE_DIRECTORY:
 		file_type_tag = "directory";
 		break;
-	case WT_FILE_TYPE_LOG:
+	case WT_OPEN_FILE_TYPE_LOG:
 		file_type_tag = "log";
 		break;
-	case WT_FILE_TYPE_REGULAR:
+	case WT_OPEN_FILE_TYPE_REGULAR:
 		file_type_tag = "regular";
 		break;
 	default:
@@ -115,18 +179,16 @@ __open_verbose(WT_SESSION_IMPL *session,
 	}
 
 	WT_OPEN_VERBOSE_FLAG(WT_OPEN_CREATE, "create");
+	WT_OPEN_VERBOSE_FLAG(WT_OPEN_DIRECTIO, "direct-IO");
 	WT_OPEN_VERBOSE_FLAG(WT_OPEN_EXCLUSIVE, "exclusive");
 	WT_OPEN_VERBOSE_FLAG(WT_OPEN_FIXED, "fixed");
 	WT_OPEN_VERBOSE_FLAG(WT_OPEN_READONLY, "readonly");
-	WT_OPEN_VERBOSE_FLAG(WT_STREAM_APPEND, "stream-append");
-	WT_OPEN_VERBOSE_FLAG(WT_STREAM_READ, "stream-read");
-	WT_OPEN_VERBOSE_FLAG(WT_STREAM_WRITE, "stream-write");
 
 	if (tmp->size != 0)
 		WT_ERR(__wt_buf_catfmt(session, tmp, ")"));
 
 	ret = __wt_verbose(session, WT_VERB_FILEOPS,
-	    "%s: handle-open: type %s%s",
+	    "%s: file-open: type %s%s",
 	    name, file_type_tag, tmp->size == 0 ? "" : (char *)tmp->data);
 
 err:	__wt_scr_free(session, &tmp);
@@ -146,17 +208,19 @@ err:	__wt_scr_free(session, &tmp);
  */
 int
 __wt_open(WT_SESSION_IMPL *session,
-    const char *name, uint32_t file_type, uint32_t flags, WT_FH **fhp)
+    const char *name, WT_OPEN_FILE_TYPE file_type, u_int flags, WT_FH **fhp)
 {
 	WT_CONNECTION_IMPL *conn;
 	WT_DECL_RET;
 	WT_FH *fh;
+	WT_FILE_SYSTEM *file_system;
 	bool lock_file, open_called;
 	char *path;
 
 	WT_ASSERT(session, file_type != 0);	/* A file type is required. */
 
 	conn = S2C(session);
+	file_system = conn->file_system;
 	fh = NULL;
 	open_called = false;
 	path = NULL;
@@ -164,21 +228,12 @@ __wt_open(WT_SESSION_IMPL *session,
 	WT_RET(__open_verbose(session, name, file_type, flags));
 
 	/* Check if the handle is already open. */
-	if (__wt_handle_search(session, name, true, NULL, &fh)) {
-		/*
-		 * XXX
-		 * The in-memory implementation has to reset the file offset
-		 * when a file is re-opened (which obviously also depends on
-		 * in-memory configurations never opening a file in more than
-		 * one thread at a time). This needs to be fixed.
-		 */
-		if (F_ISSET(fh, WT_FH_IN_MEMORY) && fh->ref == 1)
-			fh->off = 0;
+	if (__handle_search(session, name, NULL, &fh)) {
 		*fhp = fh;
 		return (0);
 	}
 
-	/* Allocate a structure and set the name. */
+	/* Allocate and initialize the handle. */
 	WT_ERR(__wt_calloc_one(session, &fh));
 	WT_ERR(__wt_strdup(session, name, &fh->name));
 
@@ -200,17 +255,21 @@ __wt_open(WT_SESSION_IMPL *session,
 		WT_ERR(__wt_filename(session, name, &path));
 
 	/* Call the underlying open function. */
-	WT_ERR(conn->handle_open(
-	    session, fh, path == NULL ? name : path, file_type, flags));
+	WT_ERR(file_system->open_file(file_system, &session->iface,
+	    path == NULL ? name : path, file_type, flags, &fh->handle));
 	open_called = true;
 
+	WT_ERR(__fhandle_method_finalize(
+	    session, fh->handle, LF_ISSET(WT_OPEN_READONLY)));
+
 	/*
 	 * Repeat the check for a match: if there's no match, link our newly
 	 * created handle onto the database's list of files.
 	 */
-	if (__wt_handle_search(session, name, true, fh, fhp)) {
+	if (__handle_search(session, name, fh, fhp)) {
 err:		if (open_called)
-			WT_TRET(fh->fh_close(session, fh));
+			WT_TRET(fh->handle->close(
+			    fh->handle, (WT_SESSION *)session));
 		if (fh != NULL) {
 			__wt_free(session, fh->name);
 			__wt_free(session, fh);
@@ -242,7 +301,7 @@ __wt_close(WT_SESSION_IMPL *session, WT_FH **fhp)
 
 	/* Track handle-close as a file operation, so open and close match. */
 	WT_RET(__wt_verbose(
-	    session, WT_VERB_FILEOPS, "%s: handle-close", fh->name));
+	    session, WT_VERB_FILEOPS, "%s: file-close", fh->name));
 
 	/*
 	 * If the reference count hasn't gone to 0, or if it's an in-memory
@@ -252,20 +311,20 @@ __wt_close(WT_SESSION_IMPL *session, WT_FH **fhp)
 	 */
 	__wt_spin_lock(session, &conn->fh_lock);
 	WT_ASSERT(session, fh->ref > 0);
-	if ((fh->ref > 0 && --fh->ref > 0) || F_ISSET(fh, WT_FH_IN_MEMORY)) {
+	if ((fh->ref > 0 && --fh->ref > 0)) {
 		__wt_spin_unlock(session, &conn->fh_lock);
 		return (0);
 	}
 
 	/* Remove from the list. */
 	bucket = fh->name_hash % WT_HASH_ARRAY_SIZE;
-	WT_CONN_FILE_REMOVE(conn, fh, bucket);
+	WT_FILE_HANDLE_REMOVE(conn, fh, bucket);
 	(void)__wt_atomic_sub32(&conn->open_file_count, 1);
 
 	__wt_spin_unlock(session, &conn->fh_lock);
 
 	/* Discard underlying resources. */
-	ret = fh->fh_close(session, fh);
+	ret = fh->handle->close(fh->handle, (WT_SESSION *)session);
 
 	__wt_free(session, fh->name);
 	__wt_free(session, fh);
@@ -287,18 +346,13 @@ __wt_close_connection_close(WT_SESSION_IMPL *session)
 	conn = S2C(session);
 
 	while ((fh = TAILQ_FIRST(&conn->fhqh)) != NULL) {
-		/*
-		 * In-memory configurations will have open files, but the ref
-		 * counts should be zero.
-		 */
-		if (!F_ISSET(conn, WT_CONN_IN_MEMORY) || fh->ref != 0) {
+		if (fh->ref != 0) {
 			ret = EBUSY;
 			__wt_errx(session,
 			    "Connection has open file handles: %s", fh->name);
 		}
 
 		fh->ref = 1;
-		F_CLR(fh, WT_FH_IN_MEMORY);
 
 		WT_TRET(__wt_close(session, &fh));
 	}