WT-3673 Fix a bug where opening the lookaside table can race with the statistics server (#3746)

7 files changed, 29 insertions, 49 deletions

diff --git a/dist/flags.py b/dist/flags.py
index 7ddbff62a63..70602333ad5 100644
--- a/dist/flags.py
+++ b/dist/flags.py
@@ -106,7 +106,7 @@ flags = {
         'CONN_EVICTION_NO_LOOKASIDE',
         'CONN_EVICTION_RUN',
         'CONN_IN_MEMORY',
-        'CONN_LAS_OPEN',
+        'CONN_LOOKASIDE_OPEN',
         'CONN_LEAK_MEMORY',
         'CONN_LSM_MERGE',
         'CONN_PANIC',
diff --git a/src/cache/cache_las.c b/src/cache/cache_las.c
index 00dafb680da..13516d80c58 100644
--- a/src/cache/cache_las.c
+++ b/src/cache/cache_las.c
@@ -27,7 +27,7 @@ __wt_las_stats_update(WT_SESSION_IMPL *session)
 	 * table data-source statistics. If there's no lookaside table, values
 	 * remain 0.
 	 */
-	if (!F_ISSET(conn, WT_CONN_LAS_OPEN))
+	if (!F_ISSET(conn, WT_CONN_LOOKASIDE_OPEN))
 		return;
 
 	/*
@@ -89,24 +89,19 @@ __wt_las_create(WT_SESSION_IMPL *session)
 	WT_RET(__wt_session_create(session, WT_LAS_URI, WT_LAS_FORMAT));
 
 	/*
-	 * Flag that the lookaside table has been created (before creating the
-	 * connection's lookaside table session, it checks before creating a
-	 * lookaside table cursor.
+	 * Open a shared internal session and cursor used for the lookaside
+	 * table. This session should never be tapped for eviction.
 	 */
-	F_SET(conn, WT_CONN_LAS_OPEN);
-
-	/*
-	 * Open a shared internal session used to access the lookaside table.
-	 * This session should never be tapped for eviction.
-	 */
-	session_flags = WT_SESSION_LOOKASIDE_CURSOR | WT_SESSION_NO_EVICTION;
-	WT_ERR(__wt_open_internal_session(
+	session_flags = WT_SESSION_NO_EVICTION;
+	WT_RET(__wt_open_internal_session(
 	    conn, "lookaside table", true, session_flags, &conn->las_session));
+	WT_RET(__wt_las_cursor_open(conn->las_session));
 
-	return (0);
+	/* The statistics server is already running, make sure we don't race. */
+	WT_WRITE_BARRIER();
+	F_SET(conn, WT_CONN_LOOKASIDE_OPEN);
 
-err:	F_CLR(conn, WT_CONN_LAS_OPEN);
-	return (ret);
+	return (0);
 }
 
 /*
@@ -122,6 +117,7 @@ __wt_las_destroy(WT_SESSION_IMPL *session)
 
 	conn = S2C(session);
 
+	F_CLR(conn, WT_CONN_LOOKASIDE_OPEN);
 	if (conn->las_session == NULL)
 		return (0);
 
@@ -138,15 +134,16 @@ __wt_las_destroy(WT_SESSION_IMPL *session)
  *	Open a new lookaside table cursor.
  */
 int
-__wt_las_cursor_open(WT_SESSION_IMPL *session, WT_CURSOR **cursorp)
+__wt_las_cursor_open(WT_SESSION_IMPL *session)
 {
 	WT_BTREE *btree;
+	WT_CURSOR *cursor;
 	WT_DECL_RET;
 	const char *open_cursor_cfg[] = {
 	    WT_CONFIG_BASE(session, WT_SESSION_open_cursor), NULL };
 
 	WT_WITHOUT_DHANDLE(session, ret = __wt_open_cursor(
-	    session, WT_LAS_URI, NULL, open_cursor_cfg, cursorp));
+	    session, WT_LAS_URI, NULL, open_cursor_cfg, &cursor));
 	WT_RET(ret);
 
 	/*
@@ -154,7 +151,7 @@ __wt_las_cursor_open(WT_SESSION_IMPL *session, WT_CURSOR **cursorp)
 	 * we don't always switch the LAS handle in to the session before
 	 * entering this function.
 	 */
-	btree = ((WT_CURSOR_BTREE *)(*cursorp))->btree;
+	btree = ((WT_CURSOR_BTREE *)cursor)->btree;
 
 	/* Track the lookaside file ID. */
 	if (S2C(session)->las_fileid == 0)
@@ -176,6 +173,9 @@ __wt_las_cursor_open(WT_SESSION_IMPL *session, WT_CURSOR **cursorp)
 	if (!F_ISSET(btree, WT_BTREE_NO_LOGGING))
 		F_SET(btree, WT_BTREE_NO_LOGGING);
 
+	session->las_cursor = cursor;
+	F_SET(session, WT_SESSION_LOOKASIDE_CURSOR);
+
 	return (0);
 }
 
diff --git a/src/evict/evict_file.c b/src/evict/evict_file.c
index bd70de8bddb..147b615c0ab 100644
--- a/src/evict/evict_file.c
+++ b/src/evict/evict_file.c
@@ -53,7 +53,7 @@ __wt_evict_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop)
 	 * the tree are removed.
 	 */
 	if (F_ISSET(dhandle, WT_DHANDLE_DEAD) &&
-	    F_ISSET(S2C(session), WT_CONN_LAS_OPEN) &&
+	    F_ISSET(S2C(session), WT_CONN_LOOKASIDE_OPEN) &&
 	    !F_ISSET(btree, WT_BTREE_LOOKASIDE)) {
 		WT_ASSERT(session, !WT_IS_METADATA(dhandle));
 
diff --git a/src/include/extern.h b/src/include/extern.h
index 5ffae3111cc..fc0b5135882 100644
--- a/src/include/extern.h
+++ b/src/include/extern.h
@@ -203,7 +203,7 @@ extern int __wt_row_search(WT_SESSION_IMPL *session, WT_ITEM *srch_key, WT_REF *
 extern void __wt_las_stats_update(WT_SESSION_IMPL *session);
 extern int __wt_las_create(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
 extern int __wt_las_destroy(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
-extern int __wt_las_cursor_open(WT_SESSION_IMPL *session, WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
+extern int __wt_las_cursor_open(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
 extern void __wt_las_cursor( WT_SESSION_IMPL *session, WT_CURSOR **cursorp, uint32_t *session_flags);
 extern int __wt_las_cursor_close( 	WT_SESSION_IMPL *session, WT_CURSOR **cursorp, uint32_t session_flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
 extern int __wt_las_insert_block(WT_SESSION_IMPL *session, WT_PAGE *page, WT_CURSOR *cursor, WT_MULTI *multi, WT_ITEM *key) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
diff --git a/src/include/flags.h b/src/include/flags.h
index ec0ef0a5311..dd98234f9e2 100644
--- a/src/include/flags.h
+++ b/src/include/flags.h
@@ -9,8 +9,8 @@
 #define	WT_CONN_EVICTION_NO_LOOKASIDE			0x00000010
 #define	WT_CONN_EVICTION_RUN				0x00000020
 #define	WT_CONN_IN_MEMORY				0x00000040
-#define	WT_CONN_LAS_OPEN				0x00000080
-#define	WT_CONN_LEAK_MEMORY				0x00000100
+#define	WT_CONN_LEAK_MEMORY				0x00000080
+#define	WT_CONN_LOOKASIDE_OPEN				0x00000100
 #define	WT_CONN_LSM_MERGE				0x00000200
 #define	WT_CONN_PANIC					0x00000400
 #define	WT_CONN_READONLY				0x00000800
diff --git a/src/session/session_api.c b/src/session/session_api.c
index 5f995bded53..fa33b55c936 100644
--- a/src/session/session_api.c
+++ b/src/session/session_api.c
@@ -1971,8 +1971,6 @@ int
 __wt_open_internal_session(WT_CONNECTION_IMPL *conn, const char *name,
     bool open_metadata, uint32_t session_flags, WT_SESSION_IMPL **sessionp)
 {
-	WT_DECL_RET;
-	WT_SESSION *wt_session;
 	WT_SESSION_IMPL *session;
 
 	*sessionp = NULL;
@@ -1989,22 +1987,6 @@ __wt_open_internal_session(WT_CONNECTION_IMPL *conn, const char *name,
 	 */
 	F_SET(session, session_flags | WT_SESSION_INTERNAL);
 
-	/*
-	 * Optionally acquire a lookaside table cursor (or clear caller's flag).
-	 * Acquiring the lookaside table cursor requires various locks; we've
-	 * seen problems in the past where deadlocks happened because sessions
-	 * deadlocked getting the cursor late in the process.  Be defensive,
-	 * get it now.
-	 */
-	if (!F_ISSET(conn, WT_CONN_LAS_OPEN))
-		F_CLR(session, WT_SESSION_LOOKASIDE_CURSOR);
-	if (F_ISSET(session, WT_SESSION_LOOKASIDE_CURSOR) &&
-	    (ret = __wt_las_cursor_open(session, &session->las_cursor)) != 0) {
-		wt_session = &session->iface;
-		WT_TRET(wt_session->close(wt_session, NULL));
-		return (ret);
-	}
-
 	*sessionp = session;
 	return (0);
 }
diff --git a/src/support/thread_group.c b/src/support/thread_group.c
index f5842bea572..ba0c57e9468 100644
--- a/src/support/thread_group.c
+++ b/src/support/thread_group.c
@@ -182,17 +182,15 @@ __thread_group_resize(
 		WT_ERR(__wt_calloc_one(session, &thread));
 		/*
 		 * Threads get their own session and lookaside table cursor
-		 * if the lookaside table is open. Note that threads are
-		 * started during recovery, before the lookaside table is
-		 * created.
+		 * (if the lookaside table is open).
 		 */
-		session_flags = 0;
-		if (LF_ISSET(WT_THREAD_CAN_WAIT))
-			FLD_SET(session_flags, WT_SESSION_CAN_WAIT);
-		if (LF_ISSET(WT_THREAD_LOOKASIDE))
-			FLD_SET(session_flags, WT_SESSION_LOOKASIDE_CURSOR);
+		session_flags =
+		    LF_ISSET(WT_THREAD_CAN_WAIT) ? WT_SESSION_CAN_WAIT : 0;
 		WT_ERR(__wt_open_internal_session(conn, group->name,
 		    false, session_flags, &thread->session));
+		if (LF_ISSET(WT_THREAD_LOOKASIDE) &&
+		    F_ISSET(conn, WT_CONN_LOOKASIDE_OPEN))
+			WT_ERR(__wt_las_cursor_open(thread->session));
 		if (LF_ISSET(WT_THREAD_PANIC_FAIL))
 			F_SET(thread, WT_THREAD_PANIC_FAIL);
 		thread->id = i;