Import wiredtiger-wiredtiger-mongodb-3.2.0-rc3-199-g4d72349.tar.gz from wiredtiger branch mongodb-3.2

ref: 4898aa4..4d72349

2cf57a6  SERVER-21585 Don't use the lookaside file until the cache is stuck full.
4ad8df7  SERVER-21585 Clean up LAS based on the current count of inserted records

6 files changed, 59 insertions, 47 deletions

diff --git a/src/third_party/wiredtiger/src/btree/bt_read.c b/src/third_party/wiredtiger/src/btree/bt_read.c
index 389ac761c5b..18fd87e78ff 100644
--- a/src/third_party/wiredtiger/src/btree/bt_read.c
+++ b/src/third_party/wiredtiger/src/btree/bt_read.c
@@ -20,9 +20,12 @@ __wt_las_remove_block(WT_SESSION_IMPL *session,
 	WT_DECL_ITEM(las_key);
 	WT_DECL_RET;
 	uint64_t las_counter, las_txnid;
+	int64_t remove_cnt;
 	uint32_t las_id;
 	int exact;
 
+	remove_cnt = 0;
+
 	WT_ERR(__wt_scr_alloc(session, 0, &las_addr));
 	WT_ERR(__wt_scr_alloc(session, 0, &las_key));
 
@@ -56,11 +59,24 @@ __wt_las_remove_block(WT_SESSION_IMPL *session,
 		 * remains positioned in that case.
 		 */
 		WT_ERR(cursor->remove(cursor));
+		++remove_cnt;
 	}
 	WT_ERR_NOTFOUND_OK(ret);
 
 err:	__wt_scr_free(session, &las_addr);
 	__wt_scr_free(session, &las_key);
+
+	/*
+	 * If there were races to remove records, we can over-count.  All
+	 * arithmetic is signed, so underflow isn't fatal, but check anyway so
+	 * we don't skew low over time.
+	 */
+	if (remove_cnt > S2C(session)->las_record_cnt)
+		S2C(session)->las_record_cnt = 0;
+	else if (remove_cnt > 0)
+		(void)__wt_atomic_subi64(
+		    &S2C(session)->las_record_cnt, remove_cnt);
+
 	return (ret);
 }
 
diff --git a/src/third_party/wiredtiger/src/cache/cache_las.c b/src/third_party/wiredtiger/src/cache/cache_las.c
index 2eb406c2af8..d3a0265c13a 100644
--- a/src/third_party/wiredtiger/src/cache/cache_las.c
+++ b/src/third_party/wiredtiger/src/cache/cache_las.c
@@ -267,12 +267,14 @@ __wt_las_sweep(WT_SESSION_IMPL *session)
 	WT_DECL_RET;
 	WT_ITEM *key;
 	uint64_t cnt, las_counter, las_txnid;
+	int64_t remove_cnt;
 	uint32_t las_id, session_flags;
 	int notused;
 
 	conn = S2C(session);
 	cursor = NULL;
 	key = &conn->las_sweep_key;
+	remove_cnt = 0;
 	session_flags = 0;		/* [-Werror=maybe-uninitialized] */
 
 	WT_ERR(__wt_scr_alloc(session, 0, &las_addr));
@@ -285,9 +287,19 @@ __wt_las_sweep(WT_SESSION_IMPL *session)
 	 * from the last call (we don't care if we're before or after the key,
 	 * just roughly in the same spot is fine).
 	 */
-	if (conn->las_sweep_call != 0 && key->data != NULL) {
+	if (key->size != 0) {
 		__wt_cursor_set_raw_key(cursor, key);
-		if ((ret = cursor->search_near(cursor, &notused)) != 0)
+		ret = cursor->search_near(cursor, &notused);
+
+		/*
+		 * Don't search for the same key twice; if we don't set a new
+		 * key below, it's because we've reached the end of the table
+		 * and we want the next pass to start at the beginning of the
+		 * table. Searching for the same key could leave us stuck at
+		 * the end of the table, repeatedly checking the same rows.
+		 */
+		key->size = 0;
+		if (ret != 0)
 			goto srch_notfound;
 	}
 
@@ -303,20 +315,11 @@ __wt_las_sweep(WT_SESSION_IMPL *session)
 	 * but possibly better, alternative might be to review all lookaside
 	 * blocks in the cache in order to get rid of them, and slowly review
 	 * lookaside blocks that have already been evicted.
-	 *
-	 * We can't know for sure how many records are in the lookaside table,
-	 * the cursor insert and remove statistics aren't updated atomically.
-	 * Start with reviewing 100 rows, and if it takes more than the target
-	 * number of calls to finish, increase the number of rows checked on
-	 * each call; if it takes less than the target calls to finish, then
-	 * decrease the number of rows reviewed on each call (but never less
-	 * than 100).
 	 */
-#define	WT_SWEEP_LOOKASIDE_MIN_CNT	100
-#define	WT_SWEEP_LOOKASIDE_PASS_TARGET	 30
-	++conn->las_sweep_call;
-	if ((cnt = conn->las_sweep_cnt) < WT_SWEEP_LOOKASIDE_MIN_CNT)
-		cnt = conn->las_sweep_cnt = WT_SWEEP_LOOKASIDE_MIN_CNT;
+	cnt = (uint64_t)WT_MAX(100, conn->las_record_cnt / 30);
+
+	/* Discard pages we read as soon as we're done with them. */
+	F_SET(session, WT_SESSION_NO_CACHE);
 
 	/* Walk the file. */
 	for (; cnt > 0 && (ret = cursor->next(cursor)) == 0; --cnt) {
@@ -345,28 +348,13 @@ __wt_las_sweep(WT_SESSION_IMPL *session)
 		 * another thread remove the record before we do, and the cursor
 		 * remains positioned in that case.
 		 */
-		if (__wt_txn_visible_all(session, las_txnid))
+		if (__wt_txn_visible_all(session, las_txnid)) {
 			WT_ERR(cursor->remove(cursor));
-	}
-
-	/*
-	 * When reaching the lookaside table end or the target number of calls,
-	 * adjust the row count. Decrease/increase the row count depending on
-	 * if the number of calls is less/more than the target.
-	 */
-	if (ret == WT_NOTFOUND ||
-	    conn->las_sweep_call > WT_SWEEP_LOOKASIDE_PASS_TARGET) {
-		if (conn->las_sweep_call < WT_SWEEP_LOOKASIDE_PASS_TARGET &&
-		    conn->las_sweep_cnt > WT_SWEEP_LOOKASIDE_MIN_CNT)
-			conn->las_sweep_cnt -= WT_SWEEP_LOOKASIDE_MIN_CNT;
-		if (conn->las_sweep_call > WT_SWEEP_LOOKASIDE_PASS_TARGET)
-			conn->las_sweep_cnt += WT_SWEEP_LOOKASIDE_MIN_CNT;
+			++remove_cnt;
+		}
 	}
 
 srch_notfound:
-	if (ret == WT_NOTFOUND)
-		conn->las_sweep_call = 0;
-
 	WT_ERR_NOTFOUND_OK(ret);
 
 	if (0) {
@@ -375,6 +363,18 @@ err:		__wt_buf_free(session, key);
 
 	WT_TRET(__wt_las_cursor_close(session, &cursor, session_flags));
 
+	/*
+	 * If there were races to remove records, we can over-count.  All
+	 * arithmetic is signed, so underflow isn't fatal, but check anyway so
+	 * we don't skew low over time.
+	 */
+	if (remove_cnt > S2C(session)->las_record_cnt)
+		S2C(session)->las_record_cnt = 0;
+	else if (remove_cnt > 0)
+		(void)__wt_atomic_subi64(&conn->las_record_cnt, remove_cnt);
+
+	F_CLR(session, WT_SESSION_NO_CACHE);
+
 	__wt_scr_free(session, &las_addr);
 	__wt_scr_free(session, &las_key);
 
diff --git a/src/third_party/wiredtiger/src/evict/evict_page.c b/src/third_party/wiredtiger/src/evict/evict_page.c
index 94c969fa5bb..2e5f82d1ff8 100644
--- a/src/third_party/wiredtiger/src/evict/evict_page.c
+++ b/src/third_party/wiredtiger/src/evict/evict_page.c
@@ -473,7 +473,8 @@ __evict_review(
 			LF_SET(WT_EVICT_IN_MEMORY | WT_EVICT_UPDATE_RESTORE);
 		else if (page->read_gen == WT_READGEN_OLDEST)
 			LF_SET(WT_EVICT_UPDATE_RESTORE);
-		else if (__wt_eviction_aggressive(session))
+		else if (F_ISSET(session, WT_SESSION_INTERNAL) &&
+		    F_ISSET(S2C(session)->cache, WT_CACHE_STUCK))
 			LF_SET(WT_EVICT_LOOKASIDE);
 	}
 
diff --git a/src/third_party/wiredtiger/src/include/cache.i b/src/third_party/wiredtiger/src/include/cache.i
index a95138c3f0f..7cbd72853c3 100644
--- a/src/third_party/wiredtiger/src/include/cache.i
+++ b/src/third_party/wiredtiger/src/include/cache.i
@@ -131,17 +131,6 @@ __wt_session_can_wait(WT_SESSION_IMPL *session)
 }
 
 /*
- * __wt_eviction_aggressive --
- *	Return if the eviction server is running in aggressive mode.
- */
-static inline bool
-__wt_eviction_aggressive(WT_SESSION_IMPL *session)
-{
-	return (FLD_ISSET(
-	    S2C(session)->cache->state, WT_EVICT_PASS_AGGRESSIVE));
-}
-
-/*
  * __wt_eviction_dirty_target --
  *	Return if the eviction server is running to reduce the number of dirty
  * pages (versus running to discard pages from the cache).
diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h
index 3e8d3705373..2367f5a0035 100644
--- a/src/third_party/wiredtiger/src/include/connection.h
+++ b/src/third_party/wiredtiger/src/include/connection.h
@@ -383,8 +383,7 @@ struct __wt_connection_impl {
 	bool		 las_written;	/* Lookaside table has been written */
 
 	WT_ITEM		 las_sweep_key;	/* Sweep server's saved key */
-	int		 las_sweep_call;/* Sweep server's call count */
-	uint64_t	 las_sweep_cnt;	/* Sweep server's per-call row count */
+	int64_t		 las_record_cnt;/* Count of lookaside records */
 
 					/* Locked: collator list */
 	TAILQ_HEAD(__wt_coll_qh, __wt_named_collator) collqh;
diff --git a/src/third_party/wiredtiger/src/reconcile/rec_write.c b/src/third_party/wiredtiger/src/reconcile/rec_write.c
index 6d53230e9e0..5cd8b10c06f 100644
--- a/src/third_party/wiredtiger/src/reconcile/rec_write.c
+++ b/src/third_party/wiredtiger/src/reconcile/rec_write.c
@@ -3331,6 +3331,7 @@ __rec_update_las(WT_SESSION_IMPL *session,
 	WT_SAVE_UPD *list;
 	WT_UPDATE *upd;
 	uint64_t las_counter;
+	int64_t insert_cnt;
 	uint32_t i, session_flags, slot;
 	uint8_t *p;
 
@@ -3338,6 +3339,7 @@ __rec_update_las(WT_SESSION_IMPL *session,
 	WT_CLEAR(las_addr);
 	WT_CLEAR(las_value);
 	page = r->page;
+	insert_cnt = 0;
 
 	/*
 	 * We're writing lookaside records: start instantiating them on pages
@@ -3434,11 +3436,16 @@ __rec_update_las(WT_SESSION_IMPL *session,
 			    cursor, upd->txnid, upd->size, &las_value);
 
 			WT_ERR(cursor->insert(cursor));
+			++insert_cnt;
 		} while ((upd = upd->next) != NULL);
 	}
 
 err:	WT_TRET(__wt_las_cursor_close(session, &cursor, session_flags));
 
+	if (insert_cnt > 0)
+		(void)__wt_atomic_addi64(
+		    &S2C(session)->las_record_cnt, insert_cnt);
+
 	__wt_scr_free(session, &key);
 	return (ret);
 }