Merge branch 'mongodb-3.8' into mongodb-3.6

13 files changed, 85 insertions, 29 deletions

diff --git a/dist/api_data.py b/dist/api_data.py
index a8b1a30a333..68a8f4908d8 100644
--- a/dist/api_data.py
+++ b/dist/api_data.py
@@ -1303,6 +1303,10 @@ methods = {
         where the application is rolling back locally committed transactions.
         The supplied value should not be older than the current oldest and
         stable timestamps.  See @ref transaction_timestamps'''),
+    Config('force', 'false', r'''
+        set timestamps even if they violate normal ordering requirements.
+        For example allow the \c oldest_timestamp to move backwards''',
+        type='boolean'),
     Config('oldest_timestamp', '', r'''
         future commits and queries will be no earlier than the specified
         timestamp.  Supplied values must be monotonically increasing, any
diff --git a/dist/s_string.ok b/dist/s_string.ok
index 75f237f39fb..1a252fdd615 100644
--- a/dist/s_string.ok
+++ b/dist/s_string.ok
@@ -1165,6 +1165,7 @@ strget
 strlen
 strncpy
 strndup
+strnlen
 strtok
 strtoll
 strtouq
diff --git a/src/btree/bt_compact.c b/src/btree/bt_compact.c
index cbc19df65fa..b3e23a8251c 100644
--- a/src/btree/bt_compact.c
+++ b/src/btree/bt_compact.c
@@ -58,7 +58,7 @@ __compact_rewrite(WT_SESSION_IMPL *session, WT_REF *ref, bool *skipp)
 	if (mod->rec_result == WT_PM_REC_MULTIBLOCK)
 		for (multi = mod->mod_multi,
 		    i = 0; i < mod->mod_multi_entries; ++multi, ++i) {
-			if (multi->disk_image != NULL)
+			if (multi->addr.addr == NULL)
 				continue;
 			if ((ret = bm->compact_page_skip(bm, session,
 			    multi->addr.addr, multi->addr.size, skipp)) != 0)
diff --git a/src/config/config_def.c b/src/config/config_def.c
index f0e1dc1f701..14234badcfa 100644
--- a/src/config/config_def.c
+++ b/src/config/config_def.c
@@ -191,6 +191,7 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = {
 
 static const WT_CONFIG_CHECK confchk_WT_CONNECTION_set_timestamp[] = {
 	{ "commit_timestamp", "string", NULL, NULL, NULL, 0 },
+	{ "force", "boolean", NULL, NULL, NULL, 0 },
 	{ "oldest_timestamp", "string", NULL, NULL, NULL, 0 },
 	{ "stable_timestamp", "string", NULL, NULL, NULL, 0 },
 	{ NULL, NULL, NULL, NULL, NULL, 0 }
@@ -1194,8 +1195,9 @@ static const WT_CONFIG_ENTRY config_entries[] = {
 	  NULL, 0
 	},
 	{ "WT_CONNECTION.set_timestamp",
-	  "commit_timestamp=,oldest_timestamp=,stable_timestamp=",
-	  confchk_WT_CONNECTION_set_timestamp, 3
+	  "commit_timestamp=,force=false,oldest_timestamp=,"
+	  "stable_timestamp=",
+	  confchk_WT_CONNECTION_set_timestamp, 4
 	},
 	{ "WT_CURSOR.close",
 	  "",
diff --git a/src/docs/schema.dox b/src/docs/schema.dox
index 65ad7f6919c..4427111ba2a 100644
--- a/src/docs/schema.dox
+++ b/src/docs/schema.dox
@@ -94,7 +94,10 @@ a size, that indicates the number of bytes to store; the default is a
 length of 1 byte.
 
 The \c 'S' type is encoded as a C language string terminated by a
-NUL character.
+NUL character. When preceded by a size, that indicates the maximum number of
+bytes the string can store. In a string with characters less than the
+specified size, the remaining bytes are NUL padded. If the supplied string
+is longer than the specified size, it will be stored without a trailing NUL.
 @m_if{java}
 Because of this, the associated Java String may not contain the NUL character.
 @m_endif
diff --git a/src/include/btree.i b/src/include/btree.i
index 35cb868ac26..19b300908b1 100644
--- a/src/include/btree.i
+++ b/src/include/btree.i
@@ -1425,15 +1425,19 @@ __wt_page_release(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags)
 	 * tree, then perform a general check if eviction will be possible.
 	 */
 	page = ref->page;
-	if (!WT_READGEN_EVICT_SOON(page->read_gen) ||
-	    LF_ISSET(WT_READ_NO_SPLIT) ||
-	    btree->evict_disabled > 0 ||
-	    !__wt_page_can_evict(session, ref, &inmem_split) ||
-	    (F_ISSET(session, WT_SESSION_NO_RECONCILE) && !inmem_split))
-		return (__wt_hazard_clear(session, ref));
-
-	WT_RET_BUSY_OK(__wt_page_release_evict(session, ref));
-	return (0);
+	if (WT_READGEN_EVICT_SOON(page->read_gen) &&
+	    btree->evict_disabled == 0 &&
+	    __wt_page_can_evict(session, ref, &inmem_split)) {
+		if ((LF_ISSET(WT_READ_NO_SPLIT) || (!inmem_split &&
+		    F_ISSET(session, WT_SESSION_NO_RECONCILE))))
+			__wt_page_evict_urgent(session, ref);
+		else {
+			WT_RET_BUSY_OK(__wt_page_release_evict(session, ref));
+			return (0);
+		}
+	}
+
+	return (__wt_hazard_clear(session, ref));
 }
 
 /*
diff --git a/src/include/extern.h b/src/include/extern.h
index 17267861717..ce9e1e57a47 100644
--- a/src/include/extern.h
+++ b/src/include/extern.h
@@ -824,7 +824,7 @@ extern int __wt_timestamp_to_hex_string( WT_SESSION_IMPL *session, char *hex_tim
 extern void __wt_verbose_timestamp(WT_SESSION_IMPL *session, const wt_timestamp_t *ts, const char *msg);
 extern int __wt_txn_parse_timestamp(WT_SESSION_IMPL *session, const char *name, wt_timestamp_t *timestamp, WT_CONFIG_ITEM *cval) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
 extern int __wt_txn_global_query_timestamp( WT_SESSION_IMPL *session, char *hex_timestamp, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
-extern int __wt_txn_update_pinned_timestamp(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
+extern int __wt_txn_update_pinned_timestamp(WT_SESSION_IMPL *session, bool force) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
 extern int __wt_txn_global_set_timestamp(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
 extern int __wt_timestamp_validate(WT_SESSION_IMPL *session, const char *name, wt_timestamp_t *ts, WT_CONFIG_ITEM *cval, bool cmp_oldest, bool cmp_stable, bool cmp_commit) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
 extern int __wt_txn_set_timestamp(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
diff --git a/src/include/misc.i b/src/include/misc.i
index dbb921f0946..bedd3121037 100644
--- a/src/include/misc.i
+++ b/src/include/misc.i
@@ -41,6 +41,20 @@ __wt_strdup(WT_SESSION_IMPL *session, const char *str, void *retp)
 }
 
 /*
+ * __wt_strnlen --
+ *      Determine the length of a fixed-size string
+ */
+static inline size_t
+__wt_strnlen(const char *s, size_t maxlen)
+{
+	size_t i;
+
+	for (i = 0; i < maxlen && *s != '\0'; i++, s++)
+		;
+	return (i);
+}
+
+/*
  * __wt_snprintf --
  *	snprintf convenience function, ignoring the returned size.
  */
diff --git a/src/include/packing.i b/src/include/packing.i
index e1cf158c660..d5181738fbd 100644
--- a/src/include/packing.i
+++ b/src/include/packing.i
@@ -342,15 +342,20 @@ __pack_write(
 		*pp += pv->size;
 		break;
 	case 'S':
-		s = strlen(pv->u.s);
+		/*
+		 * When preceded by a size, that indicates the maximum number
+		 * of bytes the string can store, this does not include the
+		 * terminating NUL character. In a string with characters
+		 * less than the specified size, the remaining bytes are
+		 * NULL padded.
+		 */
 		if (pv->havesize) {
-			if (pv->size < s) {
-				s = pv->size;
-				pad = 0;
-			} else
-				pad = pv->size - s;
-		} else
+			s = __wt_strnlen(pv->u.s, pv->size);
+			pad = (s < pv->size) ? pv->size - s : 0;
+		} else {
+			s = strlen(pv->u.s);
 			pad = 1;
+		}
 		WT_SIZE_CHECK_PACK(s + pad, maxlen);
 		if (s > 0)
 			memcpy(*pp, pv->u.s, s);
diff --git a/src/include/wiredtiger.in b/src/include/wiredtiger.in
index c24cffe2371..d4c1fa956c3 100644
--- a/src/include/wiredtiger.in
+++ b/src/include/wiredtiger.in
@@ -2341,6 +2341,9 @@ struct __wt_connection {
 	 * supplied value should not be older than the current oldest and stable
 	 * timestamps.  See @ref transaction_timestamps., a string; default
 	 * empty.}
+	 * @config{force, set timestamps even if they violate normal ordering
+	 * requirements.  For example allow the \c oldest_timestamp to move
+	 * backwards., a boolean flag; default \c false.}
 	 * @config{oldest_timestamp, future commits and queries will be no
 	 * earlier than the specified timestamp.  Supplied values must be
 	 * monotonically increasing\, any attempt to set the value to older than
diff --git a/src/txn/txn.c b/src/txn/txn.c
index c461e5f8e45..fb61b037723 100644
--- a/src/txn/txn.c
+++ b/src/txn/txn.c
@@ -287,7 +287,7 @@ __wt_txn_update_oldest(WT_SESSION_IMPL *session, uint32_t flags)
 #ifdef HAVE_TIMESTAMPS
 	/* Try to move the pinned timestamp forward. */
 	if (strict)
-		WT_RET(__wt_txn_update_pinned_timestamp(session));
+		WT_RET(__wt_txn_update_pinned_timestamp(session, false));
 #endif
 
 	/*
diff --git a/src/txn/txn_timestamp.c b/src/txn/txn_timestamp.c
index 5a39a6d84dc..6735946fbb3 100644
--- a/src/txn/txn_timestamp.c
+++ b/src/txn/txn_timestamp.c
@@ -292,7 +292,7 @@ __wt_txn_global_query_timestamp(
  *	maintained for current or future readers).
  */
 int
-__wt_txn_update_pinned_timestamp(WT_SESSION_IMPL *session)
+__wt_txn_update_pinned_timestamp(WT_SESSION_IMPL *session, bool force)
 {
 	WT_DECL_RET;
 	WT_TXN_GLOBAL *txn_global;
@@ -321,7 +321,7 @@ __wt_txn_update_pinned_timestamp(WT_SESSION_IMPL *session)
 	} else
 		__wt_timestamp_set(&pinned_timestamp, &active_timestamp);
 
-	if (txn_global->has_pinned_timestamp) {
+	if (txn_global->has_pinned_timestamp && !force) {
 		WT_WITH_TIMESTAMP_READLOCK(session, &txn_global->rwlock,
 		    __wt_timestamp_set(
 			&last_pinned_timestamp, &txn_global->pinned_timestamp));
@@ -332,7 +332,7 @@ __wt_txn_update_pinned_timestamp(WT_SESSION_IMPL *session)
 	}
 
 	__wt_writelock(session, &txn_global->rwlock);
-	if (!txn_global->has_pinned_timestamp || __wt_timestamp_cmp(
+	if (!txn_global->has_pinned_timestamp || force || __wt_timestamp_cmp(
 	    &txn_global->pinned_timestamp, &pinned_timestamp) < 0) {
 		__wt_timestamp_set(
 		    &txn_global->pinned_timestamp, &pinned_timestamp);
@@ -377,11 +377,14 @@ __wt_txn_global_set_timestamp(WT_SESSION_IMPL *session, const char *cfg[])
 
 #ifdef HAVE_TIMESTAMPS
 	{
+	WT_CONFIG_ITEM cval;
 	WT_TXN_GLOBAL *txn_global;
 	wt_timestamp_t commit_ts, oldest_ts, stable_ts;
 	wt_timestamp_t last_oldest_ts, last_stable_ts;
+	bool force;
 
 	txn_global = &S2C(session)->txn_global;
+
 	/*
 	 * Parsing will initialize the timestamp to zero even if
 	 * it is not configured.
@@ -393,6 +396,13 @@ __wt_txn_global_set_timestamp(WT_SESSION_IMPL *session, const char *cfg[])
 	WT_RET(__wt_txn_parse_timestamp(
 	    session, "stable", &stable_ts, &stable_cval));
 
+	WT_RET(__wt_config_gets_def(session,
+	    cfg, "force", 0, &cval));
+	force = cval.val != 0;
+
+	if (force)
+		goto set;
+
 	__wt_readlock(session, &txn_global->rwlock);
 
 	__wt_timestamp_set(&last_oldest_ts, &txn_global->oldest_timestamp);
@@ -460,7 +470,7 @@ __wt_txn_global_set_timestamp(WT_SESSION_IMPL *session, const char *cfg[])
 	if (!has_commit && !has_oldest && !has_stable)
 		return (0);
 
-	__wt_writelock(session, &txn_global->rwlock);
+set:	__wt_writelock(session, &txn_global->rwlock);
 	/*
 	 * This method can be called from multiple threads, check that we are
 	 * moving the global timestamps forwards.
@@ -479,7 +489,7 @@ __wt_txn_global_set_timestamp(WT_SESSION_IMPL *session, const char *cfg[])
 	}
 
 	if (has_oldest && (!txn_global->has_oldest_timestamp ||
-	    __wt_timestamp_cmp(
+	    force || __wt_timestamp_cmp(
 	    &oldest_ts, &txn_global->oldest_timestamp) > 0)) {
 		__wt_timestamp_set(&txn_global->oldest_timestamp, &oldest_ts);
 		txn_global->has_oldest_timestamp = true;
@@ -489,7 +499,7 @@ __wt_txn_global_set_timestamp(WT_SESSION_IMPL *session, const char *cfg[])
 	}
 
 	if (has_stable && (!txn_global->has_stable_timestamp ||
-	    __wt_timestamp_cmp(
+	    force || __wt_timestamp_cmp(
 	    &stable_ts, &txn_global->stable_timestamp) > 0)) {
 		__wt_timestamp_set(&txn_global->stable_timestamp, &stable_ts);
 		txn_global->has_stable_timestamp = true;
@@ -500,7 +510,7 @@ __wt_txn_global_set_timestamp(WT_SESSION_IMPL *session, const char *cfg[])
 	__wt_writeunlock(session, &txn_global->rwlock);
 
 	if (has_oldest || has_stable)
-		WT_RET(__wt_txn_update_pinned_timestamp(session));
+		WT_RET(__wt_txn_update_pinned_timestamp(session, force));
 	}
 	return (0);
 #else
diff --git a/test/suite/test_timestamp09.py b/test/suite/test_timestamp09.py
index b79521329e7..0e3a6b8dab3 100644
--- a/test/suite/test_timestamp09.py
+++ b/test/suite/test_timestamp09.py
@@ -186,5 +186,15 @@ class test_timestamp09(wttest.WiredTigerTestCase, suite_subprocess):
         self.assertEqual(c[8], 8)
         self.session.commit_transaction()
 
+        # We can move the oldest timestamp backwards with "force"
+        self.conn.set_timestamp(
+            'oldest_timestamp=' + timestamp_str(5) + ',force')
+        self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
+            lambda: self.session.begin_transaction('read_timestamp=' +
+                timestamp_str(4)),
+                '/older than oldest timestamp/')
+        self.session.begin_transaction('read_timestamp=' + timestamp_str(6))
+        self.session.commit_transaction()
+
 if __name__ == '__main__':
     wttest.run()