Import wiredtiger: c3eafe83e8e48632d8b221a86ded6b3d949da979 from branch mongodb-4.0

ref: 5e14c6107f..c3eafe83e8
for: 4.1.1

WT-4087       Store current compatibility within WiredTiger

8 files changed, 165 insertions, 46 deletions

diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data
index 12b183ea160..a2204e845f7 100644
--- a/src/third_party/wiredtiger/import.data
+++ b/src/third_party/wiredtiger/import.data
@@ -1,5 +1,5 @@
 {
-    "commit": "5e14c6107f8cae984d73e208fa7ce11341917bf1", 
+    "commit": "c3eafe83e8e48632d8b221a86ded6b3d949da979", 
     "github": "wiredtiger/wiredtiger.git", 
     "vendor": "wiredtiger", 
     "branch": "mongodb-4.0"
diff --git a/src/third_party/wiredtiger/src/conn/conn_reconfig.c b/src/third_party/wiredtiger/src/conn/conn_reconfig.c
index 216637f41bb..45c5754baf0 100644
--- a/src/third_party/wiredtiger/src/conn/conn_reconfig.c
+++ b/src/third_party/wiredtiger/src/conn/conn_reconfig.c
@@ -49,16 +49,22 @@ __wt_conn_compat_config(
 {
 	WT_CONFIG_ITEM cval;
 	WT_CONNECTION_IMPL *conn;
-	uint16_t major, minor;
+	WT_DECL_RET;
+	uint16_t min_major, min_minor, rel_major, rel_minor;
+	char *value;
 	bool txn_active;
 
 	conn = S2C(session);
+	value = NULL;
+
 	WT_RET(__wt_config_gets(session, cfg, "compatibility.release", &cval));
 	if (cval.len == 0) {
-		conn->compat_major = WIREDTIGER_VERSION_MAJOR;
-		conn->compat_minor = WIREDTIGER_VERSION_MINOR;
+		rel_major = WIREDTIGER_VERSION_MAJOR;
+		rel_minor = WIREDTIGER_VERSION_MINOR;
+		F_CLR(conn, WT_CONN_COMPATIBILITY);
 	} else {
-		WT_RET(__conn_compat_parse(session, &cval, &major, &minor));
+		WT_RET(__conn_compat_parse(
+		    session, &cval, &rel_major, &rel_minor));
 
 		/*
 		 * We're doing an upgrade or downgrade, check whether
@@ -69,16 +75,14 @@ __wt_conn_compat_config(
 			WT_RET_MSG(session, ENOTSUP,
 			    "system must be quiescent"
 			    " for upgrade or downgrade");
-		conn->compat_major = major;
-		conn->compat_minor = minor;
+		F_SET(conn, WT_CONN_COMPATIBILITY);
 	}
-
 	/*
-	 * The required minimum cannot be set via reconfigure and it is
-	 * meaningless on a newly created database. We're done in those cases.
+	 * If we're a reconfigure and the user did not set any compatibility,
+	 * we're done.
 	 */
-	if (reconfig || conn->is_new)
-		return (0);
+	if (reconfig && !F_ISSET(conn, WT_CONN_COMPATIBILITY))
+		goto done;
 
 	/*
 	 * The minimum required version for existing files is only available
@@ -87,24 +91,92 @@ __wt_conn_compat_config(
 	WT_RET(__wt_config_gets(session,
 	    cfg, "compatibility.require_min", &cval));
 	if (cval.len == 0) {
-		conn->compat_req_major = WT_CONN_COMPAT_NONE;
-		conn->compat_req_minor = WT_CONN_COMPAT_NONE;
-		return (0);
-	}
-	WT_RET(__conn_compat_parse(session, &cval, &major, &minor));
+		min_major = WT_CONN_COMPAT_NONE;
+		min_minor = WT_CONN_COMPAT_NONE;
+	} else
+		WT_RET(__conn_compat_parse(
+		    session, &cval, &min_major, &min_minor));
 
 	/*
 	 * The minimum required must be less than or equal to the compatibility
-	 * release if one was set.
+	 * release if one was set. This is on an open and we're checking the
+	 * two against each other. We'll check against what was saved on a
+	 * restart later.
 	 */
-	if ((major > conn->compat_major) ||
-	    (major == conn->compat_major && minor > conn->compat_minor))
+	if (!reconfig && F_ISSET(conn, WT_CONN_COMPATIBILITY) &&
+	    min_major != WT_CONN_COMPAT_NONE &&
+	    (min_major > rel_major ||
+	    (min_major == rel_major && min_minor > rel_minor)))
 		WT_RET_MSG(session, ENOTSUP,
-		    "required min cannot be larger than compatibility release");
-	conn->compat_req_major = major;
-	conn->compat_req_minor = minor;
+		    "required min of %" PRIu16 ".%" PRIu16
+		    "cannot be larger than compatibility release %"
+		    PRIu16 ".%" PRIu16,
+		    min_major, min_minor, rel_major, rel_minor);
 
-	return (0);
+	/*
+	 * On a reconfigure, check the new release version against any
+	 * required minimum version set on open.
+	 */
+	if (reconfig && conn->compat_req_major != WT_CONN_COMPAT_NONE &&
+	    (conn->compat_req_major > rel_major ||
+	    (conn->compat_req_major == rel_major &&
+	    conn->compat_req_minor > rel_minor)))
+		WT_RET_MSG(session, ENOTSUP,
+		    "required min of %" PRIu16 ".%" PRIu16
+		    "cannot be larger than requested compatibility release %"
+		    PRIu16 ".%" PRIu16,
+		    conn->compat_req_major, conn->compat_req_minor,
+		    rel_major, rel_minor);
+
+	conn->compat_major = rel_major;
+	conn->compat_minor = rel_minor;
+
+	/*
+	 * Only rewrite the turtle file if this is a reconfig. On startup
+	 * it will get written as part of creating the connection. We do this
+	 * after checking the required minimum version so that we don't rewrite
+	 * the turtle file if there is an error.
+	 */
+	if (reconfig)
+		WT_RET(__wt_metadata_turtle_rewrite(session));
+
+	/*
+	 * The required minimum cannot be set via reconfigure and it is
+	 * meaningless on a newly created database. We're done in those cases.
+	 */
+	if (reconfig || conn->is_new || min_major == WT_CONN_COMPAT_NONE)
+		goto done;
+
+	/*
+	 * Check the minimum required against any saved compatibility version
+	 * in the turtle file saved from an earlier run.
+	 */
+	rel_major = rel_minor = WT_CONN_COMPAT_NONE;
+	if ((ret =
+	    __wt_metadata_search(session, WT_METADATA_COMPAT, &value)) == 0) {
+		WT_ERR(__wt_config_getones(session, value, "major", &cval));
+		rel_major = (uint16_t)cval.val;
+		WT_ERR(__wt_config_getones(session, value, "minor", &cval));
+		rel_minor = (uint16_t)cval.val;
+		if (min_major > rel_major ||
+		    (min_major == rel_major && min_minor > rel_minor))
+			WT_ERR_MSG(session, ENOTSUP,
+			    "required min of %" PRIu16 ".%" PRIu16
+			    "cannot be larger than saved release %"
+			    PRIu16 ".%" PRIu16,
+			    min_major, min_minor, rel_major, rel_minor);
+	} else if (ret == WT_NOTFOUND)
+		ret = 0;
+	else
+		WT_ERR(ret);
+
+	conn->compat_req_major = min_major;
+	conn->compat_req_minor = min_minor;
+done:
+err:	if (value != NULL)
+		__wt_free(session, value);
+
+	return (ret);
 }
 
 /*
diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h
index 41c09261b29..77632646843 100644
--- a/src/third_party/wiredtiger/src/include/connection.h
+++ b/src/third_party/wiredtiger/src/include/connection.h
@@ -475,23 +475,24 @@ struct __wt_connection_impl {
 #define	WT_CONN_CLOSING			0x000008u
 #define	WT_CONN_CLOSING_NO_MORE_OPENS	0x000010u
 #define	WT_CONN_CLOSING_TIMESTAMP	0x000020u
-#define	WT_CONN_EVICTION_NO_LOOKASIDE	0x000040u
-#define	WT_CONN_EVICTION_RUN		0x000080u
-#define	WT_CONN_IN_MEMORY		0x000100u
-#define	WT_CONN_LEAK_MEMORY		0x000200u
-#define	WT_CONN_LOOKASIDE_OPEN		0x000400u
-#define	WT_CONN_LSM_MERGE		0x000800u
-#define	WT_CONN_OPTRACK			0x001000u
-#define	WT_CONN_PANIC			0x002000u
-#define	WT_CONN_READONLY		0x004000u
-#define	WT_CONN_RECOVERING		0x008000u
-#define	WT_CONN_SERVER_ASYNC		0x010000u
-#define	WT_CONN_SERVER_CHECKPOINT	0x020000u
-#define	WT_CONN_SERVER_LOG		0x040000u
-#define	WT_CONN_SERVER_LSM		0x080000u
-#define	WT_CONN_SERVER_STATISTICS	0x100000u
-#define	WT_CONN_SERVER_SWEEP		0x200000u
-#define	WT_CONN_WAS_BACKUP		0x400000u
+#define	WT_CONN_COMPATIBILITY		0x000040u
+#define	WT_CONN_EVICTION_NO_LOOKASIDE	0x000080u
+#define	WT_CONN_EVICTION_RUN		0x000100u
+#define	WT_CONN_IN_MEMORY		0x000200u
+#define	WT_CONN_LEAK_MEMORY		0x000400u
+#define	WT_CONN_LOOKASIDE_OPEN		0x000800u
+#define	WT_CONN_LSM_MERGE		0x001000u
+#define	WT_CONN_OPTRACK			0x002000u
+#define	WT_CONN_PANIC			0x004000u
+#define	WT_CONN_READONLY		0x008000u
+#define	WT_CONN_RECOVERING		0x010000u
+#define	WT_CONN_SERVER_ASYNC		0x020000u
+#define	WT_CONN_SERVER_CHECKPOINT	0x040000u
+#define	WT_CONN_SERVER_LOG		0x080000u
+#define	WT_CONN_SERVER_LSM		0x100000u
+#define	WT_CONN_SERVER_STATISTICS	0x200000u
+#define	WT_CONN_SERVER_SWEEP		0x400000u
+#define	WT_CONN_WAS_BACKUP		0x800000u
 /* AUTOMATIC FLAG VALUE GENERATION STOP */
 	uint32_t flags;
 };
diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h
index b986eff6317..004c3fcd2e3 100644
--- a/src/third_party/wiredtiger/src/include/extern.h
+++ b/src/third_party/wiredtiger/src/include/extern.h
@@ -516,6 +516,7 @@ extern int __wt_ext_metadata_search(WT_EXTENSION_API *wt_api, WT_SESSION *wt_ses
 extern int __wt_ext_metadata_update(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *key, const char *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
 extern int __wt_metadata_get_ckptlist(WT_SESSION *session, const char *name, WT_CKPT **ckptbasep) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
 extern void __wt_metadata_free_ckptlist(WT_SESSION *session, WT_CKPT *ckptbase) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default")));
+extern int __wt_metadata_turtle_rewrite(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
 extern int __wt_metadata_cursor_open(WT_SESSION_IMPL *session, const char *config, WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
 extern int __wt_metadata_cursor(WT_SESSION_IMPL *session, WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
 extern int __wt_metadata_cursor_release(WT_SESSION_IMPL *session, WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
diff --git a/src/third_party/wiredtiger/src/include/meta.h b/src/third_party/wiredtiger/src/include/meta.h
index 97d86b44051..6ca2bb27832 100644
--- a/src/third_party/wiredtiger/src/include/meta.h
+++ b/src/third_party/wiredtiger/src/include/meta.h
@@ -38,6 +38,7 @@
 #define	WT_IS_METADATA(dh)      F_ISSET((dh), WT_DHANDLE_IS_METADATA)
 #define	WT_METAFILE_ID		0			/* Metadata file ID */
 
+#define	WT_METADATA_COMPAT	"Compatibility version"
 #define	WT_METADATA_VERSION	"WiredTiger version"	/* Version keys */
 #define	WT_METADATA_VERSION_STR	"WiredTiger version string"
 
diff --git a/src/third_party/wiredtiger/src/meta/meta_table.c b/src/third_party/wiredtiger/src/meta/meta_table.c
index d338c7769aa..48ff3e9ab32 100644
--- a/src/third_party/wiredtiger/src/meta/meta_table.c
+++ b/src/third_party/wiredtiger/src/meta/meta_table.c
@@ -16,14 +16,18 @@ static bool
 __metadata_turtle(const char *key)
 {
 	switch (key[0]) {
+	case 'C':
+		if (strcmp(key, WT_METADATA_COMPAT) == 0)
+			return (true);
+		break;
 	case 'f':
 		if (strcmp(key, WT_METAFILE_URI) == 0)
 			return (true);
 		break;
 	case 'W':
-		if (strcmp(key, "WiredTiger version") == 0)
+		if (strcmp(key, WT_METADATA_VERSION) == 0)
 			return (true);
-		if (strcmp(key, "WiredTiger version string") == 0)
+		if (strcmp(key, WT_METADATA_VERSION_STR) == 0)
 			return (true);
 		break;
 	}
@@ -31,6 +35,25 @@ __metadata_turtle(const char *key)
 }
 
 /*
+ * __wt_metadata_turtle_rewrite --
+ *	Rewrite the turtle file. We wrap this because the lower functions
+ *	expect a URI key and config value pair for the metadata. This function
+ *	exists to push out the other contents to the turtle file such as a
+ *	change in compatibility information.
+ */
+int
+__wt_metadata_turtle_rewrite(WT_SESSION_IMPL *session)
+{
+	WT_DECL_RET;
+	char *value;
+
+	WT_RET(__wt_metadata_search(session, WT_METAFILE_URI, &value));
+	ret = __wt_metadata_update(session, WT_METAFILE_URI, value);
+	__wt_free(session, value);
+	return (ret);
+}
+
+/*
  * __wt_metadata_cursor_open --
  *	Opens a cursor on the metadata.
  */
diff --git a/src/third_party/wiredtiger/src/meta/meta_turtle.c b/src/third_party/wiredtiger/src/meta/meta_turtle.c
index 8ce13384c0a..d6acaed98fc 100644
--- a/src/third_party/wiredtiger/src/meta/meta_turtle.c
+++ b/src/third_party/wiredtiger/src/meta/meta_turtle.c
@@ -286,9 +286,11 @@ err:	WT_TRET(__wt_fclose(session, &fs));
 
 	/*
 	 * A file error or a missing key/value pair in the turtle file means
-	 * something has gone horribly wrong -- we're done.
+	 * something has gone horribly wrong, except for the compatibility
+	 * setting which is optional.
 	 */
-	return (ret == 0 ? 0 : __wt_illegal_value(session, WT_METADATA_TURTLE));
+	return (ret == 0 || strcmp(key, WT_METADATA_COMPAT) == 0 ? ret :
+	    __wt_illegal_value(session, WT_METADATA_TURTLE));
 }
 
 /*
@@ -298,12 +300,14 @@ err:	WT_TRET(__wt_fclose(session, &fs));
 int
 __wt_turtle_update(WT_SESSION_IMPL *session, const char *key, const char *value)
 {
+	WT_CONNECTION_IMPL *conn;
 	WT_DECL_RET;
 	WT_FSTREAM *fs;
 	int vmajor, vminor, vpatch;
 	const char *version;
 
 	fs = NULL;
+	conn = S2C(session);
 
 	/* Require single-threading. */
 	WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_TURTLE));
@@ -315,6 +319,16 @@ __wt_turtle_update(WT_SESSION_IMPL *session, const char *key, const char *value)
 	WT_RET(__wt_fopen(session, WT_METADATA_TURTLE_SET,
 	    WT_FS_OPEN_CREATE | WT_FS_OPEN_EXCLUSIVE, WT_STREAM_WRITE, &fs));
 
+	/*
+	 * If a compatibility setting has been explicitly set, save it out
+	 * to the turtle file.
+	 */
+	if (F_ISSET(conn, WT_CONN_COMPATIBILITY))
+		WT_ERR(__wt_fprintf(session, fs,
+		    "%s\n" "major=%d,minor=%d\n",
+		    WT_METADATA_COMPAT,
+		    conn->compat_major, conn->compat_minor));
+
 	version = wiredtiger_version(&vmajor, &vminor, &vpatch);
 	WT_ERR(__wt_fprintf(session, fs,
 	    "%s\n%s\n%s\n" "major=%d,minor=%d,patch=%d\n%s\n%s\n",
diff --git a/src/third_party/wiredtiger/test/suite/test_compat02.py b/src/third_party/wiredtiger/test/suite/test_compat02.py
index 8dadfbbbfd8..6bf1fc28c05 100644
--- a/src/third_party/wiredtiger/test/suite/test_compat02.py
+++ b/src/third_party/wiredtiger/test/suite/test_compat02.py
@@ -128,7 +128,8 @@ class test_compat02(wttest.WiredTigerTestCase, suite_subprocess):
         #
         msgunsup = "/unsupported major version/"
         msglog = "/this build requires a minimum version/"
-        msgcompat = "/required min cannot be larger/"
+        msgcompat = "/cannot be larger than compatibility release/"
+        msgsave = "/cannot be larger than saved release/"
         if (self.log_req >= self.future_logv):
             self.pr("EXPECT: " + msgunsup)
             self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
@@ -139,6 +140,12 @@ class test_compat02(wttest.WiredTigerTestCase, suite_subprocess):
             # setting we expect an error.
             self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
                 lambda: self.wiredtiger_open('.', restart_config), msgcompat)
+        elif (self.log_req > self.log_create and self.create_rel != 'none'):
+            self.pr("EXPECT: " + msgsave)
+            # If required minimum is larger than the setting we created the
+            # database with, we expect an error.
+            self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
+                lambda: self.wiredtiger_open('.', restart_config), msgsave)
         elif (self.log_req > self.log_create):
             self.pr("EXPECT: " + msglog)
             # If required minimum is larger than the setting we created the