Import wiredtiger: ad779a189c81c868c31a22667d670d56c8c41cbf from branch mongodb-5.0

ref: 6f517e60fb..ad779a189c
for: 5.0.6

WT-8371       Simplify overflow key block frees
WT-8395       Inconsistent data after upgrade from 4.4.2-4.4.4 to 4.4.8+ and 5.0.2+

12 files changed, 68 insertions, 41 deletions

diff --git a/src/third_party/wiredtiger/INSTALL b/src/third_party/wiredtiger/INSTALL
index b4437c7e864..ef25361a312 100644
--- a/src/third_party/wiredtiger/INSTALL
+++ b/src/third_party/wiredtiger/INSTALL
@@ -1,6 +1,6 @@
-WiredTiger 10.0.1: (April 12, 2021)
+WiredTiger 10.0.2: (November 30, 2021)
 
-This is version 10.0.1 of WiredTiger.
+This is version 10.0.2 of WiredTiger.
 
 Instructions for configuring, building, and installing WiredTiger are available online.
 
diff --git a/src/third_party/wiredtiger/README b/src/third_party/wiredtiger/README
index 57b9e083e23..df921a6d884 100644
--- a/src/third_party/wiredtiger/README
+++ b/src/third_party/wiredtiger/README
@@ -1,6 +1,6 @@
-WiredTiger 10.0.1: (April 12, 2021)
+WiredTiger 10.0.2: (November 30, 2021)
 
-This is version 10.0.1 of WiredTiger.
+This is version 10.0.2 of WiredTiger.
 
 WiredTiger release packages and documentation can be found at:
 
diff --git a/src/third_party/wiredtiger/RELEASE_INFO b/src/third_party/wiredtiger/RELEASE_INFO
index 85f216a7850..954ca8646b1 100644
--- a/src/third_party/wiredtiger/RELEASE_INFO
+++ b/src/third_party/wiredtiger/RELEASE_INFO
@@ -1,6 +1,6 @@
 WIREDTIGER_VERSION_MAJOR=10
 WIREDTIGER_VERSION_MINOR=0
-WIREDTIGER_VERSION_PATCH=1
+WIREDTIGER_VERSION_PATCH=2
 WIREDTIGER_VERSION="$WIREDTIGER_VERSION_MAJOR.$WIREDTIGER_VERSION_MINOR.$WIREDTIGER_VERSION_PATCH"
 
 WIREDTIGER_RELEASE_DATE=`date "+%B %e, %Y"`
diff --git a/src/third_party/wiredtiger/build_posix/aclocal/version-set.m4 b/src/third_party/wiredtiger/build_posix/aclocal/version-set.m4
index f03a1f7819f..53dadedd6ad 100644
--- a/src/third_party/wiredtiger/build_posix/aclocal/version-set.m4
+++ b/src/third_party/wiredtiger/build_posix/aclocal/version-set.m4
@@ -2,8 +2,8 @@ dnl build by dist/s_version
 
 VERSION_MAJOR=10
 VERSION_MINOR=0
-VERSION_PATCH=1
-VERSION_STRING='"WiredTiger 10.0.1: (April 12, 2021)"'
+VERSION_PATCH=2
+VERSION_STRING='"WiredTiger 10.0.2: (November 30, 2021)"'
 
 AC_SUBST(VERSION_MAJOR)
 AC_SUBST(VERSION_MINOR)
diff --git a/src/third_party/wiredtiger/build_posix/aclocal/version.m4 b/src/third_party/wiredtiger/build_posix/aclocal/version.m4
index c76126e107a..4e08838ca75 100644
--- a/src/third_party/wiredtiger/build_posix/aclocal/version.m4
+++ b/src/third_party/wiredtiger/build_posix/aclocal/version.m4
@@ -1,2 +1,2 @@
 dnl WiredTiger product version for AC_INIT.  Maintained by dist/s_version
-10.0.1
+10.0.2
diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data
index 80d01e4d3f4..89c6f6b17fa 100644
--- a/src/third_party/wiredtiger/import.data
+++ b/src/third_party/wiredtiger/import.data
@@ -2,5 +2,5 @@
     "vendor": "wiredtiger",
     "github": "wiredtiger/wiredtiger.git",
     "branch": "mongodb-5.0",
-    "commit": "6f517e60fbc9c8e33032e75d683f53ae62ca531c"
+    "commit": "ad779a189c81c868c31a22667d670d56c8c41cbf"
 }
diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c
index 21b2f0675fa..58ee8677aad 100644
--- a/src/third_party/wiredtiger/src/conn/conn_api.c
+++ b/src/third_party/wiredtiger/src/conn/conn_api.c
@@ -2834,6 +2834,11 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, const char *c
     WT_ERR(__wt_connection_open(conn, cfg));
     session = conn->default_session;
 
+#ifndef WT_STANDALONE_BUILD
+    /* Explicitly set the flag to indicate whether the database that was not shutdown cleanly. */
+    conn->unclean_shutdown = false;
+#endif
+
     /*
      * This function expects the cache to be created so parse this after the rest of the connection
      * is set up.
diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h
index 89283c3cc19..9e9b98d9229 100644
--- a/src/third_party/wiredtiger/src/include/connection.h
+++ b/src/third_party/wiredtiger/src/include/connection.h
@@ -246,6 +246,14 @@ struct __wt_connection_impl {
     uint64_t hash_size;       /* General hash bucket array size */
     int is_new;               /* Connection created database */
 
+    uint32_t recovery_major; /* Database recovery major version */
+    uint32_t recovery_minor; /* Database recovery minor version */
+    uint32_t recovery_patch; /* Database recovery patch version */
+
+#ifndef WT_STANDALONE_BUILD
+    bool unclean_shutdown; /* Flag to indicate the earlier shutdown status */
+#endif
+
     uint16_t compat_major; /* Compatibility major version */
     uint16_t compat_minor; /* Compatibility minor version */
 #define WT_CONN_COMPAT_NONE UINT16_MAX
diff --git a/src/third_party/wiredtiger/src/meta/meta_turtle.c b/src/third_party/wiredtiger/src/meta/meta_turtle.c
index aa2e93623c9..309c80eaa05 100644
--- a/src/third_party/wiredtiger/src/meta/meta_turtle.c
+++ b/src/third_party/wiredtiger/src/meta/meta_turtle.c
@@ -153,7 +153,7 @@ int
 __wt_turtle_validate_version(WT_SESSION_IMPL *session)
 {
     WT_DECL_RET;
-    uint32_t major, minor;
+    uint32_t major, minor, patch;
     char *version_string;
 
     WT_WITH_TURTLE_LOCK(
@@ -162,7 +162,7 @@ __wt_turtle_validate_version(WT_SESSION_IMPL *session)
     if (ret != 0)
         WT_ERR_MSG(session, ret, "Unable to read version string from turtle file");
 
-    if ((ret = sscanf(version_string, "major=%u,minor=%u", &major, &minor)) != 2)
+    if ((ret = sscanf(version_string, "major=%u,minor=%u,patch=%u", &major, &minor, &patch)) != 3)
         WT_ERR_MSG(session, ret, "Unable to parse turtle file version string");
 
     ret = 0;
@@ -171,6 +171,10 @@ __wt_turtle_validate_version(WT_SESSION_IMPL *session)
       (major == WT_MIN_STARTUP_VERSION_MAJOR && minor < WT_MIN_STARTUP_VERSION_MINOR))
         WT_ERR_MSG(session, WT_ERROR, "WiredTiger version incompatible with current binary");
 
+    S2C(session)->recovery_major = major;
+    S2C(session)->recovery_minor = minor;
+    S2C(session)->recovery_patch = patch;
+
 err:
     __wt_free(session, version_string);
     return (ret);
diff --git a/src/third_party/wiredtiger/src/reconcile/rec_row.c b/src/third_party/wiredtiger/src/reconcile/rec_row.c
index 6f68749b614..c0e6207f173 100644
--- a/src/third_party/wiredtiger/src/reconcile/rec_row.c
+++ b/src/third_party/wiredtiger/src/reconcile/rec_row.c
@@ -292,7 +292,7 @@ __wt_rec_row_int(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page)
     WT_REF *ref;
     WT_TIME_AGGREGATE ta;
     size_t size;
-    bool hazard, key_onpage_ovfl;
+    bool hazard;
     const void *p;
 
     btree = S2BT(session);
@@ -336,13 +336,18 @@ __wt_rec_row_int(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page)
          * instantiated, off-page key, we don't bother setting them if that's not possible.
          */
         cell = NULL;
-        key_onpage_ovfl = false;
         ikey = __wt_ref_key_instantiated(ref);
         if (ikey != NULL && ikey->cell_offset != 0) {
             cell = WT_PAGE_REF_OFFSET(page, ikey->cell_offset);
             __wt_cell_unpack_addr(session, page->dsk, cell, kpack);
-            key_onpage_ovfl =
-              F_ISSET(kpack, WT_CELL_UNPACK_OVERFLOW) && kpack->raw != WT_CELL_KEY_OVFL_RM;
+
+            /*
+             * Historically, we stored overflow cookies on internal pages, discard any underlying
+             * blocks. We have a copy to build the key (the key was instantiated when we read the
+             * page into memory), they won't be needed in the future as we're rewriting the page.
+             */
+            if (F_ISSET(kpack, WT_CELL_UNPACK_OVERFLOW) && kpack->raw != WT_CELL_KEY_OVFL_RM)
+                WT_ERR(__wt_ovfl_discard_add(session, page, kpack->cell));
         }
 
         WT_ERR(__wt_rec_child_modify(session, r, ref, &hazard, &state));
@@ -353,14 +358,7 @@ __wt_rec_row_int(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page)
         case WT_CHILD_IGNORE:
             /*
              * Ignored child.
-             *
-             * Overflow keys referencing pages we're not writing are no longer useful, schedule them
-             * for discard. Don't worry about instantiation, internal page keys are always
-             * instantiated. Don't worry about reuse, reusing this key in this reconciliation is
-             * unlikely.
              */
-            if (key_onpage_ovfl)
-                WT_ERR(__wt_ovfl_discard_add(session, page, kpack->cell));
             WT_CHILD_RELEASE_ERR(session, hazard, ref);
             continue;
 
@@ -370,26 +368,9 @@ __wt_rec_row_int(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page)
              */
             switch (child->modify->rec_result) {
             case WT_PM_REC_EMPTY:
-                /*
-                 * Overflow keys referencing empty pages are no longer useful, schedule them for
-                 * discard. Don't worry about instantiation, internal page keys are always
-                 * instantiated. Don't worry about reuse, reusing this key in this reconciliation is
-                 * unlikely.
-                 */
-                if (key_onpage_ovfl)
-                    WT_ERR(__wt_ovfl_discard_add(session, page, kpack->cell));
                 WT_CHILD_RELEASE_ERR(session, hazard, ref);
                 continue;
             case WT_PM_REC_MULTIBLOCK:
-                /*
-                 * Overflow keys referencing split pages are no longer useful (the split page's key
-                 * is the interesting key); schedule them for discard. Don't worry about
-                 * instantiation, internal page keys are always instantiated. Don't worry about
-                 * reuse, reusing this key in this reconciliation is unlikely.
-                 */
-                if (key_onpage_ovfl)
-                    WT_ERR(__wt_ovfl_discard_add(session, page, kpack->cell));
-
                 WT_ERR(__rec_row_merge(session, r, child));
                 WT_CHILD_RELEASE_ERR(session, hazard, ref);
                 continue;
diff --git a/src/third_party/wiredtiger/src/txn/txn_recover.c b/src/third_party/wiredtiger/src/txn/txn_recover.c
index 4d23db24ac4..450ac41bbdf 100644
--- a/src/third_party/wiredtiger/src/txn/txn_recover.c
+++ b/src/third_party/wiredtiger/src/txn/txn_recover.c
@@ -84,6 +84,14 @@ __recovery_cursor(
             WT_RET(__wt_open_cursor(session, r->files[id].uri, NULL, cfg, &c));
             r->files[id].c = c;
         }
+#ifndef WT_STANDALONE_BUILD
+        /*
+         * In the event of a clean shutdown, there shouldn't be any other table log records other
+         * than metadata.
+         */
+        if (!metadata_op)
+            S2C(session)->unclean_shutdown = true;
+#endif
     }
 
     if (duplicate && c != NULL)
@@ -475,6 +483,14 @@ __recovery_set_checkpoint_snapshot(WT_SESSION_IMPL *session)
     counter = 0;
 
     /*
+     * WiredTiger versions 10.0.1 onward have a valid checkpoint snapshot on-disk. Ignore reading
+     * the on-disk checkpoint snapshot from older versions.
+     */
+    if (conn->recovery_major < 10 ||
+      (conn->recovery_major == 10 && conn->recovery_minor == 0 && conn->recovery_patch == 0))
+        return (0);
+
+    /*
      * Read the system checkpoint information from the metadata file and save the snapshot related
      * details of the last checkpoint for later query. This gets saved in the connection.
      */
@@ -959,6 +975,18 @@ __wt_txn_recover(WT_SESSION_IMPL *session, const char *cfg[])
 done:
     /* Close cached cursors, rollback-to-stable asserts exclusive access. */
     WT_ERR(__recovery_close_cursors(&r));
+#ifndef WT_STANDALONE_BUILD
+    /*
+     * There is a known problem with upgrading from release 10.0.0 specifically. There are now fixes
+     * that can properly upgrade from 10.0.0 without hitting the problem but only from a clean
+     * shutdown of 10.0.0. Earlier releases are not affected by the upgrade issue.
+     */
+    if (conn->unclean_shutdown && conn->recovery_major == 10 && conn->recovery_minor == 0 &&
+      conn->recovery_patch == 0)
+        WT_ERR_MSG(session, WT_ERROR,
+          "Upgrading from a WiredTiger version 10.0.0 database that was not shutdown cleanly is "
+          "not allowed. Perform a clean shutdown on version 10.0.0 and then upgrade.");
+#endif
 
     WT_ERR(__recovery_set_checkpoint_timestamp(&r));
     WT_ERR(__recovery_set_oldest_timestamp(&r));
diff --git a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c
index 6639f019709..1524df5218a 100644
--- a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c
+++ b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c
@@ -8,8 +8,9 @@
 
 #include "wt_internal.h"
 
-#define WT_CHECK_RECOVERY_FLAG_TXNID(session, txnid) \
-    (F_ISSET(S2C(session), WT_CONN_RECOVERING) && (txnid) >= S2C(session)->recovery_ckpt_snap_min)
+#define WT_CHECK_RECOVERY_FLAG_TXNID(session, txnid)                                           \
+    (F_ISSET(S2C(session), WT_CONN_RECOVERING) && S2C(session)->recovery_ckpt_snap_min != 0 && \
+      (txnid) >= S2C(session)->recovery_ckpt_snap_min)
 
 /* Enable rollback to stable verbose messaging during recovery. */
 #define WT_VERB_RECOVERY_RTS(session)                                                              \