Import wiredtiger: 58baf804dd6e5a72c4e122cfb696e2d06a9fc888 from branch mongodb-4.4

ref: d74cb1a913..58baf804dd
for: 4.3.4

WT-5371       Fix test_stat08.py assertion failure
WT-5521       Cache stuck during format initial load, configured with library checkpoints
WT-5584       Change format to ignore Evergreen timestamps in configuration files
WT-5587       Limit how many checkpoints are dropped by a subsequent checkpoint

10 files changed, 80 insertions, 26 deletions

diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data
index 190c65b0da5..9b1647d3c2f 100644
--- a/src/third_party/wiredtiger/import.data
+++ b/src/third_party/wiredtiger/import.data
@@ -2,5 +2,5 @@
     "vendor": "wiredtiger",
     "github": "wiredtiger/wiredtiger.git",
     "branch": "mongodb-4.4",
-    "commit": "d74cb1a91399e1197a95667502002ffed2b7b82b"
+    "commit": "58baf804dd6e5a72c4e122cfb696e2d06a9fc888"
 }
diff --git a/src/third_party/wiredtiger/lang/python/wiredtiger.i b/src/third_party/wiredtiger/lang/python/wiredtiger.i
index 319e703d445..b72abdf072d 100644
--- a/src/third_party/wiredtiger/lang/python/wiredtiger.i
+++ b/src/third_party/wiredtiger/lang/python/wiredtiger.i
@@ -42,6 +42,7 @@ This provides an API similar to the C API, with the following modifications:
   - Statistics cursors behave a little differently and are best handled using the C-like functions
   - C Constants starting with WT_STAT_DSRC are instead exposed under wiredtiger.stat.dsrc
   - C Constants starting with WT_STAT_CONN are instead exposed under wiredtiger.stat.conn
+  - C Constants starting with WT_STAT_SESSION are instead exposed under wiredtiger.stat.session
 "
 %enddef
 
@@ -1472,12 +1473,17 @@ class stat:
 		'''keys for cursors on data source statistics'''
 		pass
 
+	class session:
+		'''keys for cursors on session statistics'''
+		pass
+
 ## @}
 
 import sys
 # All names starting with 'WT_STAT_DSRC_' are renamed to
 # the wiredtiger.stat.dsrc class, those starting with 'WT_STAT_CONN' are
-# renamed to wiredtiger.stat.conn class.
+# renamed to the wiredtiger.stat.conn class. All names starting with 'WT_STAT_SESSION'
+# are renamed to the wiredtiger.stat.session class.
 def _rename_with_prefix(prefix, toclass):
 	curmodule = sys.modules[__name__]
 	for name in dir(curmodule):
@@ -1488,5 +1494,6 @@ def _rename_with_prefix(prefix, toclass):
 
 _rename_with_prefix('WT_STAT_CONN_', stat.conn)
 _rename_with_prefix('WT_STAT_DSRC_', stat.dsrc)
+_rename_with_prefix('WT_STAT_SESSION_', stat.session)
 del _rename_with_prefix
 %}
diff --git a/src/third_party/wiredtiger/src/reconcile/rec_write.c b/src/third_party/wiredtiger/src/reconcile/rec_write.c
index a92e578546b..a24cd11116b 100644
--- a/src/third_party/wiredtiger/src/reconcile/rec_write.c
+++ b/src/third_party/wiredtiger/src/reconcile/rec_write.c
@@ -1835,6 +1835,13 @@ __rec_split_write(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_REC_CHUNK *chunk
     verify_image = true;
 #endif
 
+    /*
+     * If reconciliation requires multiple blocks and checkpoint is running we'll eventually fail,
+     * unless we're the checkpoint thread. Big pages take a lot of writes, avoid wasting work.
+     */
+    if (!last_block && WT_BTREE_SYNCING(btree) && !WT_SESSION_BTREE_SYNC(session))
+        return (__wt_set_return(session, EBUSY));
+
     /* Make sure there's enough room for another write. */
     WT_RET(__wt_realloc_def(session, &r->multi_allocated, r->multi_next + 1, &r->multi));
     multi = &r->multi[r->multi_next++];
diff --git a/src/third_party/wiredtiger/src/txn/txn_ckpt.c b/src/third_party/wiredtiger/src/txn/txn_ckpt.c
index c70c092a916..5f538f7fcda 100644
--- a/src/third_party/wiredtiger/src/txn/txn_ckpt.c
+++ b/src/third_party/wiredtiger/src/txn/txn_ckpt.c
@@ -1112,6 +1112,7 @@ static void
 __drop(WT_CKPT *ckptbase, const char *name, size_t len)
 {
     WT_CKPT *ckpt;
+    u_int max_ckpt_drop;
 
     /*
      * If we're dropping internal checkpoints, match to the '.' separating the checkpoint name from
@@ -1120,9 +1121,20 @@ __drop(WT_CKPT *ckptbase, const char *name, size_t len)
      * it's one we want to drop.
      */
     if (strncmp(WT_CHECKPOINT, name, len) == 0) {
+        /*
+         * Currently, hot backup cursors block checkpoint drop, which means releasing a hot backup
+         * cursor can result in immediately attempting to drop lots of checkpoints, which involves a
+         * fair amount of work while holding locks. Limit the number of standard checkpoints dropped
+         * per checkpoint.
+         */
+        max_ckpt_drop = 0;
         WT_CKPT_FOREACH (ckptbase, ckpt)
-            if (WT_PREFIX_MATCH(ckpt->name, WT_CHECKPOINT))
+            if (WT_PREFIX_MATCH(ckpt->name, WT_CHECKPOINT)) {
                 F_SET(ckpt, WT_CKPT_DELETE);
+#define WT_MAX_CHECKPOINT_DROP 4
+                if (++max_ckpt_drop >= WT_MAX_CHECKPOINT_DROP)
+                    break;
+            }
     } else
         WT_CKPT_FOREACH (ckptbase, ckpt)
             if (WT_STRING_MATCH(ckpt->name, name, len))
diff --git a/src/third_party/wiredtiger/test/format/config.c b/src/third_party/wiredtiger/test/format/config.c
index 59a23bee458..1bfe87473c4 100644
--- a/src/third_party/wiredtiger/test/format/config.c
+++ b/src/third_party/wiredtiger/test/format/config.c
@@ -840,9 +840,9 @@ config_file(const char *name)
         testutil_die(errno, "fopen: %s", name);
 
     /*
-     * Skip leading Evergreen timestamps by skipping past anything that precedes a blank, unless we
-     * find a hash (comment) character. This is a little fragile, it requires Evergreen not include
-     * comment characters in its timestamps, and no format configuration commands including blanks.
+     * Skip leading Evergreen timestamps by skipping up to a closing brace and following whitespace.
+     * This is a little fragile: we're in trouble if Evergreen changes its timestamp format or if
+     * this program includes closing braces in its commands.
      */
     while (fgets(buf, sizeof(buf), fp) != NULL) {
         for (p = t = buf; *p != '\0'; ++p) {
@@ -854,8 +854,11 @@ config_file(const char *name)
                 t = p;
                 break;
             }
-            if (isblank(*p)) /* Blank, configuration starts after it. */
-                t = p + 1;
+            if (t == buf && *p == ']') { /* Closing brace, configuration starts after it. */
+                while (isblank(*++p))
+                    ;
+                t = p--;
+            }
         }
         if (*t == '\0' || *t == '#')
             continue;
diff --git a/src/third_party/wiredtiger/test/format/format.h b/src/third_party/wiredtiger/test/format/format.h
index 7a87f4a9715..6407ee652d0 100644
--- a/src/third_party/wiredtiger/test/format/format.h
+++ b/src/third_party/wiredtiger/test/format/format.h
@@ -359,6 +359,7 @@ void val_gen(WT_RAND_STATE *, WT_ITEM *, uint64_t);
 void val_gen_init(WT_ITEM *);
 void val_gen_teardown(WT_ITEM *);
 void val_init(void);
+void wts_checkpoints(void);
 void wts_close(void);
 void wts_dump(const char *, bool);
 void wts_init(void);
diff --git a/src/third_party/wiredtiger/test/format/ops.c b/src/third_party/wiredtiger/test/format/ops.c
index c977b3cb020..93d5fc204ac 100644
--- a/src/third_party/wiredtiger/test/format/ops.c
+++ b/src/third_party/wiredtiger/test/format/ops.c
@@ -588,8 +588,7 @@ ops_open_session(TINFO *tinfo, bool *ckpt_handlep)
          */
         while ((ret = session->open_cursor(session, g.uri, NULL, "append", &cursor)) == EBUSY)
             __wt_yield();
-
-        testutil_check(ret);
+        testutil_checkfmt(ret, "%s", g.uri);
         *ckpt_handlep = false;
     }
 
@@ -1027,7 +1026,7 @@ wts_read_scan(void)
     WT_DECL_RET;
     WT_ITEM key, value;
     WT_SESSION *session;
-    uint64_t keyno, last_keyno;
+    uint64_t keyno;
 
     conn = g.wts_conn;
 
@@ -1052,14 +1051,10 @@ wts_read_scan(void)
     testutil_check(ret);
 
     /* Check a random subset of the records using the key. */
-    for (last_keyno = keyno = 0; keyno < g.key_cnt;) {
-        keyno += mmrand(NULL, 1, 17);
+    for (keyno = 0; keyno < g.key_cnt;) {
+        keyno += mmrand(NULL, 1, 1000);
         if (keyno > g.rows)
             keyno = g.rows;
-        if (keyno - last_keyno > 1000) {
-            track("read row scan", keyno, NULL);
-            last_keyno = keyno;
-        }
 
         switch (ret = read_row_worker(cursor, keyno, &key, &value, false)) {
         case 0:
diff --git a/src/third_party/wiredtiger/test/format/t.c b/src/third_party/wiredtiger/test/format/t.c
index 4e072a4fe19..8b7ba07fe9e 100644
--- a/src/third_party/wiredtiger/test/format/t.c
+++ b/src/third_party/wiredtiger/test/format/t.c
@@ -267,15 +267,15 @@ main(int argc, char *argv[])
 
         track("starting up", 0ULL, NULL);
 
+        /* Load and verify initial records */
         wts_open(g.home, true, &g.wts_conn);
         wts_init();
-
-        /* Load and verify initial records */
         TIMED_MAJOR_OP(wts_load());
         TIMED_MAJOR_OP(wts_verify("post-bulk verify"));
         TIMED_MAJOR_OP(wts_read_scan());
 
         /* Operations. */
+        wts_checkpoints();
         for (reps = 1; reps <= FORMAT_OPERATION_REPS; ++reps)
             wts_ops(ops_seconds, reps == FORMAT_OPERATION_REPS);
 
diff --git a/src/third_party/wiredtiger/test/format/wts.c b/src/third_party/wiredtiger/test/format/wts.c
index c5993466f5d..b05893598ea 100644
--- a/src/third_party/wiredtiger/test/format/wts.c
+++ b/src/third_party/wiredtiger/test/format/wts.c
@@ -177,11 +177,6 @@ wts_open(const char *home, bool set_api, WT_CONNECTION **connp)
     if (DATASOURCE("lsm") || g.c_cache < 20)
         CONFIG_APPEND(p, ",eviction_dirty_trigger=95");
 
-    /* Checkpoints. */
-    if (g.c_checkpoint_flag == CHECKPOINT_WIREDTIGER)
-        CONFIG_APPEND(p, ",checkpoint=(wait=%" PRIu32 ",log_size=%" PRIu32 ")", g.c_checkpoint_wait,
-          MEGABYTE(g.c_checkpoint_log_size));
-
     /* Eviction worker configuration. */
     if (g.c_evict_max != 0)
         CONFIG_APPEND(p, ",eviction=(threads_max=%" PRIu32 ")", g.c_evict_max);
@@ -299,6 +294,32 @@ wts_reopen(void)
 }
 
 /*
+ * wts_checkpoints --
+ *     Configure WiredTiger library checkpoints.
+ */
+void
+wts_checkpoints(void)
+{
+    char config[1024];
+
+    /*
+     * Configuring WiredTiger library checkpoints is done separately, rather than as part of the
+     * original database open because format tests small caches and you can get into cache stuck
+     * trouble during the initial load (where bulk load isn't configured). There's a single thread
+     * doing lots of inserts and creating huge leaf pages. Those pages can't be evicted if there's a
+     * checkpoint running in the tree, and the cache can get stuck. That workload is unlikely enough
+     * we're not going to fix it in the library, so configure it away here.
+     */
+    if (g.c_checkpoint_flag != CHECKPOINT_WIREDTIGER)
+        return;
+
+    testutil_check(
+      __wt_snprintf(config, sizeof(config), ",checkpoint=(wait=%" PRIu32 ",log_size=%" PRIu32 ")",
+        g.c_checkpoint_wait, MEGABYTE(g.c_checkpoint_log_size)));
+    testutil_check(g.wts_conn->reconfigure(g.wts_conn, config));
+}
+
+/*
  * wts_create --
  *     Create the underlying store.
  */
diff --git a/src/third_party/wiredtiger/test/suite/test_stat08.py b/src/third_party/wiredtiger/test/suite/test_stat08.py
index c6d20a3b02f..bfa692e754d 100644
--- a/src/third_party/wiredtiger/test/suite/test_stat08.py
+++ b/src/third_party/wiredtiger/test/suite/test_stat08.py
@@ -26,6 +26,7 @@
 # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 # OTHER DEALINGS IN THE SOFTWARE.
 
+import os
 import wiredtiger, wttest
 
 # test_stat08.py
@@ -35,12 +36,19 @@ class test_stat08(wttest.WiredTigerTestCase):
     nentries = 350000
     conn_config = 'cache_size=10MB,statistics=(all)'
     entry_value = "abcde" * 40
-    BYTES_READ = 4000
-    READ_TIME = 4003
+    BYTES_READ = wiredtiger.stat.session.bytes_read
+    READ_TIME = wiredtiger.stat.session.read_time
     session_stats = { BYTES_READ : "session: bytes read into cache",           \
         READ_TIME : "session: page read from disk to cache time (usecs)"}
 
     def check_stats(self, cur, k):
+        #
+        # Some Windows machines lack the time granularity to detect microseconds.
+        # Skip the time check on Windows.
+        #
+        if os.name == "nt" and k is self.READ_TIME:
+            return
+
         exp_desc = self.session_stats[k]
         cur.set_key(k)
         cur.search()