Import wiredtiger: 8f7554c85299a49079c7f06290909f791678b794 from branch mongodb-4.2r4.2.8-rc0r4.2.8

ref: 8de74488f2..8f7554c852
for: 4.2.8

WT-6366       Off-by-one overflow in block-modification bitmaps for incremental backup

10 files changed, 80 insertions, 19 deletions

diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py
index a5df6227eee..5eb48bcda59 100644
--- a/src/third_party/wiredtiger/dist/api_data.py
+++ b/src/third_party/wiredtiger/dist/api_data.py
@@ -1245,7 +1245,7 @@ methods = {
             this setting manages the granularity of how WiredTiger maintains modification
             maps internally. The larger the granularity, the smaller amount of information
             WiredTiger need to maintain''',
-            min='1MB', max='2GB'),
+            min='4KB', max='2GB'),
         Config('src_id', '', r'''
             a string that identifies a previous checkpoint backup source as the source
             of this incremental backup. This identifier must have already been created
diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data
index 28314b55190..95527d5ba06 100644
--- a/src/third_party/wiredtiger/import.data
+++ b/src/third_party/wiredtiger/import.data
@@ -2,5 +2,5 @@
     "vendor": "wiredtiger",
     "github": "wiredtiger/wiredtiger.git",
     "branch": "mongodb-4.2",
-    "commit": "8de74488f2bb2b5cba0404c345f568a2f72478d3"
+    "commit": "8f7554c85299a49079c7f06290909f791678b794"
 }
diff --git a/src/third_party/wiredtiger/src/block/block_ckpt.c b/src/third_party/wiredtiger/src/block/block_ckpt.c
index 1a148606741..4f41c5f0120 100644
--- a/src/third_party/wiredtiger/src/block/block_ckpt.c
+++ b/src/third_party/wiredtiger/src/block/block_ckpt.c
@@ -355,16 +355,16 @@ __ckpt_add_blkmod_entry(
 
     WT_ASSERT(session, blk_mod->granularity != 0);
     /*
-     * Figure out how the starting and ending bits based on the granularity and our offset and
-     * length.
+     * Figure out the starting and ending locations in the bitmap based on its granularity and our
+     * offset and length. The bit locations are zero-based; be careful translating to sizes.
      */
     start_bit = (uint64_t)offset / blk_mod->granularity;
     end_bit = (uint64_t)(offset + len - 1) / blk_mod->granularity;
     WT_ASSERT(session, end_bit < UINT32_MAX);
     /* We want to grow the bitmap by 64 bits, or 8 bytes at a time. */
-    end_rdup_bits = WT_MAX(__wt_rduppo2((uint32_t)end_bit, 64), WT_BLOCK_MODS_LIST_MIN);
-    end_rdup_bytes = end_rdup_bits >> 3;
-    end_buf_bytes = (uint32_t)blk_mod->nbits >> 3;
+    end_rdup_bits = WT_MAX(__wt_rduppo2((uint32_t)end_bit + 1, 64), WT_BLOCK_MODS_LIST_MIN);
+    end_rdup_bytes = __bitstr_size(end_rdup_bits);
+    end_buf_bytes = __bitstr_size((uint32_t)blk_mod->nbits);
     /*
      * We are doing a lot of shifting. Make sure that the number of bytes we end up with is a
      * multiple of eight. We guarantee that in the rounding up call, but also make sure that the
@@ -384,6 +384,9 @@ __ckpt_add_blkmod_entry(
         }
         blk_mod->nbits = end_rdup_bits;
     }
+    /* Make sure we're not going to run past the end of the bitmap */
+    WT_ASSERT(session, blk_mod->bitstring.size >= __bitstr_size((uint32_t)blk_mod->nbits));
+    WT_ASSERT(session, end_bit < blk_mod->nbits);
     /* Set all the bits needed to record this offset/length pair. */
     __bit_nset(blk_mod->bitstring.mem, start_bit, end_bit);
     return (0);
diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c
index bba4a9b914b..531af81ae00 100644
--- a/src/third_party/wiredtiger/src/config/config_def.c
+++ b/src/third_party/wiredtiger/src/config/config_def.c
@@ -297,7 +297,7 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_log_flush[] = {
 static const WT_CONFIG_CHECK confchk_WT_SESSION_open_cursor_incremental_subconfigs[] = {
   {"enabled", "boolean", NULL, NULL, NULL, 0}, {"file", "string", NULL, NULL, NULL, 0},
   {"force_stop", "boolean", NULL, NULL, NULL, 0},
-  {"granularity", "int", NULL, "min=1MB,max=2GB", NULL, 0},
+  {"granularity", "int", NULL, "min=4KB,max=2GB", NULL, 0},
   {"src_id", "string", NULL, NULL, NULL, 0}, {"this_id", "string", NULL, NULL, NULL, 0},
   {NULL, NULL, NULL, NULL, NULL, 0}};
 
diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in
index d9f691e8630..abf55b4bc14 100644
--- a/src/third_party/wiredtiger/src/include/wiredtiger.in
+++ b/src/third_party/wiredtiger/src/include/wiredtiger.in
@@ -1081,7 +1081,7 @@ struct __wt_session {
 	 * @config{&nbsp;&nbsp;&nbsp;&nbsp;granularity,
 	 * this setting manages the granularity of how WiredTiger maintains modification maps
 	 * internally.  The larger the granularity\, the smaller amount of information WiredTiger
-	 * need to maintain., an integer between 1MB and 2GB; default \c 16MB.}
+	 * need to maintain., an integer between 4KB and 2GB; default \c 16MB.}
 	 * @config{&nbsp;&nbsp;&nbsp;&nbsp;src_id, a string that identifies a previous checkpoint
 	 * backup source as the source of this incremental backup.  This identifier must have
 	 * already been created by use of the 'this_id' configuration in an earlier backup.  A
diff --git a/src/third_party/wiredtiger/test/csuite/incr_backup/main.c b/src/third_party/wiredtiger/test/csuite/incr_backup/main.c
index 2c8893780eb..8a5a67ec760 100644
--- a/src/third_party/wiredtiger/test/csuite/incr_backup/main.c
+++ b/src/third_party/wiredtiger/test/csuite/incr_backup/main.c
@@ -460,6 +460,7 @@ base_backup(WT_CONNECTION *conn, WT_RAND_STATE *rand, const char *home, const ch
     int nfiles, ret;
     char buf[4096];
     char *filename;
+    char granularity_unit;
 
     nfiles = 0;
 
@@ -474,14 +475,18 @@ base_backup(WT_CONNECTION *conn, WT_RAND_STATE *rand, const char *home, const ch
     testutil_check(conn->open_session(conn, NULL, NULL, &session));
     tinfo->full_backup_number = tinfo->incr_backup_number++;
 
-    /* Half of the runs with a low granularity: 1M */
-    if (__wt_random(rand) % 2 == 0)
-        granularity = 1;
-    else
-        granularity = 1 + __wt_random(rand) % 20;
+    /* Half of the runs with very low granularity to stress bitmaps */
+    granularity = __wt_random(rand) % 20;
+    if (__wt_random(rand) % 2 == 0) {
+        granularity_unit = 'K';
+        granularity += 4;
+    } else {
+        granularity_unit = 'M';
+        granularity += 1;
+    }
     testutil_check(__wt_snprintf(buf, sizeof(buf),
-      "incremental=(granularity=%" PRIu32 "M,enabled=true,this_id=ID%d)", granularity,
-      (int)tinfo->full_backup_number));
+      "incremental=(granularity=%" PRIu32 "%c,enabled=true,this_id=ID%d)", granularity,
+      granularity_unit, (int)tinfo->full_backup_number));
     VERBOSE(3, "open_cursor(session, \"backup:\", NULL, \"%s\", &cursor)\n", buf);
     testutil_check(session->open_cursor(session, "backup:", NULL, buf, &cursor));
 
diff --git a/src/third_party/wiredtiger/test/format/backup.c b/src/third_party/wiredtiger/test/format/backup.c
index 5ad1cfe65dc..66d859849a5 100644
--- a/src/third_party/wiredtiger/test/format/backup.c
+++ b/src/third_party/wiredtiger/test/format/backup.c
@@ -388,15 +388,16 @@ backup(void *arg)
         if (g.c_backup_incr_flag == INCREMENTAL_BLOCK) {
             /*
              * If we're doing a full backup as the start of the incremental backup, only send in an
-             * identifier for this one.
+             * identifier for this one. Also set the block granularity.
              */
             if (incr_full) {
                 active_files_free(&active[0]);
                 active_files_free(&active[1]);
                 active_now = &active[g.backup_id % 2];
                 active_prev = NULL;
-                testutil_check(__wt_snprintf(
-                  cfg, sizeof(cfg), "incremental=(enabled,this_id=ID%" PRIu64 ")", g.backup_id++));
+                testutil_check(__wt_snprintf(cfg, sizeof(cfg),
+                  "incremental=(enabled,granularity=%" PRIu32 "K,this_id=ID%" PRIu64 ")",
+                  g.c_backup_incr_granularity, g.backup_id++));
                 full = true;
                 incr_full = false;
             } else {
diff --git a/src/third_party/wiredtiger/test/format/config.c b/src/third_party/wiredtiger/test/format/config.c
index eb86c0860e7..de68fd9fe60 100644
--- a/src/third_party/wiredtiger/test/format/config.c
+++ b/src/third_party/wiredtiger/test/format/config.c
@@ -31,6 +31,7 @@
 
 static void config(void);
 static void config_backup_incr(void);
+static void config_backup_incr_granularity(void);
 static void config_backward_compatible(void);
 static void config_cache(void);
 static void config_checkpoint(void);
@@ -283,6 +284,8 @@ config_backup_incr(void)
             if (g.c_logging_archive)
                 config_single("logging.archive=0", false);
         }
+        if (g.c_backup_incr_flag == INCREMENTAL_BLOCK)
+            config_backup_incr_granularity();
         return;
     }
 
@@ -311,11 +314,56 @@ config_backup_incr(void)
     case 9:
     case 10:
         config_single("backup.incremental=block", false);
+        config_backup_incr_granularity();
         break;
     }
 }
 
 /*
+ * config_backup_incr_granularity --
+ *     Configuration of block granularity for incremental backup
+ */
+static void
+config_backup_incr_granularity(void)
+{
+    uint32_t granularity, i;
+    char confbuf[128];
+
+    if (config_is_perm("backup.incr_granularity"))
+        return;
+
+    /*
+     * Three block sizes are interesting. 16MB is the default for WiredTiger and MongoDB. 1MB is the
+     * minimum allowed by MongoDB. Smaller sizes stress block tracking and are good for testing. The
+     * granularity is in units of KB.
+     */
+    granularity = 0;
+    i = mmrand(NULL, 1, 10);
+    switch (i) {
+    case 1: /* 50% small size for stress testing */
+    case 2:
+    case 3:
+    case 4:
+    case 5:
+        granularity = 4 * i;
+        break;
+    case 6: /* 20% 1MB granularity */
+    case 7:
+        granularity = 1024;
+        break;
+    case 8: /* 30% 16MB granularity */
+    case 9:
+    case 10:
+        granularity = 16 * 1024;
+        break;
+    }
+
+    testutil_check(
+      __wt_snprintf(confbuf, sizeof(confbuf), "backup.incr_granularity=%u", granularity));
+    config_single(confbuf, false);
+}
+
+/*
  * config_backward_compatible --
  *     Backward compatibility configuration.
  */
diff --git a/src/third_party/wiredtiger/test/format/config.h b/src/third_party/wiredtiger/test/format/config.h
index 04da641386b..19e5c96e03f 100644
--- a/src/third_party/wiredtiger/test/format/config.h
+++ b/src/third_party/wiredtiger/test/format/config.h
@@ -74,6 +74,9 @@ static CONFIG c[] = {
   {"backup.incremental", "type of backup (block | log | off)", C_IGNORE | C_STRING, 0, 0, 0, NULL,
     &g.c_backup_incremental},
 
+  {"backup.incr_granularity", "incremental backup block granularity in KB", 0x0, 4, 16384, 16384,
+    &g.c_backup_incr_granularity, NULL},
+
   {"btree.bitcnt", "number of bits for fixed-length column-store files", 0x0, 1, 8, 8, &g.c_bitcnt,
     NULL},
 
diff --git a/src/third_party/wiredtiger/test/format/format.h b/src/third_party/wiredtiger/test/format/format.h
index 6bc213a65ef..5133f27c925 100644
--- a/src/third_party/wiredtiger/test/format/format.h
+++ b/src/third_party/wiredtiger/test/format/format.h
@@ -137,6 +137,7 @@ typedef struct {
     uint32_t c_auto_throttle;
     uint32_t c_backups;
     char *c_backup_incremental;
+    uint32_t c_backup_incr_granularity;
     uint32_t c_bitcnt;
     uint32_t c_bloom;
     uint32_t c_bloom_bit_count;