From 43d25964249164d76d5e04dd6cf38f6111e21f5f Mon Sep 17 00:00:00 2001 From: Luke Chen Date: Tue, 9 Jun 2020 11:09:18 +1000 Subject: Import wiredtiger: 8f7554c85299a49079c7f06290909f791678b794 from branch mongodb-4.2 ref: 8de74488f2..8f7554c852 for: 4.2.8 WT-6366 Off-by-one overflow in block-modification bitmaps for incremental backup --- src/third_party/wiredtiger/dist/api_data.py | 2 +- src/third_party/wiredtiger/import.data | 2 +- src/third_party/wiredtiger/src/block/block_ckpt.c | 13 +++--- src/third_party/wiredtiger/src/config/config_def.c | 2 +- .../wiredtiger/src/include/wiredtiger.in | 2 +- .../wiredtiger/test/csuite/incr_backup/main.c | 19 +++++---- src/third_party/wiredtiger/test/format/backup.c | 7 ++-- src/third_party/wiredtiger/test/format/config.c | 48 ++++++++++++++++++++++ src/third_party/wiredtiger/test/format/config.h | 3 ++ src/third_party/wiredtiger/test/format/format.h | 1 + 10 files changed, 80 insertions(+), 19 deletions(-) diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index a5df6227eee..5eb48bcda59 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -1245,7 +1245,7 @@ methods = { this setting manages the granularity of how WiredTiger maintains modification maps internally. The larger the granularity, the smaller amount of information WiredTiger need to maintain''', - min='1MB', max='2GB'), + min='4KB', max='2GB'), Config('src_id', '', r''' a string that identifies a previous checkpoint backup source as the source of this incremental backup. This identifier must have already been created diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 28314b55190..95527d5ba06 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-4.2", - "commit": "8de74488f2bb2b5cba0404c345f568a2f72478d3" + "commit": "8f7554c85299a49079c7f06290909f791678b794" } diff --git a/src/third_party/wiredtiger/src/block/block_ckpt.c b/src/third_party/wiredtiger/src/block/block_ckpt.c index 1a148606741..4f41c5f0120 100644 --- a/src/third_party/wiredtiger/src/block/block_ckpt.c +++ b/src/third_party/wiredtiger/src/block/block_ckpt.c @@ -355,16 +355,16 @@ __ckpt_add_blkmod_entry( WT_ASSERT(session, blk_mod->granularity != 0); /* - * Figure out how the starting and ending bits based on the granularity and our offset and - * length. + * Figure out the starting and ending locations in the bitmap based on its granularity and our + * offset and length. The bit locations are zero-based; be careful translating to sizes. */ start_bit = (uint64_t)offset / blk_mod->granularity; end_bit = (uint64_t)(offset + len - 1) / blk_mod->granularity; WT_ASSERT(session, end_bit < UINT32_MAX); /* We want to grow the bitmap by 64 bits, or 8 bytes at a time. */ - end_rdup_bits = WT_MAX(__wt_rduppo2((uint32_t)end_bit, 64), WT_BLOCK_MODS_LIST_MIN); - end_rdup_bytes = end_rdup_bits >> 3; - end_buf_bytes = (uint32_t)blk_mod->nbits >> 3; + end_rdup_bits = WT_MAX(__wt_rduppo2((uint32_t)end_bit + 1, 64), WT_BLOCK_MODS_LIST_MIN); + end_rdup_bytes = __bitstr_size(end_rdup_bits); + end_buf_bytes = __bitstr_size((uint32_t)blk_mod->nbits); /* * We are doing a lot of shifting. Make sure that the number of bytes we end up with is a * multiple of eight. We guarantee that in the rounding up call, but also make sure that the @@ -384,6 +384,9 @@ __ckpt_add_blkmod_entry( } blk_mod->nbits = end_rdup_bits; } + /* Make sure we're not going to run past the end of the bitmap */ + WT_ASSERT(session, blk_mod->bitstring.size >= __bitstr_size((uint32_t)blk_mod->nbits)); + WT_ASSERT(session, end_bit < blk_mod->nbits); /* Set all the bits needed to record this offset/length pair. */ __bit_nset(blk_mod->bitstring.mem, start_bit, end_bit); return (0); diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index bba4a9b914b..531af81ae00 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -297,7 +297,7 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_log_flush[] = { static const WT_CONFIG_CHECK confchk_WT_SESSION_open_cursor_incremental_subconfigs[] = { {"enabled", "boolean", NULL, NULL, NULL, 0}, {"file", "string", NULL, NULL, NULL, 0}, {"force_stop", "boolean", NULL, NULL, NULL, 0}, - {"granularity", "int", NULL, "min=1MB,max=2GB", NULL, 0}, + {"granularity", "int", NULL, "min=4KB,max=2GB", NULL, 0}, {"src_id", "string", NULL, NULL, NULL, 0}, {"this_id", "string", NULL, NULL, NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index d9f691e8630..abf55b4bc14 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -1081,7 +1081,7 @@ struct __wt_session { * @config{    granularity, * this setting manages the granularity of how WiredTiger maintains modification maps * internally. The larger the granularity\, the smaller amount of information WiredTiger - * need to maintain., an integer between 1MB and 2GB; default \c 16MB.} + * need to maintain., an integer between 4KB and 2GB; default \c 16MB.} * @config{    src_id, a string that identifies a previous checkpoint * backup source as the source of this incremental backup. This identifier must have * already been created by use of the 'this_id' configuration in an earlier backup. A diff --git a/src/third_party/wiredtiger/test/csuite/incr_backup/main.c b/src/third_party/wiredtiger/test/csuite/incr_backup/main.c index 2c8893780eb..8a5a67ec760 100644 --- a/src/third_party/wiredtiger/test/csuite/incr_backup/main.c +++ b/src/third_party/wiredtiger/test/csuite/incr_backup/main.c @@ -460,6 +460,7 @@ base_backup(WT_CONNECTION *conn, WT_RAND_STATE *rand, const char *home, const ch int nfiles, ret; char buf[4096]; char *filename; + char granularity_unit; nfiles = 0; @@ -474,14 +475,18 @@ base_backup(WT_CONNECTION *conn, WT_RAND_STATE *rand, const char *home, const ch testutil_check(conn->open_session(conn, NULL, NULL, &session)); tinfo->full_backup_number = tinfo->incr_backup_number++; - /* Half of the runs with a low granularity: 1M */ - if (__wt_random(rand) % 2 == 0) - granularity = 1; - else - granularity = 1 + __wt_random(rand) % 20; + /* Half of the runs with very low granularity to stress bitmaps */ + granularity = __wt_random(rand) % 20; + if (__wt_random(rand) % 2 == 0) { + granularity_unit = 'K'; + granularity += 4; + } else { + granularity_unit = 'M'; + granularity += 1; + } testutil_check(__wt_snprintf(buf, sizeof(buf), - "incremental=(granularity=%" PRIu32 "M,enabled=true,this_id=ID%d)", granularity, - (int)tinfo->full_backup_number)); + "incremental=(granularity=%" PRIu32 "%c,enabled=true,this_id=ID%d)", granularity, + granularity_unit, (int)tinfo->full_backup_number)); VERBOSE(3, "open_cursor(session, \"backup:\", NULL, \"%s\", &cursor)\n", buf); testutil_check(session->open_cursor(session, "backup:", NULL, buf, &cursor)); diff --git a/src/third_party/wiredtiger/test/format/backup.c b/src/third_party/wiredtiger/test/format/backup.c index 5ad1cfe65dc..66d859849a5 100644 --- a/src/third_party/wiredtiger/test/format/backup.c +++ b/src/third_party/wiredtiger/test/format/backup.c @@ -388,15 +388,16 @@ backup(void *arg) if (g.c_backup_incr_flag == INCREMENTAL_BLOCK) { /* * If we're doing a full backup as the start of the incremental backup, only send in an - * identifier for this one. + * identifier for this one. Also set the block granularity. */ if (incr_full) { active_files_free(&active[0]); active_files_free(&active[1]); active_now = &active[g.backup_id % 2]; active_prev = NULL; - testutil_check(__wt_snprintf( - cfg, sizeof(cfg), "incremental=(enabled,this_id=ID%" PRIu64 ")", g.backup_id++)); + testutil_check(__wt_snprintf(cfg, sizeof(cfg), + "incremental=(enabled,granularity=%" PRIu32 "K,this_id=ID%" PRIu64 ")", + g.c_backup_incr_granularity, g.backup_id++)); full = true; incr_full = false; } else { diff --git a/src/third_party/wiredtiger/test/format/config.c b/src/third_party/wiredtiger/test/format/config.c index eb86c0860e7..de68fd9fe60 100644 --- a/src/third_party/wiredtiger/test/format/config.c +++ b/src/third_party/wiredtiger/test/format/config.c @@ -31,6 +31,7 @@ static void config(void); static void config_backup_incr(void); +static void config_backup_incr_granularity(void); static void config_backward_compatible(void); static void config_cache(void); static void config_checkpoint(void); @@ -283,6 +284,8 @@ config_backup_incr(void) if (g.c_logging_archive) config_single("logging.archive=0", false); } + if (g.c_backup_incr_flag == INCREMENTAL_BLOCK) + config_backup_incr_granularity(); return; } @@ -311,10 +314,55 @@ config_backup_incr(void) case 9: case 10: config_single("backup.incremental=block", false); + config_backup_incr_granularity(); break; } } +/* + * config_backup_incr_granularity -- + * Configuration of block granularity for incremental backup + */ +static void +config_backup_incr_granularity(void) +{ + uint32_t granularity, i; + char confbuf[128]; + + if (config_is_perm("backup.incr_granularity")) + return; + + /* + * Three block sizes are interesting. 16MB is the default for WiredTiger and MongoDB. 1MB is the + * minimum allowed by MongoDB. Smaller sizes stress block tracking and are good for testing. The + * granularity is in units of KB. + */ + granularity = 0; + i = mmrand(NULL, 1, 10); + switch (i) { + case 1: /* 50% small size for stress testing */ + case 2: + case 3: + case 4: + case 5: + granularity = 4 * i; + break; + case 6: /* 20% 1MB granularity */ + case 7: + granularity = 1024; + break; + case 8: /* 30% 16MB granularity */ + case 9: + case 10: + granularity = 16 * 1024; + break; + } + + testutil_check( + __wt_snprintf(confbuf, sizeof(confbuf), "backup.incr_granularity=%u", granularity)); + config_single(confbuf, false); +} + /* * config_backward_compatible -- * Backward compatibility configuration. diff --git a/src/third_party/wiredtiger/test/format/config.h b/src/third_party/wiredtiger/test/format/config.h index 04da641386b..19e5c96e03f 100644 --- a/src/third_party/wiredtiger/test/format/config.h +++ b/src/third_party/wiredtiger/test/format/config.h @@ -74,6 +74,9 @@ static CONFIG c[] = { {"backup.incremental", "type of backup (block | log | off)", C_IGNORE | C_STRING, 0, 0, 0, NULL, &g.c_backup_incremental}, + {"backup.incr_granularity", "incremental backup block granularity in KB", 0x0, 4, 16384, 16384, + &g.c_backup_incr_granularity, NULL}, + {"btree.bitcnt", "number of bits for fixed-length column-store files", 0x0, 1, 8, 8, &g.c_bitcnt, NULL}, diff --git a/src/third_party/wiredtiger/test/format/format.h b/src/third_party/wiredtiger/test/format/format.h index 6bc213a65ef..5133f27c925 100644 --- a/src/third_party/wiredtiger/test/format/format.h +++ b/src/third_party/wiredtiger/test/format/format.h @@ -137,6 +137,7 @@ typedef struct { uint32_t c_auto_throttle; uint32_t c_backups; char *c_backup_incremental; + uint32_t c_backup_incr_granularity; uint32_t c_bitcnt; uint32_t c_bloom; uint32_t c_bloom_bit_count; -- cgit v1.2.1