From 0c52e9773275d0c4a11c77e56d5ea0f914a903be Mon Sep 17 00:00:00 2001 From: Luke Chen Date: Fri, 28 Aug 2020 18:16:31 +1000 Subject: Import wiredtiger: cf3c0c18fe24a595ca523e91f40213a9007e4f29 from branch mongodb-4.6 ref: 1fd34059ee..cf3c0c18fe for: 4.5.1 WT-6610 Fix incremental backup checkpoint parsing to handle upgrades WT-6625 Remove outdated TODO --- src/third_party/wiredtiger/dist/api_data.py | 2 +- src/third_party/wiredtiger/import.data | 2 +- src/third_party/wiredtiger/src/config/config_def.c | 45 +++++++----- src/third_party/wiredtiger/src/conn/conn_api.c | 1 + .../wiredtiger/src/cursor/cur_backup_incr.c | 27 +++++-- .../wiredtiger/src/include/connection.h | 25 ++++--- src/third_party/wiredtiger/src/include/cursor.h | 19 ++--- src/third_party/wiredtiger/src/include/extern.h | 4 + src/third_party/wiredtiger/src/include/meta.h | 5 +- src/third_party/wiredtiger/src/meta/meta_ckpt.c | 85 +++++++++++++++------- .../wiredtiger/src/schema/schema_rename.c | 39 +++++++++- .../wiredtiger/test/csuite/incr_backup/main.c | 11 ++- src/third_party/wiredtiger/test/format/config.c | 6 -- src/third_party/wiredtiger/test/suite/test_hs06.py | 7 +- 14 files changed, 187 insertions(+), 91 deletions(-) (limited to 'src/third_party') diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index d018f94077a..fcc32e3b864 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -722,7 +722,7 @@ connection_runtime_config = [ intended for use with internal stress testing of WiredTiger.''', type='list', undoc=True, choices=[ - 'aggressive_sweep', 'checkpoint_slow', 'history_store_checkpoint_delay', + 'aggressive_sweep', 'backup_rename', 'checkpoint_slow', 'history_store_checkpoint_delay', 'history_store_sweep_race', 'prepare_checkpoint_delay', 'split_1', 'split_2', 'split_3', 'split_4', 'split_5', 'split_6', 'split_7', 'split_8']), Config('verbose', '', r''' diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 00ddac8eea9..a339ccd3624 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-4.6", - "commit": "1fd34059ee6ad303fe1f0200fd799dcb290df277" + "commit": "cf3c0c18fe24a595ca523e91f40213a9007e4f29" } diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index ed4960362af..4b5248d6764 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -136,10 +136,11 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = { {"statistics_log", "category", NULL, NULL, confchk_WT_CONNECTION_reconfigure_statistics_log_subconfigs, 5}, {"timing_stress_for_test", "list", NULL, - "choices=[\"aggressive_sweep\",\"checkpoint_slow\"," - "\"history_store_checkpoint_delay\",\"history_store_sweep_race\"," - "\"prepare_checkpoint_delay\",\"split_1\",\"split_2\",\"split_3\"" - ",\"split_4\",\"split_5\",\"split_6\",\"split_7\",\"split_8\"]", + "choices=[\"aggressive_sweep\",\"backup_rename\"," + "\"checkpoint_slow\",\"history_store_checkpoint_delay\"," + "\"history_store_sweep_race\",\"prepare_checkpoint_delay\"," + "\"split_1\",\"split_2\",\"split_3\",\"split_4\",\"split_5\"," + "\"split_6\",\"split_7\",\"split_8\"]", NULL, 0}, {"verbose", "list", NULL, "choices=[\"api\",\"backup\",\"block\",\"checkpoint\"," @@ -589,10 +590,11 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { NULL, 0}, {"statistics_log", "category", NULL, NULL, confchk_wiredtiger_open_statistics_log_subconfigs, 6}, {"timing_stress_for_test", "list", NULL, - "choices=[\"aggressive_sweep\",\"checkpoint_slow\"," - "\"history_store_checkpoint_delay\",\"history_store_sweep_race\"," - "\"prepare_checkpoint_delay\",\"split_1\",\"split_2\",\"split_3\"" - ",\"split_4\",\"split_5\",\"split_6\",\"split_7\",\"split_8\"]", + "choices=[\"aggressive_sweep\",\"backup_rename\"," + "\"checkpoint_slow\",\"history_store_checkpoint_delay\"," + "\"history_store_sweep_race\",\"prepare_checkpoint_delay\"," + "\"split_1\",\"split_2\",\"split_3\",\"split_4\",\"split_5\"," + "\"split_6\",\"split_7\",\"split_8\"]", NULL, 0}, {"transaction_sync", "category", NULL, NULL, confchk_wiredtiger_open_transaction_sync_subconfigs, 2}, @@ -666,10 +668,11 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { NULL, 0}, {"statistics_log", "category", NULL, NULL, confchk_wiredtiger_open_statistics_log_subconfigs, 6}, {"timing_stress_for_test", "list", NULL, - "choices=[\"aggressive_sweep\",\"checkpoint_slow\"," - "\"history_store_checkpoint_delay\",\"history_store_sweep_race\"," - "\"prepare_checkpoint_delay\",\"split_1\",\"split_2\",\"split_3\"" - ",\"split_4\",\"split_5\",\"split_6\",\"split_7\",\"split_8\"]", + "choices=[\"aggressive_sweep\",\"backup_rename\"," + "\"checkpoint_slow\",\"history_store_checkpoint_delay\"," + "\"history_store_sweep_race\",\"prepare_checkpoint_delay\"," + "\"split_1\",\"split_2\",\"split_3\",\"split_4\",\"split_5\"," + "\"split_6\",\"split_7\",\"split_8\"]", NULL, 0}, {"transaction_sync", "category", NULL, NULL, confchk_wiredtiger_open_transaction_sync_subconfigs, 2}, @@ -740,10 +743,11 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { NULL, 0}, {"statistics_log", "category", NULL, NULL, confchk_wiredtiger_open_statistics_log_subconfigs, 6}, {"timing_stress_for_test", "list", NULL, - "choices=[\"aggressive_sweep\",\"checkpoint_slow\"," - "\"history_store_checkpoint_delay\",\"history_store_sweep_race\"," - "\"prepare_checkpoint_delay\",\"split_1\",\"split_2\",\"split_3\"" - ",\"split_4\",\"split_5\",\"split_6\",\"split_7\",\"split_8\"]", + "choices=[\"aggressive_sweep\",\"backup_rename\"," + "\"checkpoint_slow\",\"history_store_checkpoint_delay\"," + "\"history_store_sweep_race\",\"prepare_checkpoint_delay\"," + "\"split_1\",\"split_2\",\"split_3\",\"split_4\",\"split_5\"," + "\"split_6\",\"split_7\",\"split_8\"]", NULL, 0}, {"transaction_sync", "category", NULL, NULL, confchk_wiredtiger_open_transaction_sync_subconfigs, 2}, @@ -812,10 +816,11 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { NULL, 0}, {"statistics_log", "category", NULL, NULL, confchk_wiredtiger_open_statistics_log_subconfigs, 6}, {"timing_stress_for_test", "list", NULL, - "choices=[\"aggressive_sweep\",\"checkpoint_slow\"," - "\"history_store_checkpoint_delay\",\"history_store_sweep_race\"," - "\"prepare_checkpoint_delay\",\"split_1\",\"split_2\",\"split_3\"" - ",\"split_4\",\"split_5\",\"split_6\",\"split_7\",\"split_8\"]", + "choices=[\"aggressive_sweep\",\"backup_rename\"," + "\"checkpoint_slow\",\"history_store_checkpoint_delay\"," + "\"history_store_sweep_race\",\"prepare_checkpoint_delay\"," + "\"split_1\",\"split_2\",\"split_3\",\"split_4\",\"split_5\"," + "\"split_6\",\"split_7\",\"split_8\"]", NULL, 0}, {"transaction_sync", "category", NULL, NULL, confchk_wiredtiger_open_transaction_sync_subconfigs, 2}, diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index ef55e8ec0bd..5d8c8b777d5 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -2035,6 +2035,7 @@ __wt_timing_stress_config(WT_SESSION_IMPL *session, const char *cfg[]) */ static const WT_NAME_FLAG stress_types[] = { {"aggressive_sweep", WT_TIMING_STRESS_AGGRESSIVE_SWEEP}, + {"backup_rename", WT_TIMING_STRESS_BACKUP_RENAME}, {"checkpoint_slow", WT_TIMING_STRESS_CHECKPOINT_SLOW}, {"history_store_checkpoint_delay", WT_TIMING_STRESS_HS_CHECKPOINT_DELAY}, {"history_store_sweep_race", WT_TIMING_STRESS_HS_SWEEP}, diff --git a/src/third_party/wiredtiger/src/cursor/cur_backup_incr.c b/src/third_party/wiredtiger/src/cursor/cur_backup_incr.c index f30bd1c4d3b..c71676b2082 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_backup_incr.c +++ b/src/third_party/wiredtiger/src/cursor/cur_backup_incr.c @@ -70,6 +70,15 @@ __curbackup_incr_blkmod(WT_SESSION_IMPL *session, WT_BTREE *btree, WT_CURSOR_BAC cb->nbits = (uint64_t)b.val; WT_ERR(__wt_config_subgets(session, &v, "offset", &b)); cb->offset = (uint64_t)b.val; + /* + * The rename configuration string component was added later. So don't error if we don't + * find it in the string. If we don't have it, we're not doing a rename. + */ + WT_ERR_NOTFOUND_OK(__wt_config_subgets(session, &v, "rename", &b), true); + if (ret == 0 && b.val) + F_SET(cb, WT_CURBACKUP_RENAME); + else + F_CLR(cb, WT_CURBACKUP_RENAME); /* * We found a match. Load the block information into the cursor. @@ -112,7 +121,7 @@ __curbackup_incr_next(WT_CURSOR *cursor) F_CLR(cursor, WT_CURSTD_RAW); if (!F_ISSET(cb, WT_CURBACKUP_INCR_INIT) && - (btree == NULL || F_ISSET(cb, WT_CURBACKUP_FORCE_FULL))) { + (btree == NULL || F_ISSET(cb, WT_CURBACKUP_FORCE_FULL | WT_CURBACKUP_RENAME))) { /* * We don't have this object's incremental information or it's a forced file copy. If this * is a log file, use the full pathname that may include the log path. @@ -155,19 +164,21 @@ __curbackup_incr_next(WT_CURSOR *cursor) */ WT_ERR(__curbackup_incr_blkmod(session, btree, cb)); /* - * There are three cases where we do not have block modification information for + * There are several cases where we do not have block modification information for * the file. They are described and handled as follows: * - * 1. Newly created file without checkpoint information. Return the whole file - * information. - * 2. File created and checkpointed before incremental backups were configured. + * 1. Renamed file. Always return the whole file information. + * 2. Newly created file without checkpoint information. Return the whole + * file information. + * 3. File created and checkpointed before incremental backups were configured. * Return no file information as it was copied in the initial full backup. - * 3. File that has not been modified since the previous incremental backup. + * 4. File that has not been modified since the previous incremental backup. * Return no file information as there is no new information. */ - if (cb->bitstring.mem == NULL) { + if (cb->bitstring.mem == NULL || F_ISSET(cb, WT_CURBACKUP_RENAME)) { F_SET(cb, WT_CURBACKUP_INCR_INIT); - if (F_ISSET(cb, WT_CURBACKUP_CKPT_FAKE) && F_ISSET(cb, WT_CURBACKUP_HAS_CB_INFO)) { + if (F_ISSET(cb, WT_CURBACKUP_RENAME) || + (F_ISSET(cb, WT_CURBACKUP_CKPT_FAKE) && F_ISSET(cb, WT_CURBACKUP_HAS_CB_INFO))) { WT_ERR(__wt_fs_size(session, cb->incr_file, &size)); __wt_cursor_set_key(cursor, 0, size, WT_BACKUP_FILE); goto done; diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h index 80568a48bc9..48b7784d23f 100644 --- a/src/third_party/wiredtiger/src/include/connection.h +++ b/src/third_party/wiredtiger/src/include/connection.h @@ -509,18 +509,19 @@ struct __wt_connection_impl { */ /* AUTOMATIC FLAG VALUE GENERATION START */ #define WT_TIMING_STRESS_AGGRESSIVE_SWEEP 0x0001u -#define WT_TIMING_STRESS_CHECKPOINT_SLOW 0x0002u -#define WT_TIMING_STRESS_HS_CHECKPOINT_DELAY 0x0004u -#define WT_TIMING_STRESS_HS_SWEEP 0x0008u -#define WT_TIMING_STRESS_PREPARE_CHECKPOINT_DELAY 0x0010u -#define WT_TIMING_STRESS_SPLIT_1 0x0020u -#define WT_TIMING_STRESS_SPLIT_2 0x0040u -#define WT_TIMING_STRESS_SPLIT_3 0x0080u -#define WT_TIMING_STRESS_SPLIT_4 0x0100u -#define WT_TIMING_STRESS_SPLIT_5 0x0200u -#define WT_TIMING_STRESS_SPLIT_6 0x0400u -#define WT_TIMING_STRESS_SPLIT_7 0x0800u -#define WT_TIMING_STRESS_SPLIT_8 0x1000u +#define WT_TIMING_STRESS_BACKUP_RENAME 0x0002u +#define WT_TIMING_STRESS_CHECKPOINT_SLOW 0x0004u +#define WT_TIMING_STRESS_HS_CHECKPOINT_DELAY 0x0008u +#define WT_TIMING_STRESS_HS_SWEEP 0x0010u +#define WT_TIMING_STRESS_PREPARE_CHECKPOINT_DELAY 0x0020u +#define WT_TIMING_STRESS_SPLIT_1 0x0040u +#define WT_TIMING_STRESS_SPLIT_2 0x0080u +#define WT_TIMING_STRESS_SPLIT_3 0x0100u +#define WT_TIMING_STRESS_SPLIT_4 0x0200u +#define WT_TIMING_STRESS_SPLIT_5 0x0400u +#define WT_TIMING_STRESS_SPLIT_6 0x0800u +#define WT_TIMING_STRESS_SPLIT_7 0x1000u +#define WT_TIMING_STRESS_SPLIT_8 0x2000u /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint64_t timing_stress_flags; diff --git a/src/third_party/wiredtiger/src/include/cursor.h b/src/third_party/wiredtiger/src/include/cursor.h index 1cb3fae58bd..c344710120d 100644 --- a/src/third_party/wiredtiger/src/include/cursor.h +++ b/src/third_party/wiredtiger/src/include/cursor.h @@ -61,15 +61,16 @@ struct __wt_cursor_backup { uint64_t granularity; /* Length, transfer size */ /* AUTOMATIC FLAG VALUE GENERATION START */ -#define WT_CURBACKUP_CKPT_FAKE 0x01u /* Object has fake checkpoint */ -#define WT_CURBACKUP_DUP 0x02u /* Duplicated backup cursor */ -#define WT_CURBACKUP_FORCE_FULL 0x04u /* Force full file copy for this cursor */ -#define WT_CURBACKUP_FORCE_STOP 0x08u /* Force stop incremental backup */ -#define WT_CURBACKUP_HAS_CB_INFO 0x10u /* Object has checkpoint backup info */ -#define WT_CURBACKUP_INCR 0x20u /* Incremental backup cursor */ -#define WT_CURBACKUP_INCR_INIT 0x40u /* Cursor traversal initialized */ -#define WT_CURBACKUP_LOCKER 0x80u /* Hot-backup started */ - /* AUTOMATIC FLAG VALUE GENERATION STOP */ +#define WT_CURBACKUP_CKPT_FAKE 0x001u /* Object has fake checkpoint */ +#define WT_CURBACKUP_DUP 0x002u /* Duplicated backup cursor */ +#define WT_CURBACKUP_FORCE_FULL 0x004u /* Force full file copy for this cursor */ +#define WT_CURBACKUP_FORCE_STOP 0x008u /* Force stop incremental backup */ +#define WT_CURBACKUP_HAS_CB_INFO 0x010u /* Object has checkpoint backup info */ +#define WT_CURBACKUP_INCR 0x020u /* Incremental backup cursor */ +#define WT_CURBACKUP_INCR_INIT 0x040u /* Cursor traversal initialized */ +#define WT_CURBACKUP_LOCKER 0x080u /* Hot-backup started */ +#define WT_CURBACKUP_RENAME 0x100u /* Object had a rename */ + /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t flags; }; diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index 90f4eee7884..c8b4427f972 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -358,6 +358,8 @@ extern int __wt_checkpoint_server_destroy(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_checkpoint_sync(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_ckpt_blkmod_to_meta(WT_SESSION_IMPL *session, WT_ITEM *buf, WT_CKPT *ckpt) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_close(WT_SESSION_IMPL *session, WT_FH **fhp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_close_connection_close(WT_SESSION_IMPL *session) @@ -1032,6 +1034,8 @@ extern int __wt_meta_apply_all(WT_SESSION_IMPL *session, int (*file_func)(WT_SESSION_IMPL *, const char *[]), int (*name_func)(WT_SESSION_IMPL *, const char *, bool *), const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_meta_blk_mods_load(WT_SESSION_IMPL *session, const char *config, WT_CKPT *ckpt, + bool rename) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_meta_block_metadata(WT_SESSION_IMPL *session, const char *config, WT_CKPT *ckpt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_meta_checkpoint(WT_SESSION_IMPL *session, const char *fname, const char *checkpoint, diff --git a/src/third_party/wiredtiger/src/include/meta.h b/src/third_party/wiredtiger/src/include/meta.h index dac2cc59573..a0241a22ce7 100644 --- a/src/third_party/wiredtiger/src/include/meta.h +++ b/src/third_party/wiredtiger/src/include/meta.h @@ -98,8 +98,9 @@ struct __wt_block_mods { uint64_t offset; /* Zero bit offset for bitstring */ uint64_t granularity; /* AUTOMATIC FLAG VALUE GENERATION START */ -#define WT_BLOCK_MODS_VALID 0x1u /* Entry is valid */ - /* AUTOMATIC FLAG VALUE GENERATION STOP */ +#define WT_BLOCK_MODS_RENAME 0x1u /* Entry is from a rename */ +#define WT_BLOCK_MODS_VALID 0x2u /* Entry is valid */ + /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t flags; }; diff --git a/src/third_party/wiredtiger/src/meta/meta_ckpt.c b/src/third_party/wiredtiger/src/meta/meta_ckpt.c index 0edaee8254f..c555279fca6 100644 --- a/src/third_party/wiredtiger/src/meta/meta_ckpt.c +++ b/src/third_party/wiredtiger/src/meta/meta_ckpt.c @@ -11,7 +11,6 @@ static int __ckpt_last(WT_SESSION_IMPL *, const char *, WT_CKPT *); static int __ckpt_last_name(WT_SESSION_IMPL *, const char *, const char **); static int __ckpt_load(WT_SESSION_IMPL *, WT_CONFIG_ITEM *, WT_CONFIG_ITEM *, WT_CKPT *); -static int __ckpt_load_blk_mods(WT_SESSION_IMPL *, const char *, WT_CKPT *); static int __ckpt_named(WT_SESSION_IMPL *, const char *, const char *, WT_CKPT *); static int __ckpt_set(WT_SESSION_IMPL *, const char *, const char *, bool); static int __ckpt_version_chk(WT_SESSION_IMPL *, const char *, const char *); @@ -73,6 +72,16 @@ __ckpt_load_blk_mods(WT_SESSION_IMPL *session, const char *config, WT_CKPT *ckpt blk_mod->nbits = (uint64_t)b.val; WT_RET(__wt_config_subgets(session, &v, "offset", &b)); blk_mod->offset = (uint64_t)b.val; + /* + * The rename configuration string component was added later. So don't error if we don't + * find it in the string. If we don't have it, we're not doing a rename. + */ + ret = __wt_config_subgets(session, &v, "rename", &b); + WT_RET_NOTFOUND_OK(ret); + if (ret == 0 && b.val) + F_SET(blk_mod, WT_BLOCK_MODS_RENAME); + else + F_CLR(blk_mod, WT_BLOCK_MODS_RENAME); ret = __wt_config_subgets(session, &v, "blocks", &b); WT_RET_NOTFOUND_OK(ret); if (ret != WT_NOTFOUND) { @@ -387,7 +396,7 @@ __ckpt_compare_order(const void *a, const void *b) * information. */ static int -__ckpt_valid_blk_mods(WT_SESSION_IMPL *session, WT_CKPT *ckpt) +__ckpt_valid_blk_mods(WT_SESSION_IMPL *session, WT_CKPT *ckpt, bool rename) { WT_BLKINCR *blk; WT_BLOCK_MODS *blk_mod; @@ -425,6 +434,10 @@ __ckpt_valid_blk_mods(WT_SESSION_IMPL *session, WT_CKPT *ckpt) setup = true; } + /* If we are keeping or setting up an entry on a rename, set the flag. */ + if (rename && (!free || setup)) + F_SET(blk_mod, WT_BLOCK_MODS_RENAME); + /* Free any old information if we need to do so. */ if (free && F_ISSET(blk_mod, WT_BLOCK_MODS_VALID)) { __wt_free(session, blk_mod->id_str); @@ -448,6 +461,32 @@ __ckpt_valid_blk_mods(WT_SESSION_IMPL *session, WT_CKPT *ckpt) return (0); } +/* + * __wt_meta_blk_mods_load -- + * Load the block mods for a given checkpoint and set up all the information to store. + */ +int +__wt_meta_blk_mods_load(WT_SESSION_IMPL *session, const char *config, WT_CKPT *ckpt, bool rename) +{ + /* + * Load most recent checkpoint backup blocks to this checkpoint. + */ + WT_RET(__ckpt_load_blk_mods(session, config, ckpt)); + + WT_RET(__wt_meta_block_metadata(session, config, ckpt)); + + /* + * Set the add-a-checkpoint flag, and if we're doing incremental backups, request a list of the + * checkpoint's modified blocks from the block manager. + */ + F_SET(ckpt, WT_CKPT_ADD); + if (F_ISSET(S2C(session), WT_CONN_INCR_BACKUP)) { + F_SET(ckpt, WT_CKPT_BLOCK_MODS); + WT_RET(__ckpt_valid_blk_mods(session, ckpt, rename)); + } + return (0); +} + /* * __wt_meta_ckptlist_get -- * Load all available checkpoint information for a file. @@ -522,22 +561,7 @@ __wt_meta_ckptlist_get( __wt_atomic_cas64(&conn->ckpt_most_recent, most_recent, ckpt->sec)) break; } - /* - * Load most recent checkpoint backup blocks to this checkpoint. - */ - WT_ERR(__ckpt_load_blk_mods(session, config, ckpt)); - - WT_ERR(__wt_meta_block_metadata(session, config, ckpt)); - - /* - * Set the add-a-checkpoint flag, and if we're doing incremental backups, request a list of - * the checkpoint's modified blocks from the block manager. - */ - F_SET(ckpt, WT_CKPT_ADD); - if (F_ISSET(conn, WT_CONN_INCR_BACKUP)) { - F_SET(ckpt, WT_CKPT_BLOCK_MODS); - WT_ERR(__ckpt_valid_blk_mods(session, ckpt)); - } + WT_ERR(__wt_meta_blk_mods_load(session, config, ckpt, false)); } /* Return the array to our caller. */ @@ -763,19 +787,19 @@ __wt_meta_ckptlist_to_meta(WT_SESSION_IMPL *session, WT_CKPT *ckptbase, WT_ITEM } /* - * __ckpt_blkmod_to_meta -- + * __wt_ckpt_blkmod_to_meta -- * Add in any modification block string needed, including an empty one. */ -static int -__ckpt_blkmod_to_meta(WT_SESSION_IMPL *session, WT_ITEM *buf, WT_CKPT *ckpt) +int +__wt_ckpt_blkmod_to_meta(WT_SESSION_IMPL *session, WT_ITEM *buf, WT_CKPT *ckpt) { WT_BLOCK_MODS *blk; WT_ITEM bitstring; u_int i; - bool valid; + bool skip_rename, valid; WT_CLEAR(bitstring); - valid = false; + skip_rename = valid = false; for (i = 0, blk = &ckpt->backup_blocks[0]; i < WT_BLKINCR_MAX; ++i, ++blk) if (F_ISSET(blk, WT_BLOCK_MODS_VALID)) valid = true; @@ -795,11 +819,22 @@ __ckpt_blkmod_to_meta(WT_SESSION_IMPL *session, WT_ITEM *buf, WT_CKPT *ckpt) for (i = 0, blk = &ckpt->backup_blocks[0]; i < WT_BLKINCR_MAX; ++i, ++blk) { if (!F_ISSET(blk, WT_BLOCK_MODS_VALID)) continue; + + /* + * Occasionally skip including the rename string at all when it's not necessary for + * correctness, that lets us simulate what is generated in the config string by earlier + * versions of WiredTiger + */ + if (FLD_ISSET(S2C(session)->timing_stress_flags, WT_TIMING_STRESS_BACKUP_RENAME) && + !F_ISSET(blk, WT_BLOCK_MODS_RENAME) && __wt_random(&session->rnd) % 10 == 0) + skip_rename = true; + WT_RET(__wt_raw_to_hex(session, blk->bitstring.data, blk->bitstring.size, &bitstring)); WT_RET(__wt_buf_catfmt(session, buf, "%s\"%s\"=(id=%" PRIu32 ",granularity=%" PRIu64 ",nbits=%" PRIu64 ",offset=%" PRIu64 - ",blocks=%.*s)", + "%s,blocks=%.*s)", i == 0 ? "" : ",", blk->id_str, i, blk->granularity, blk->nbits, blk->offset, + skip_rename ? "" : F_ISSET(blk, WT_BLOCK_MODS_RENAME) ? ",rename=1" : ",rename=0", (int)bitstring.size, (char *)bitstring.data)); /* The hex string length should match the appropriate number of bits. */ WT_ASSERT(session, (blk->nbits >> 2) <= bitstring.size); @@ -828,7 +863,7 @@ __wt_meta_ckptlist_set( /* Add backup block modifications for any added checkpoint. */ WT_CKPT_FOREACH (ckptbase, ckpt) if (F_ISSET(ckpt, WT_CKPT_ADD)) - WT_ERR(__ckpt_blkmod_to_meta(session, buf, ckpt)); + WT_ERR(__wt_ckpt_blkmod_to_meta(session, buf, ckpt)); has_lsn = ckptlsn != NULL; if (ckptlsn != NULL) diff --git a/src/third_party/wiredtiger/src/schema/schema_rename.c b/src/third_party/wiredtiger/src/schema/schema_rename.c index 3dfc99493c4..53c006ed226 100644 --- a/src/third_party/wiredtiger/src/schema/schema_rename.c +++ b/src/third_party/wiredtiger/src/schema/schema_rename.c @@ -8,6 +8,30 @@ #include "wt_internal.h" +/* + * __rename_blkmod -- + * Reset the incremental backup information for a rename. + */ +static int +__rename_blkmod(WT_SESSION_IMPL *session, const char *oldvalue, WT_ITEM *buf) +{ + WT_CKPT ckpt; + WT_DECL_RET; + + WT_CLEAR(ckpt); + /* + * Replace the old file entries with new file entries. We need to recreate the incremental + * backup information to indicate copying the entire file in its bitmap. + */ + /* First load any existing backup information into a temp checkpoint structure. */ + WT_RET(__wt_meta_blk_mods_load(session, oldvalue, &ckpt, true)); + + /* Take the checkpoint structure and generate the metadata string. */ + ret = __wt_ckpt_blkmod_to_meta(session, buf, &ckpt); + __wt_meta_checkpoint_free(session, &ckpt); + return (ret); +} + /* * __rename_file -- * WT_SESSION::rename for a file. @@ -15,8 +39,10 @@ static int __rename_file(WT_SESSION_IMPL *session, const char *uri, const char *newuri) { + WT_DECL_ITEM(buf); WT_DECL_RET; char *newvalue, *oldvalue; + const char *filecfg[3] = {NULL, NULL, NULL}; const char *filename, *newfile; bool exist; @@ -33,6 +59,7 @@ __rename_file(WT_SESSION_IMPL *session, const char *uri, const char *newuri) WT_WITH_HANDLE_LIST_WRITE_LOCK( session, ret = __wt_conn_dhandle_close_all(session, uri, true, false)); WT_ERR(ret); + WT_ERR(__wt_scr_alloc(session, 1024, &buf)); /* * First, check if the file being renamed exists in the system. Doing this check first matches @@ -54,13 +81,20 @@ __rename_file(WT_SESSION_IMPL *session, const char *uri, const char *newuri) default: WT_ERR(ret); } + __wt_free(session, newvalue); WT_ERR(__wt_fs_exist(session, newfile, &exist)); if (exist) WT_ERR_MSG(session, EEXIST, "%s", newfile); - /* Replace the old file entries with new file entries. */ WT_ERR(__wt_metadata_remove(session, uri)); - WT_ERR(__wt_metadata_insert(session, newuri, oldvalue)); + filecfg[0] = oldvalue; + if (F_ISSET(S2C(session), WT_CONN_INCR_BACKUP)) { + WT_ERR(__rename_blkmod(session, oldvalue, buf)); + filecfg[1] = buf->mem; + } else + filecfg[1] = NULL; + WT_ERR(__wt_config_collapse(session, filecfg, &newvalue)); + WT_ERR(__wt_metadata_insert(session, newuri, newvalue)); /* Rename the underlying file. */ WT_ERR(__wt_fs_rename(session, filename, newfile, false)); @@ -68,6 +102,7 @@ __rename_file(WT_SESSION_IMPL *session, const char *uri, const char *newuri) WT_ERR(__wt_meta_track_fileop(session, uri, newuri)); err: + __wt_scr_free(session, &buf); __wt_free(session, newvalue); __wt_free(session, oldvalue); return (ret); diff --git a/src/third_party/wiredtiger/test/csuite/incr_backup/main.c b/src/third_party/wiredtiger/test/csuite/incr_backup/main.c index 792e10275b2..c9520f5621b 100644 --- a/src/third_party/wiredtiger/test/csuite/incr_backup/main.c +++ b/src/third_party/wiredtiger/test/csuite/incr_backup/main.c @@ -48,6 +48,8 @@ #define URI_FORMAT "table:t%d-%d" #define KEY_FORMAT "key-%d-%d" +#define CONN_CONFIG_COMMON "timing_stress_for_test=[backup_rename]" + static int verbose_level = 0; static uint64_t seed = 0; @@ -59,7 +61,7 @@ static void usage(void) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static bool slow_incremental = false; static bool do_drop = true; -static bool do_rename = false; +static bool do_rename = true; #define VERBOSE(level, fmt, ...) \ do { \ @@ -730,7 +732,7 @@ check_backup(const char *backup_home, const char *backup_check, TABLE_INFO *tinf buf, sizeof(buf), "rm -rf %s && cp -r %s %s", backup_check, backup_home, backup_check)); testutil_check(system(buf)); - testutil_check(wiredtiger_open(backup_check, NULL, NULL, &conn)); + testutil_check(wiredtiger_open(backup_check, NULL, CONN_CONFIG_COMMON, &conn)); testutil_check(conn->open_session(conn, NULL, NULL, &session)); for (slot = 0; slot < tinfo->table_count; slot++) { @@ -818,8 +820,9 @@ main(int argc, char *argv[]) file_max = 200 + __wt_random(&rnd) % 1000; /* 200K to ~1M */ else file_max = 1000 + __wt_random(&rnd) % 20000; /* 1M to ~20M */ - testutil_check(__wt_snprintf(conf, sizeof(conf), - "create,%s,log=(enabled=true,file_max=%" PRIu32 "K)", backup_verbose, file_max)); + testutil_check( + __wt_snprintf(conf, sizeof(conf), "%s,create,%s,log=(enabled=true,file_max=%" PRIu32 "K)", + CONN_CONFIG_COMMON, backup_verbose, file_max)); VERBOSE(2, "wiredtiger config: %s\n", conf); testutil_check(wiredtiger_open(home, NULL, conf, &conn)); testutil_check(conn->open_session(conn, NULL, NULL, &session)); diff --git a/src/third_party/wiredtiger/test/format/config.c b/src/third_party/wiredtiger/test/format/config.c index 5030d99cdf3..a6eb4c5f898 100644 --- a/src/third_party/wiredtiger/test/format/config.c +++ b/src/third_party/wiredtiger/test/format/config.c @@ -380,12 +380,6 @@ config_backward_compatible(void) if (!backward_compatible) return; - if (g.c_backup_incr_flag != INCREMENTAL_OFF) { - if (config_is_perm("backup.incremental")) - testutil_die(EINVAL, "incremental backup not supported in backward compatibility mode"); - config_single("backup.incremental=off", false); - } - if (g.c_mmap_all) { if (config_is_perm("disk.mmap_all")) testutil_die(EINVAL, "disk.mmap_all not supported in backward compatibility mode"); diff --git a/src/third_party/wiredtiger/test/suite/test_hs06.py b/src/third_party/wiredtiger/test/suite/test_hs06.py index b00a196c809..967f1519807 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs06.py +++ b/src/third_party/wiredtiger/test/suite/test_hs06.py @@ -95,7 +95,12 @@ class test_hs06(wttest.WiredTigerTestCase): self.conn.set_timestamp('stable_timestamp=' + timestamp_str(2)) self.session.checkpoint() - # Check the checkpoint wrote the expected values. Todo: Fix checkpoint cursors WT-5492. + # Check the checkpoint wrote the expected values. + # + # FIXME-WT-5927: Checkpoint cursors are known to have issues in durable history so we've + # removing the use of checkpoint handles in this test. As part of WT-5927, we should either + # re-enable the testing of checkpoint cursors or remove this comment. + # # cursor2 = self.session.open_cursor(uri, None, 'checkpoint=WiredTigerCheckpoint') cursor2 = self.session.open_cursor(uri) self.session.begin_transaction('read_timestamp=' + timestamp_str(2)) -- cgit v1.2.1