summaryrefslogtreecommitdiff
path: root/src/third_party
diff options
context:
space:
mode:
authorLuke Chen <luke.chen@mongodb.com>2020-08-28 18:16:31 +1000
committerEvergreen Agent <no-reply@evergreen.mongodb.com>2020-08-28 08:40:07 +0000
commit0c52e9773275d0c4a11c77e56d5ea0f914a903be (patch)
tree7a5278c500125bc6ec761281036701a767b499a9 /src/third_party
parent21dd69e4539f4f1c3fd94b186a074c5631a67cdc (diff)
downloadmongo-0c52e9773275d0c4a11c77e56d5ea0f914a903be.tar.gz
Import wiredtiger: cf3c0c18fe24a595ca523e91f40213a9007e4f29 from branch mongodb-4.6
ref: 1fd34059ee..cf3c0c18fe for: 4.5.1 WT-6610 Fix incremental backup checkpoint parsing to handle upgrades WT-6625 Remove outdated TODO
Diffstat (limited to 'src/third_party')
-rw-r--r--src/third_party/wiredtiger/dist/api_data.py2
-rw-r--r--src/third_party/wiredtiger/import.data2
-rw-r--r--src/third_party/wiredtiger/src/config/config_def.c45
-rw-r--r--src/third_party/wiredtiger/src/conn/conn_api.c1
-rw-r--r--src/third_party/wiredtiger/src/cursor/cur_backup_incr.c27
-rw-r--r--src/third_party/wiredtiger/src/include/connection.h25
-rw-r--r--src/third_party/wiredtiger/src/include/cursor.h19
-rw-r--r--src/third_party/wiredtiger/src/include/extern.h4
-rw-r--r--src/third_party/wiredtiger/src/include/meta.h5
-rw-r--r--src/third_party/wiredtiger/src/meta/meta_ckpt.c85
-rw-r--r--src/third_party/wiredtiger/src/schema/schema_rename.c39
-rw-r--r--src/third_party/wiredtiger/test/csuite/incr_backup/main.c11
-rw-r--r--src/third_party/wiredtiger/test/format/config.c6
-rw-r--r--src/third_party/wiredtiger/test/suite/test_hs06.py7
14 files changed, 187 insertions, 91 deletions
diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py
index d018f94077a..fcc32e3b864 100644
--- a/src/third_party/wiredtiger/dist/api_data.py
+++ b/src/third_party/wiredtiger/dist/api_data.py
@@ -722,7 +722,7 @@ connection_runtime_config = [
intended for use with internal stress testing of WiredTiger.''',
type='list', undoc=True,
choices=[
- 'aggressive_sweep', 'checkpoint_slow', 'history_store_checkpoint_delay',
+ 'aggressive_sweep', 'backup_rename', 'checkpoint_slow', 'history_store_checkpoint_delay',
'history_store_sweep_race', 'prepare_checkpoint_delay', 'split_1', 'split_2',
'split_3', 'split_4', 'split_5', 'split_6', 'split_7', 'split_8']),
Config('verbose', '', r'''
diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data
index 00ddac8eea9..a339ccd3624 100644
--- a/src/third_party/wiredtiger/import.data
+++ b/src/third_party/wiredtiger/import.data
@@ -2,5 +2,5 @@
"vendor": "wiredtiger",
"github": "wiredtiger/wiredtiger.git",
"branch": "mongodb-4.6",
- "commit": "1fd34059ee6ad303fe1f0200fd799dcb290df277"
+ "commit": "cf3c0c18fe24a595ca523e91f40213a9007e4f29"
}
diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c
index ed4960362af..4b5248d6764 100644
--- a/src/third_party/wiredtiger/src/config/config_def.c
+++ b/src/third_party/wiredtiger/src/config/config_def.c
@@ -136,10 +136,11 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = {
{"statistics_log", "category", NULL, NULL,
confchk_WT_CONNECTION_reconfigure_statistics_log_subconfigs, 5},
{"timing_stress_for_test", "list", NULL,
- "choices=[\"aggressive_sweep\",\"checkpoint_slow\","
- "\"history_store_checkpoint_delay\",\"history_store_sweep_race\","
- "\"prepare_checkpoint_delay\",\"split_1\",\"split_2\",\"split_3\""
- ",\"split_4\",\"split_5\",\"split_6\",\"split_7\",\"split_8\"]",
+ "choices=[\"aggressive_sweep\",\"backup_rename\","
+ "\"checkpoint_slow\",\"history_store_checkpoint_delay\","
+ "\"history_store_sweep_race\",\"prepare_checkpoint_delay\","
+ "\"split_1\",\"split_2\",\"split_3\",\"split_4\",\"split_5\","
+ "\"split_6\",\"split_7\",\"split_8\"]",
NULL, 0},
{"verbose", "list", NULL,
"choices=[\"api\",\"backup\",\"block\",\"checkpoint\","
@@ -589,10 +590,11 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = {
NULL, 0},
{"statistics_log", "category", NULL, NULL, confchk_wiredtiger_open_statistics_log_subconfigs, 6},
{"timing_stress_for_test", "list", NULL,
- "choices=[\"aggressive_sweep\",\"checkpoint_slow\","
- "\"history_store_checkpoint_delay\",\"history_store_sweep_race\","
- "\"prepare_checkpoint_delay\",\"split_1\",\"split_2\",\"split_3\""
- ",\"split_4\",\"split_5\",\"split_6\",\"split_7\",\"split_8\"]",
+ "choices=[\"aggressive_sweep\",\"backup_rename\","
+ "\"checkpoint_slow\",\"history_store_checkpoint_delay\","
+ "\"history_store_sweep_race\",\"prepare_checkpoint_delay\","
+ "\"split_1\",\"split_2\",\"split_3\",\"split_4\",\"split_5\","
+ "\"split_6\",\"split_7\",\"split_8\"]",
NULL, 0},
{"transaction_sync", "category", NULL, NULL, confchk_wiredtiger_open_transaction_sync_subconfigs,
2},
@@ -666,10 +668,11 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = {
NULL, 0},
{"statistics_log", "category", NULL, NULL, confchk_wiredtiger_open_statistics_log_subconfigs, 6},
{"timing_stress_for_test", "list", NULL,
- "choices=[\"aggressive_sweep\",\"checkpoint_slow\","
- "\"history_store_checkpoint_delay\",\"history_store_sweep_race\","
- "\"prepare_checkpoint_delay\",\"split_1\",\"split_2\",\"split_3\""
- ",\"split_4\",\"split_5\",\"split_6\",\"split_7\",\"split_8\"]",
+ "choices=[\"aggressive_sweep\",\"backup_rename\","
+ "\"checkpoint_slow\",\"history_store_checkpoint_delay\","
+ "\"history_store_sweep_race\",\"prepare_checkpoint_delay\","
+ "\"split_1\",\"split_2\",\"split_3\",\"split_4\",\"split_5\","
+ "\"split_6\",\"split_7\",\"split_8\"]",
NULL, 0},
{"transaction_sync", "category", NULL, NULL, confchk_wiredtiger_open_transaction_sync_subconfigs,
2},
@@ -740,10 +743,11 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = {
NULL, 0},
{"statistics_log", "category", NULL, NULL, confchk_wiredtiger_open_statistics_log_subconfigs, 6},
{"timing_stress_for_test", "list", NULL,
- "choices=[\"aggressive_sweep\",\"checkpoint_slow\","
- "\"history_store_checkpoint_delay\",\"history_store_sweep_race\","
- "\"prepare_checkpoint_delay\",\"split_1\",\"split_2\",\"split_3\""
- ",\"split_4\",\"split_5\",\"split_6\",\"split_7\",\"split_8\"]",
+ "choices=[\"aggressive_sweep\",\"backup_rename\","
+ "\"checkpoint_slow\",\"history_store_checkpoint_delay\","
+ "\"history_store_sweep_race\",\"prepare_checkpoint_delay\","
+ "\"split_1\",\"split_2\",\"split_3\",\"split_4\",\"split_5\","
+ "\"split_6\",\"split_7\",\"split_8\"]",
NULL, 0},
{"transaction_sync", "category", NULL, NULL, confchk_wiredtiger_open_transaction_sync_subconfigs,
2},
@@ -812,10 +816,11 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = {
NULL, 0},
{"statistics_log", "category", NULL, NULL, confchk_wiredtiger_open_statistics_log_subconfigs, 6},
{"timing_stress_for_test", "list", NULL,
- "choices=[\"aggressive_sweep\",\"checkpoint_slow\","
- "\"history_store_checkpoint_delay\",\"history_store_sweep_race\","
- "\"prepare_checkpoint_delay\",\"split_1\",\"split_2\",\"split_3\""
- ",\"split_4\",\"split_5\",\"split_6\",\"split_7\",\"split_8\"]",
+ "choices=[\"aggressive_sweep\",\"backup_rename\","
+ "\"checkpoint_slow\",\"history_store_checkpoint_delay\","
+ "\"history_store_sweep_race\",\"prepare_checkpoint_delay\","
+ "\"split_1\",\"split_2\",\"split_3\",\"split_4\",\"split_5\","
+ "\"split_6\",\"split_7\",\"split_8\"]",
NULL, 0},
{"transaction_sync", "category", NULL, NULL, confchk_wiredtiger_open_transaction_sync_subconfigs,
2},
diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c
index ef55e8ec0bd..5d8c8b777d5 100644
--- a/src/third_party/wiredtiger/src/conn/conn_api.c
+++ b/src/third_party/wiredtiger/src/conn/conn_api.c
@@ -2035,6 +2035,7 @@ __wt_timing_stress_config(WT_SESSION_IMPL *session, const char *cfg[])
*/
static const WT_NAME_FLAG stress_types[] = {
{"aggressive_sweep", WT_TIMING_STRESS_AGGRESSIVE_SWEEP},
+ {"backup_rename", WT_TIMING_STRESS_BACKUP_RENAME},
{"checkpoint_slow", WT_TIMING_STRESS_CHECKPOINT_SLOW},
{"history_store_checkpoint_delay", WT_TIMING_STRESS_HS_CHECKPOINT_DELAY},
{"history_store_sweep_race", WT_TIMING_STRESS_HS_SWEEP},
diff --git a/src/third_party/wiredtiger/src/cursor/cur_backup_incr.c b/src/third_party/wiredtiger/src/cursor/cur_backup_incr.c
index f30bd1c4d3b..c71676b2082 100644
--- a/src/third_party/wiredtiger/src/cursor/cur_backup_incr.c
+++ b/src/third_party/wiredtiger/src/cursor/cur_backup_incr.c
@@ -70,6 +70,15 @@ __curbackup_incr_blkmod(WT_SESSION_IMPL *session, WT_BTREE *btree, WT_CURSOR_BAC
cb->nbits = (uint64_t)b.val;
WT_ERR(__wt_config_subgets(session, &v, "offset", &b));
cb->offset = (uint64_t)b.val;
+ /*
+ * The rename configuration string component was added later. So don't error if we don't
+ * find it in the string. If we don't have it, we're not doing a rename.
+ */
+ WT_ERR_NOTFOUND_OK(__wt_config_subgets(session, &v, "rename", &b), true);
+ if (ret == 0 && b.val)
+ F_SET(cb, WT_CURBACKUP_RENAME);
+ else
+ F_CLR(cb, WT_CURBACKUP_RENAME);
/*
* We found a match. Load the block information into the cursor.
@@ -112,7 +121,7 @@ __curbackup_incr_next(WT_CURSOR *cursor)
F_CLR(cursor, WT_CURSTD_RAW);
if (!F_ISSET(cb, WT_CURBACKUP_INCR_INIT) &&
- (btree == NULL || F_ISSET(cb, WT_CURBACKUP_FORCE_FULL))) {
+ (btree == NULL || F_ISSET(cb, WT_CURBACKUP_FORCE_FULL | WT_CURBACKUP_RENAME))) {
/*
* We don't have this object's incremental information or it's a forced file copy. If this
* is a log file, use the full pathname that may include the log path.
@@ -155,19 +164,21 @@ __curbackup_incr_next(WT_CURSOR *cursor)
*/
WT_ERR(__curbackup_incr_blkmod(session, btree, cb));
/*
- * There are three cases where we do not have block modification information for
+ * There are several cases where we do not have block modification information for
* the file. They are described and handled as follows:
*
- * 1. Newly created file without checkpoint information. Return the whole file
- * information.
- * 2. File created and checkpointed before incremental backups were configured.
+ * 1. Renamed file. Always return the whole file information.
+ * 2. Newly created file without checkpoint information. Return the whole
+ * file information.
+ * 3. File created and checkpointed before incremental backups were configured.
* Return no file information as it was copied in the initial full backup.
- * 3. File that has not been modified since the previous incremental backup.
+ * 4. File that has not been modified since the previous incremental backup.
* Return no file information as there is no new information.
*/
- if (cb->bitstring.mem == NULL) {
+ if (cb->bitstring.mem == NULL || F_ISSET(cb, WT_CURBACKUP_RENAME)) {
F_SET(cb, WT_CURBACKUP_INCR_INIT);
- if (F_ISSET(cb, WT_CURBACKUP_CKPT_FAKE) && F_ISSET(cb, WT_CURBACKUP_HAS_CB_INFO)) {
+ if (F_ISSET(cb, WT_CURBACKUP_RENAME) ||
+ (F_ISSET(cb, WT_CURBACKUP_CKPT_FAKE) && F_ISSET(cb, WT_CURBACKUP_HAS_CB_INFO))) {
WT_ERR(__wt_fs_size(session, cb->incr_file, &size));
__wt_cursor_set_key(cursor, 0, size, WT_BACKUP_FILE);
goto done;
diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h
index 80568a48bc9..48b7784d23f 100644
--- a/src/third_party/wiredtiger/src/include/connection.h
+++ b/src/third_party/wiredtiger/src/include/connection.h
@@ -509,18 +509,19 @@ struct __wt_connection_impl {
*/
/* AUTOMATIC FLAG VALUE GENERATION START */
#define WT_TIMING_STRESS_AGGRESSIVE_SWEEP 0x0001u
-#define WT_TIMING_STRESS_CHECKPOINT_SLOW 0x0002u
-#define WT_TIMING_STRESS_HS_CHECKPOINT_DELAY 0x0004u
-#define WT_TIMING_STRESS_HS_SWEEP 0x0008u
-#define WT_TIMING_STRESS_PREPARE_CHECKPOINT_DELAY 0x0010u
-#define WT_TIMING_STRESS_SPLIT_1 0x0020u
-#define WT_TIMING_STRESS_SPLIT_2 0x0040u
-#define WT_TIMING_STRESS_SPLIT_3 0x0080u
-#define WT_TIMING_STRESS_SPLIT_4 0x0100u
-#define WT_TIMING_STRESS_SPLIT_5 0x0200u
-#define WT_TIMING_STRESS_SPLIT_6 0x0400u
-#define WT_TIMING_STRESS_SPLIT_7 0x0800u
-#define WT_TIMING_STRESS_SPLIT_8 0x1000u
+#define WT_TIMING_STRESS_BACKUP_RENAME 0x0002u
+#define WT_TIMING_STRESS_CHECKPOINT_SLOW 0x0004u
+#define WT_TIMING_STRESS_HS_CHECKPOINT_DELAY 0x0008u
+#define WT_TIMING_STRESS_HS_SWEEP 0x0010u
+#define WT_TIMING_STRESS_PREPARE_CHECKPOINT_DELAY 0x0020u
+#define WT_TIMING_STRESS_SPLIT_1 0x0040u
+#define WT_TIMING_STRESS_SPLIT_2 0x0080u
+#define WT_TIMING_STRESS_SPLIT_3 0x0100u
+#define WT_TIMING_STRESS_SPLIT_4 0x0200u
+#define WT_TIMING_STRESS_SPLIT_5 0x0400u
+#define WT_TIMING_STRESS_SPLIT_6 0x0800u
+#define WT_TIMING_STRESS_SPLIT_7 0x1000u
+#define WT_TIMING_STRESS_SPLIT_8 0x2000u
/* AUTOMATIC FLAG VALUE GENERATION STOP */
uint64_t timing_stress_flags;
diff --git a/src/third_party/wiredtiger/src/include/cursor.h b/src/third_party/wiredtiger/src/include/cursor.h
index 1cb3fae58bd..c344710120d 100644
--- a/src/third_party/wiredtiger/src/include/cursor.h
+++ b/src/third_party/wiredtiger/src/include/cursor.h
@@ -61,15 +61,16 @@ struct __wt_cursor_backup {
uint64_t granularity; /* Length, transfer size */
/* AUTOMATIC FLAG VALUE GENERATION START */
-#define WT_CURBACKUP_CKPT_FAKE 0x01u /* Object has fake checkpoint */
-#define WT_CURBACKUP_DUP 0x02u /* Duplicated backup cursor */
-#define WT_CURBACKUP_FORCE_FULL 0x04u /* Force full file copy for this cursor */
-#define WT_CURBACKUP_FORCE_STOP 0x08u /* Force stop incremental backup */
-#define WT_CURBACKUP_HAS_CB_INFO 0x10u /* Object has checkpoint backup info */
-#define WT_CURBACKUP_INCR 0x20u /* Incremental backup cursor */
-#define WT_CURBACKUP_INCR_INIT 0x40u /* Cursor traversal initialized */
-#define WT_CURBACKUP_LOCKER 0x80u /* Hot-backup started */
- /* AUTOMATIC FLAG VALUE GENERATION STOP */
+#define WT_CURBACKUP_CKPT_FAKE 0x001u /* Object has fake checkpoint */
+#define WT_CURBACKUP_DUP 0x002u /* Duplicated backup cursor */
+#define WT_CURBACKUP_FORCE_FULL 0x004u /* Force full file copy for this cursor */
+#define WT_CURBACKUP_FORCE_STOP 0x008u /* Force stop incremental backup */
+#define WT_CURBACKUP_HAS_CB_INFO 0x010u /* Object has checkpoint backup info */
+#define WT_CURBACKUP_INCR 0x020u /* Incremental backup cursor */
+#define WT_CURBACKUP_INCR_INIT 0x040u /* Cursor traversal initialized */
+#define WT_CURBACKUP_LOCKER 0x080u /* Hot-backup started */
+#define WT_CURBACKUP_RENAME 0x100u /* Object had a rename */
+ /* AUTOMATIC FLAG VALUE GENERATION STOP */
uint32_t flags;
};
diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h
index 90f4eee7884..c8b4427f972 100644
--- a/src/third_party/wiredtiger/src/include/extern.h
+++ b/src/third_party/wiredtiger/src/include/extern.h
@@ -358,6 +358,8 @@ extern int __wt_checkpoint_server_destroy(WT_SESSION_IMPL *session)
WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern int __wt_checkpoint_sync(WT_SESSION_IMPL *session, const char *cfg[])
WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
+extern int __wt_ckpt_blkmod_to_meta(WT_SESSION_IMPL *session, WT_ITEM *buf, WT_CKPT *ckpt)
+ WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern int __wt_close(WT_SESSION_IMPL *session, WT_FH **fhp)
WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern int __wt_close_connection_close(WT_SESSION_IMPL *session)
@@ -1032,6 +1034,8 @@ extern int __wt_meta_apply_all(WT_SESSION_IMPL *session,
int (*file_func)(WT_SESSION_IMPL *, const char *[]),
int (*name_func)(WT_SESSION_IMPL *, const char *, bool *), const char *cfg[])
WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
+extern int __wt_meta_blk_mods_load(WT_SESSION_IMPL *session, const char *config, WT_CKPT *ckpt,
+ bool rename) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern int __wt_meta_block_metadata(WT_SESSION_IMPL *session, const char *config, WT_CKPT *ckpt)
WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern int __wt_meta_checkpoint(WT_SESSION_IMPL *session, const char *fname, const char *checkpoint,
diff --git a/src/third_party/wiredtiger/src/include/meta.h b/src/third_party/wiredtiger/src/include/meta.h
index dac2cc59573..a0241a22ce7 100644
--- a/src/third_party/wiredtiger/src/include/meta.h
+++ b/src/third_party/wiredtiger/src/include/meta.h
@@ -98,8 +98,9 @@ struct __wt_block_mods {
uint64_t offset; /* Zero bit offset for bitstring */
uint64_t granularity;
/* AUTOMATIC FLAG VALUE GENERATION START */
-#define WT_BLOCK_MODS_VALID 0x1u /* Entry is valid */
- /* AUTOMATIC FLAG VALUE GENERATION STOP */
+#define WT_BLOCK_MODS_RENAME 0x1u /* Entry is from a rename */
+#define WT_BLOCK_MODS_VALID 0x2u /* Entry is valid */
+ /* AUTOMATIC FLAG VALUE GENERATION STOP */
uint32_t flags;
};
diff --git a/src/third_party/wiredtiger/src/meta/meta_ckpt.c b/src/third_party/wiredtiger/src/meta/meta_ckpt.c
index 0edaee8254f..c555279fca6 100644
--- a/src/third_party/wiredtiger/src/meta/meta_ckpt.c
+++ b/src/third_party/wiredtiger/src/meta/meta_ckpt.c
@@ -11,7 +11,6 @@
static int __ckpt_last(WT_SESSION_IMPL *, const char *, WT_CKPT *);
static int __ckpt_last_name(WT_SESSION_IMPL *, const char *, const char **);
static int __ckpt_load(WT_SESSION_IMPL *, WT_CONFIG_ITEM *, WT_CONFIG_ITEM *, WT_CKPT *);
-static int __ckpt_load_blk_mods(WT_SESSION_IMPL *, const char *, WT_CKPT *);
static int __ckpt_named(WT_SESSION_IMPL *, const char *, const char *, WT_CKPT *);
static int __ckpt_set(WT_SESSION_IMPL *, const char *, const char *, bool);
static int __ckpt_version_chk(WT_SESSION_IMPL *, const char *, const char *);
@@ -73,6 +72,16 @@ __ckpt_load_blk_mods(WT_SESSION_IMPL *session, const char *config, WT_CKPT *ckpt
blk_mod->nbits = (uint64_t)b.val;
WT_RET(__wt_config_subgets(session, &v, "offset", &b));
blk_mod->offset = (uint64_t)b.val;
+ /*
+ * The rename configuration string component was added later. So don't error if we don't
+ * find it in the string. If we don't have it, we're not doing a rename.
+ */
+ ret = __wt_config_subgets(session, &v, "rename", &b);
+ WT_RET_NOTFOUND_OK(ret);
+ if (ret == 0 && b.val)
+ F_SET(blk_mod, WT_BLOCK_MODS_RENAME);
+ else
+ F_CLR(blk_mod, WT_BLOCK_MODS_RENAME);
ret = __wt_config_subgets(session, &v, "blocks", &b);
WT_RET_NOTFOUND_OK(ret);
if (ret != WT_NOTFOUND) {
@@ -387,7 +396,7 @@ __ckpt_compare_order(const void *a, const void *b)
* information.
*/
static int
-__ckpt_valid_blk_mods(WT_SESSION_IMPL *session, WT_CKPT *ckpt)
+__ckpt_valid_blk_mods(WT_SESSION_IMPL *session, WT_CKPT *ckpt, bool rename)
{
WT_BLKINCR *blk;
WT_BLOCK_MODS *blk_mod;
@@ -425,6 +434,10 @@ __ckpt_valid_blk_mods(WT_SESSION_IMPL *session, WT_CKPT *ckpt)
setup = true;
}
+ /* If we are keeping or setting up an entry on a rename, set the flag. */
+ if (rename && (!free || setup))
+ F_SET(blk_mod, WT_BLOCK_MODS_RENAME);
+
/* Free any old information if we need to do so. */
if (free && F_ISSET(blk_mod, WT_BLOCK_MODS_VALID)) {
__wt_free(session, blk_mod->id_str);
@@ -449,6 +462,32 @@ __ckpt_valid_blk_mods(WT_SESSION_IMPL *session, WT_CKPT *ckpt)
}
/*
+ * __wt_meta_blk_mods_load --
+ * Load the block mods for a given checkpoint and set up all the information to store.
+ */
+int
+__wt_meta_blk_mods_load(WT_SESSION_IMPL *session, const char *config, WT_CKPT *ckpt, bool rename)
+{
+ /*
+ * Load most recent checkpoint backup blocks to this checkpoint.
+ */
+ WT_RET(__ckpt_load_blk_mods(session, config, ckpt));
+
+ WT_RET(__wt_meta_block_metadata(session, config, ckpt));
+
+ /*
+ * Set the add-a-checkpoint flag, and if we're doing incremental backups, request a list of the
+ * checkpoint's modified blocks from the block manager.
+ */
+ F_SET(ckpt, WT_CKPT_ADD);
+ if (F_ISSET(S2C(session), WT_CONN_INCR_BACKUP)) {
+ F_SET(ckpt, WT_CKPT_BLOCK_MODS);
+ WT_RET(__ckpt_valid_blk_mods(session, ckpt, rename));
+ }
+ return (0);
+}
+
+/*
* __wt_meta_ckptlist_get --
* Load all available checkpoint information for a file.
*/
@@ -522,22 +561,7 @@ __wt_meta_ckptlist_get(
__wt_atomic_cas64(&conn->ckpt_most_recent, most_recent, ckpt->sec))
break;
}
- /*
- * Load most recent checkpoint backup blocks to this checkpoint.
- */
- WT_ERR(__ckpt_load_blk_mods(session, config, ckpt));
-
- WT_ERR(__wt_meta_block_metadata(session, config, ckpt));
-
- /*
- * Set the add-a-checkpoint flag, and if we're doing incremental backups, request a list of
- * the checkpoint's modified blocks from the block manager.
- */
- F_SET(ckpt, WT_CKPT_ADD);
- if (F_ISSET(conn, WT_CONN_INCR_BACKUP)) {
- F_SET(ckpt, WT_CKPT_BLOCK_MODS);
- WT_ERR(__ckpt_valid_blk_mods(session, ckpt));
- }
+ WT_ERR(__wt_meta_blk_mods_load(session, config, ckpt, false));
}
/* Return the array to our caller. */
@@ -763,19 +787,19 @@ __wt_meta_ckptlist_to_meta(WT_SESSION_IMPL *session, WT_CKPT *ckptbase, WT_ITEM
}
/*
- * __ckpt_blkmod_to_meta --
+ * __wt_ckpt_blkmod_to_meta --
* Add in any modification block string needed, including an empty one.
*/
-static int
-__ckpt_blkmod_to_meta(WT_SESSION_IMPL *session, WT_ITEM *buf, WT_CKPT *ckpt)
+int
+__wt_ckpt_blkmod_to_meta(WT_SESSION_IMPL *session, WT_ITEM *buf, WT_CKPT *ckpt)
{
WT_BLOCK_MODS *blk;
WT_ITEM bitstring;
u_int i;
- bool valid;
+ bool skip_rename, valid;
WT_CLEAR(bitstring);
- valid = false;
+ skip_rename = valid = false;
for (i = 0, blk = &ckpt->backup_blocks[0]; i < WT_BLKINCR_MAX; ++i, ++blk)
if (F_ISSET(blk, WT_BLOCK_MODS_VALID))
valid = true;
@@ -795,11 +819,22 @@ __ckpt_blkmod_to_meta(WT_SESSION_IMPL *session, WT_ITEM *buf, WT_CKPT *ckpt)
for (i = 0, blk = &ckpt->backup_blocks[0]; i < WT_BLKINCR_MAX; ++i, ++blk) {
if (!F_ISSET(blk, WT_BLOCK_MODS_VALID))
continue;
+
+ /*
+ * Occasionally skip including the rename string at all when it's not necessary for
+ * correctness, that lets us simulate what is generated in the config string by earlier
+ * versions of WiredTiger
+ */
+ if (FLD_ISSET(S2C(session)->timing_stress_flags, WT_TIMING_STRESS_BACKUP_RENAME) &&
+ !F_ISSET(blk, WT_BLOCK_MODS_RENAME) && __wt_random(&session->rnd) % 10 == 0)
+ skip_rename = true;
+
WT_RET(__wt_raw_to_hex(session, blk->bitstring.data, blk->bitstring.size, &bitstring));
WT_RET(__wt_buf_catfmt(session, buf,
"%s\"%s\"=(id=%" PRIu32 ",granularity=%" PRIu64 ",nbits=%" PRIu64 ",offset=%" PRIu64
- ",blocks=%.*s)",
+ "%s,blocks=%.*s)",
i == 0 ? "" : ",", blk->id_str, i, blk->granularity, blk->nbits, blk->offset,
+ skip_rename ? "" : F_ISSET(blk, WT_BLOCK_MODS_RENAME) ? ",rename=1" : ",rename=0",
(int)bitstring.size, (char *)bitstring.data));
/* The hex string length should match the appropriate number of bits. */
WT_ASSERT(session, (blk->nbits >> 2) <= bitstring.size);
@@ -828,7 +863,7 @@ __wt_meta_ckptlist_set(
/* Add backup block modifications for any added checkpoint. */
WT_CKPT_FOREACH (ckptbase, ckpt)
if (F_ISSET(ckpt, WT_CKPT_ADD))
- WT_ERR(__ckpt_blkmod_to_meta(session, buf, ckpt));
+ WT_ERR(__wt_ckpt_blkmod_to_meta(session, buf, ckpt));
has_lsn = ckptlsn != NULL;
if (ckptlsn != NULL)
diff --git a/src/third_party/wiredtiger/src/schema/schema_rename.c b/src/third_party/wiredtiger/src/schema/schema_rename.c
index 3dfc99493c4..53c006ed226 100644
--- a/src/third_party/wiredtiger/src/schema/schema_rename.c
+++ b/src/third_party/wiredtiger/src/schema/schema_rename.c
@@ -9,14 +9,40 @@
#include "wt_internal.h"
/*
+ * __rename_blkmod --
+ * Reset the incremental backup information for a rename.
+ */
+static int
+__rename_blkmod(WT_SESSION_IMPL *session, const char *oldvalue, WT_ITEM *buf)
+{
+ WT_CKPT ckpt;
+ WT_DECL_RET;
+
+ WT_CLEAR(ckpt);
+ /*
+ * Replace the old file entries with new file entries. We need to recreate the incremental
+ * backup information to indicate copying the entire file in its bitmap.
+ */
+ /* First load any existing backup information into a temp checkpoint structure. */
+ WT_RET(__wt_meta_blk_mods_load(session, oldvalue, &ckpt, true));
+
+ /* Take the checkpoint structure and generate the metadata string. */
+ ret = __wt_ckpt_blkmod_to_meta(session, buf, &ckpt);
+ __wt_meta_checkpoint_free(session, &ckpt);
+ return (ret);
+}
+
+/*
* __rename_file --
* WT_SESSION::rename for a file.
*/
static int
__rename_file(WT_SESSION_IMPL *session, const char *uri, const char *newuri)
{
+ WT_DECL_ITEM(buf);
WT_DECL_RET;
char *newvalue, *oldvalue;
+ const char *filecfg[3] = {NULL, NULL, NULL};
const char *filename, *newfile;
bool exist;
@@ -33,6 +59,7 @@ __rename_file(WT_SESSION_IMPL *session, const char *uri, const char *newuri)
WT_WITH_HANDLE_LIST_WRITE_LOCK(
session, ret = __wt_conn_dhandle_close_all(session, uri, true, false));
WT_ERR(ret);
+ WT_ERR(__wt_scr_alloc(session, 1024, &buf));
/*
* First, check if the file being renamed exists in the system. Doing this check first matches
@@ -54,13 +81,20 @@ __rename_file(WT_SESSION_IMPL *session, const char *uri, const char *newuri)
default:
WT_ERR(ret);
}
+ __wt_free(session, newvalue);
WT_ERR(__wt_fs_exist(session, newfile, &exist));
if (exist)
WT_ERR_MSG(session, EEXIST, "%s", newfile);
- /* Replace the old file entries with new file entries. */
WT_ERR(__wt_metadata_remove(session, uri));
- WT_ERR(__wt_metadata_insert(session, newuri, oldvalue));
+ filecfg[0] = oldvalue;
+ if (F_ISSET(S2C(session), WT_CONN_INCR_BACKUP)) {
+ WT_ERR(__rename_blkmod(session, oldvalue, buf));
+ filecfg[1] = buf->mem;
+ } else
+ filecfg[1] = NULL;
+ WT_ERR(__wt_config_collapse(session, filecfg, &newvalue));
+ WT_ERR(__wt_metadata_insert(session, newuri, newvalue));
/* Rename the underlying file. */
WT_ERR(__wt_fs_rename(session, filename, newfile, false));
@@ -68,6 +102,7 @@ __rename_file(WT_SESSION_IMPL *session, const char *uri, const char *newuri)
WT_ERR(__wt_meta_track_fileop(session, uri, newuri));
err:
+ __wt_scr_free(session, &buf);
__wt_free(session, newvalue);
__wt_free(session, oldvalue);
return (ret);
diff --git a/src/third_party/wiredtiger/test/csuite/incr_backup/main.c b/src/third_party/wiredtiger/test/csuite/incr_backup/main.c
index 792e10275b2..c9520f5621b 100644
--- a/src/third_party/wiredtiger/test/csuite/incr_backup/main.c
+++ b/src/third_party/wiredtiger/test/csuite/incr_backup/main.c
@@ -48,6 +48,8 @@
#define URI_FORMAT "table:t%d-%d"
#define KEY_FORMAT "key-%d-%d"
+#define CONN_CONFIG_COMMON "timing_stress_for_test=[backup_rename]"
+
static int verbose_level = 0;
static uint64_t seed = 0;
@@ -59,7 +61,7 @@ static void usage(void) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn));
static bool slow_incremental = false;
static bool do_drop = true;
-static bool do_rename = false;
+static bool do_rename = true;
#define VERBOSE(level, fmt, ...) \
do { \
@@ -730,7 +732,7 @@ check_backup(const char *backup_home, const char *backup_check, TABLE_INFO *tinf
buf, sizeof(buf), "rm -rf %s && cp -r %s %s", backup_check, backup_home, backup_check));
testutil_check(system(buf));
- testutil_check(wiredtiger_open(backup_check, NULL, NULL, &conn));
+ testutil_check(wiredtiger_open(backup_check, NULL, CONN_CONFIG_COMMON, &conn));
testutil_check(conn->open_session(conn, NULL, NULL, &session));
for (slot = 0; slot < tinfo->table_count; slot++) {
@@ -818,8 +820,9 @@ main(int argc, char *argv[])
file_max = 200 + __wt_random(&rnd) % 1000; /* 200K to ~1M */
else
file_max = 1000 + __wt_random(&rnd) % 20000; /* 1M to ~20M */
- testutil_check(__wt_snprintf(conf, sizeof(conf),
- "create,%s,log=(enabled=true,file_max=%" PRIu32 "K)", backup_verbose, file_max));
+ testutil_check(
+ __wt_snprintf(conf, sizeof(conf), "%s,create,%s,log=(enabled=true,file_max=%" PRIu32 "K)",
+ CONN_CONFIG_COMMON, backup_verbose, file_max));
VERBOSE(2, "wiredtiger config: %s\n", conf);
testutil_check(wiredtiger_open(home, NULL, conf, &conn));
testutil_check(conn->open_session(conn, NULL, NULL, &session));
diff --git a/src/third_party/wiredtiger/test/format/config.c b/src/third_party/wiredtiger/test/format/config.c
index 5030d99cdf3..a6eb4c5f898 100644
--- a/src/third_party/wiredtiger/test/format/config.c
+++ b/src/third_party/wiredtiger/test/format/config.c
@@ -380,12 +380,6 @@ config_backward_compatible(void)
if (!backward_compatible)
return;
- if (g.c_backup_incr_flag != INCREMENTAL_OFF) {
- if (config_is_perm("backup.incremental"))
- testutil_die(EINVAL, "incremental backup not supported in backward compatibility mode");
- config_single("backup.incremental=off", false);
- }
-
if (g.c_mmap_all) {
if (config_is_perm("disk.mmap_all"))
testutil_die(EINVAL, "disk.mmap_all not supported in backward compatibility mode");
diff --git a/src/third_party/wiredtiger/test/suite/test_hs06.py b/src/third_party/wiredtiger/test/suite/test_hs06.py
index b00a196c809..967f1519807 100644
--- a/src/third_party/wiredtiger/test/suite/test_hs06.py
+++ b/src/third_party/wiredtiger/test/suite/test_hs06.py
@@ -95,7 +95,12 @@ class test_hs06(wttest.WiredTigerTestCase):
self.conn.set_timestamp('stable_timestamp=' + timestamp_str(2))
self.session.checkpoint()
- # Check the checkpoint wrote the expected values. Todo: Fix checkpoint cursors WT-5492.
+ # Check the checkpoint wrote the expected values.
+ #
+ # FIXME-WT-5927: Checkpoint cursors are known to have issues in durable history so we've
+ # removing the use of checkpoint handles in this test. As part of WT-5927, we should either
+ # re-enable the testing of checkpoint cursors or remove this comment.
+ #
# cursor2 = self.session.open_cursor(uri, None, 'checkpoint=WiredTigerCheckpoint')
cursor2 = self.session.open_cursor(uri)
self.session.begin_transaction('read_timestamp=' + timestamp_str(2))