From 29e21d0ff597e6dd10bbb9bab4086612339443ca Mon Sep 17 00:00:00 2001 From: Luke Chen Date: Mon, 24 Aug 2020 16:07:45 +1000 Subject: Import wiredtiger: d437e51e78120ab2999ffe40c0b6d46f3f878126 from branch mongodb-4.4 ref: 9c79fedf12..d437e51e78 for: 4.4.1 WT-6611 Revert enhancement allowing rename and incremental backup --- src/third_party/wiredtiger/import.data | 2 +- .../wiredtiger/src/cursor/cur_backup_incr.c | 23 +++----- src/third_party/wiredtiger/src/include/cursor.h | 19 +++--- src/third_party/wiredtiger/src/include/extern.h | 4 -- src/third_party/wiredtiger/src/include/meta.h | 5 +- src/third_party/wiredtiger/src/meta/meta_ckpt.c | 67 ++++++++-------------- .../wiredtiger/src/schema/schema_rename.c | 39 +------------ .../wiredtiger/test/csuite/incr_backup/main.c | 2 +- 8 files changed, 47 insertions(+), 114 deletions(-) diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 5b1b9bc03aa..2f62e46c2e0 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-4.4", - "commit": "9c79fedf125e1d63250d767a8bddffcfa9fb7757" + "commit": "d437e51e78120ab2999ffe40c0b6d46f3f878126" } diff --git a/src/third_party/wiredtiger/src/cursor/cur_backup_incr.c b/src/third_party/wiredtiger/src/cursor/cur_backup_incr.c index 8ddb771a403..f30bd1c4d3b 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_backup_incr.c +++ b/src/third_party/wiredtiger/src/cursor/cur_backup_incr.c @@ -70,11 +70,6 @@ __curbackup_incr_blkmod(WT_SESSION_IMPL *session, WT_BTREE *btree, WT_CURSOR_BAC cb->nbits = (uint64_t)b.val; WT_ERR(__wt_config_subgets(session, &v, "offset", &b)); cb->offset = (uint64_t)b.val; - WT_ERR(__wt_config_subgets(session, &v, "rename", &b)); - if (b.val) - F_SET(cb, WT_CURBACKUP_RENAME); - else - F_CLR(cb, WT_CURBACKUP_RENAME); /* * We found a match. Load the block information into the cursor. @@ -117,7 +112,7 @@ __curbackup_incr_next(WT_CURSOR *cursor) F_CLR(cursor, WT_CURSTD_RAW); if (!F_ISSET(cb, WT_CURBACKUP_INCR_INIT) && - (btree == NULL || F_ISSET(cb, WT_CURBACKUP_FORCE_FULL | WT_CURBACKUP_RENAME))) { + (btree == NULL || F_ISSET(cb, WT_CURBACKUP_FORCE_FULL))) { /* * We don't have this object's incremental information or it's a forced file copy. If this * is a log file, use the full pathname that may include the log path. @@ -160,21 +155,19 @@ __curbackup_incr_next(WT_CURSOR *cursor) */ WT_ERR(__curbackup_incr_blkmod(session, btree, cb)); /* - * There are several cases where we do not have block modification information for + * There are three cases where we do not have block modification information for * the file. They are described and handled as follows: * - * 1. Renamed file. Always return the whole file information. - * 2. Newly created file without checkpoint information. Return the whole - * file information. - * 3. File created and checkpointed before incremental backups were configured. + * 1. Newly created file without checkpoint information. Return the whole file + * information. + * 2. File created and checkpointed before incremental backups were configured. * Return no file information as it was copied in the initial full backup. - * 4. File that has not been modified since the previous incremental backup. + * 3. File that has not been modified since the previous incremental backup. * Return no file information as there is no new information. */ - if (cb->bitstring.mem == NULL || F_ISSET(cb, WT_CURBACKUP_RENAME)) { + if (cb->bitstring.mem == NULL) { F_SET(cb, WT_CURBACKUP_INCR_INIT); - if (F_ISSET(cb, WT_CURBACKUP_RENAME) || - (F_ISSET(cb, WT_CURBACKUP_CKPT_FAKE) && F_ISSET(cb, WT_CURBACKUP_HAS_CB_INFO))) { + if (F_ISSET(cb, WT_CURBACKUP_CKPT_FAKE) && F_ISSET(cb, WT_CURBACKUP_HAS_CB_INFO)) { WT_ERR(__wt_fs_size(session, cb->incr_file, &size)); __wt_cursor_set_key(cursor, 0, size, WT_BACKUP_FILE); goto done; diff --git a/src/third_party/wiredtiger/src/include/cursor.h b/src/third_party/wiredtiger/src/include/cursor.h index 3d04cfaa73e..be3672f1ac5 100644 --- a/src/third_party/wiredtiger/src/include/cursor.h +++ b/src/third_party/wiredtiger/src/include/cursor.h @@ -61,16 +61,15 @@ struct __wt_cursor_backup { uint64_t granularity; /* Length, transfer size */ /* AUTOMATIC FLAG VALUE GENERATION START */ -#define WT_CURBACKUP_CKPT_FAKE 0x001u /* Object has fake checkpoint */ -#define WT_CURBACKUP_DUP 0x002u /* Duplicated backup cursor */ -#define WT_CURBACKUP_FORCE_FULL 0x004u /* Force full file copy for this cursor */ -#define WT_CURBACKUP_FORCE_STOP 0x008u /* Force stop incremental backup */ -#define WT_CURBACKUP_HAS_CB_INFO 0x010u /* Object has checkpoint backup info */ -#define WT_CURBACKUP_INCR 0x020u /* Incremental backup cursor */ -#define WT_CURBACKUP_INCR_INIT 0x040u /* Cursor traversal initialized */ -#define WT_CURBACKUP_LOCKER 0x080u /* Hot-backup started */ -#define WT_CURBACKUP_RENAME 0x100u /* Object had a rename */ - /* AUTOMATIC FLAG VALUE GENERATION STOP */ +#define WT_CURBACKUP_CKPT_FAKE 0x01u /* Object has fake checkpoint */ +#define WT_CURBACKUP_DUP 0x02u /* Duplicated backup cursor */ +#define WT_CURBACKUP_FORCE_FULL 0x04u /* Force full file copy for this cursor */ +#define WT_CURBACKUP_FORCE_STOP 0x08u /* Force stop incremental backup */ +#define WT_CURBACKUP_HAS_CB_INFO 0x10u /* Object has checkpoint backup info */ +#define WT_CURBACKUP_INCR 0x20u /* Incremental backup cursor */ +#define WT_CURBACKUP_INCR_INIT 0x40u /* Cursor traversal initialized */ +#define WT_CURBACKUP_LOCKER 0x80u /* Hot-backup started */ + /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t flags; }; diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index 5ce0cc553eb..51d7bd996dd 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -358,8 +358,6 @@ extern int __wt_checkpoint_server_destroy(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_checkpoint_sync(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_ckpt_blkmod_to_meta(WT_SESSION_IMPL *session, WT_ITEM *buf, WT_CKPT *ckpt) - WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_close(WT_SESSION_IMPL *session, WT_FH **fhp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_close_connection_close(WT_SESSION_IMPL *session) @@ -1034,8 +1032,6 @@ extern int __wt_meta_apply_all(WT_SESSION_IMPL *session, int (*file_func)(WT_SESSION_IMPL *, const char *[]), int (*name_func)(WT_SESSION_IMPL *, const char *, bool *), const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_meta_blk_mods_load(WT_SESSION_IMPL *session, const char *config, WT_CKPT *ckpt, - bool rename) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_meta_block_metadata(WT_SESSION_IMPL *session, const char *config, WT_CKPT *ckpt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_meta_checkpoint(WT_SESSION_IMPL *session, const char *fname, const char *checkpoint, diff --git a/src/third_party/wiredtiger/src/include/meta.h b/src/third_party/wiredtiger/src/include/meta.h index a0241a22ce7..dac2cc59573 100644 --- a/src/third_party/wiredtiger/src/include/meta.h +++ b/src/third_party/wiredtiger/src/include/meta.h @@ -98,9 +98,8 @@ struct __wt_block_mods { uint64_t offset; /* Zero bit offset for bitstring */ uint64_t granularity; /* AUTOMATIC FLAG VALUE GENERATION START */ -#define WT_BLOCK_MODS_RENAME 0x1u /* Entry is from a rename */ -#define WT_BLOCK_MODS_VALID 0x2u /* Entry is valid */ - /* AUTOMATIC FLAG VALUE GENERATION STOP */ +#define WT_BLOCK_MODS_VALID 0x1u /* Entry is valid */ + /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t flags; }; diff --git a/src/third_party/wiredtiger/src/meta/meta_ckpt.c b/src/third_party/wiredtiger/src/meta/meta_ckpt.c index 65dae99e577..cb536467878 100644 --- a/src/third_party/wiredtiger/src/meta/meta_ckpt.c +++ b/src/third_party/wiredtiger/src/meta/meta_ckpt.c @@ -11,6 +11,7 @@ static int __ckpt_last(WT_SESSION_IMPL *, const char *, WT_CKPT *); static int __ckpt_last_name(WT_SESSION_IMPL *, const char *, const char **); static int __ckpt_load(WT_SESSION_IMPL *, WT_CONFIG_ITEM *, WT_CONFIG_ITEM *, WT_CKPT *); +static int __ckpt_load_blk_mods(WT_SESSION_IMPL *, const char *, WT_CKPT *); static int __ckpt_named(WT_SESSION_IMPL *, const char *, const char *, WT_CKPT *); static int __ckpt_set(WT_SESSION_IMPL *, const char *, const char *, bool); static int __ckpt_version_chk(WT_SESSION_IMPL *, const char *, const char *); @@ -72,11 +73,6 @@ __ckpt_load_blk_mods(WT_SESSION_IMPL *session, const char *config, WT_CKPT *ckpt blk_mod->nbits = (uint64_t)b.val; WT_RET(__wt_config_subgets(session, &v, "offset", &b)); blk_mod->offset = (uint64_t)b.val; - WT_RET(__wt_config_subgets(session, &v, "rename", &b)); - if (b.val) - F_SET(blk_mod, WT_BLOCK_MODS_RENAME); - else - F_CLR(blk_mod, WT_BLOCK_MODS_RENAME); ret = __wt_config_subgets(session, &v, "blocks", &b); WT_RET_NOTFOUND_OK(ret); if (ret != WT_NOTFOUND) { @@ -392,7 +388,7 @@ __ckpt_compare_order(const void *a, const void *b) * information. */ static int -__ckpt_valid_blk_mods(WT_SESSION_IMPL *session, WT_CKPT *ckpt, bool rename) +__ckpt_valid_blk_mods(WT_SESSION_IMPL *session, WT_CKPT *ckpt) { WT_BLKINCR *blk; WT_BLOCK_MODS *blk_mod; @@ -430,10 +426,6 @@ __ckpt_valid_blk_mods(WT_SESSION_IMPL *session, WT_CKPT *ckpt, bool rename) setup = true; } - /* If we are keeping or setting up an entry on a rename, set the flag. */ - if (rename && (!free || setup)) - F_SET(blk_mod, WT_BLOCK_MODS_RENAME); - /* Free any old information if we need to do so. */ if (free && F_ISSET(blk_mod, WT_BLOCK_MODS_VALID)) { __wt_free(session, blk_mod->id_str); @@ -457,32 +449,6 @@ __ckpt_valid_blk_mods(WT_SESSION_IMPL *session, WT_CKPT *ckpt, bool rename) return (0); } -/* - * __wt_meta_blk_mods_load -- - * Load the block mods for a given checkpoint and set up all the information to store. - */ -int -__wt_meta_blk_mods_load(WT_SESSION_IMPL *session, const char *config, WT_CKPT *ckpt, bool rename) -{ - /* - * Load most recent checkpoint backup blocks to this checkpoint. - */ - WT_RET(__ckpt_load_blk_mods(session, config, ckpt)); - - WT_RET(__wt_meta_block_metadata(session, config, ckpt)); - - /* - * Set the add-a-checkpoint flag, and if we're doing incremental backups, request a list of the - * checkpoint's modified blocks from the block manager. - */ - F_SET(ckpt, WT_CKPT_ADD); - if (F_ISSET(S2C(session), WT_CONN_INCR_BACKUP)) { - F_SET(ckpt, WT_CKPT_BLOCK_MODS); - WT_RET(__ckpt_valid_blk_mods(session, ckpt, rename)); - } - return (0); -} - /* * __wt_meta_ckptlist_get -- * Load all available checkpoint information for a file. @@ -557,7 +523,22 @@ __wt_meta_ckptlist_get( __wt_atomic_cas64(&conn->ckpt_most_recent, most_recent, ckpt->sec)) break; } - WT_ERR(__wt_meta_blk_mods_load(session, config, ckpt, false)); + /* + * Load most recent checkpoint backup blocks to this checkpoint. + */ + WT_ERR(__ckpt_load_blk_mods(session, config, ckpt)); + + WT_ERR(__wt_meta_block_metadata(session, config, ckpt)); + + /* + * Set the add-a-checkpoint flag, and if we're doing incremental backups, request a list of + * the checkpoint's modified blocks from the block manager. + */ + F_SET(ckpt, WT_CKPT_ADD); + if (F_ISSET(conn, WT_CONN_INCR_BACKUP)) { + F_SET(ckpt, WT_CKPT_BLOCK_MODS); + WT_ERR(__ckpt_valid_blk_mods(session, ckpt)); + } } /* Return the array to our caller. */ @@ -783,11 +764,11 @@ __wt_meta_ckptlist_to_meta(WT_SESSION_IMPL *session, WT_CKPT *ckptbase, WT_ITEM } /* - * __wt_ckpt_blkmod_to_meta -- + * __ckpt_blkmod_to_meta -- * Add in any modification block string needed, including an empty one. */ -int -__wt_ckpt_blkmod_to_meta(WT_SESSION_IMPL *session, WT_ITEM *buf, WT_CKPT *ckpt) +static int +__ckpt_blkmod_to_meta(WT_SESSION_IMPL *session, WT_ITEM *buf, WT_CKPT *ckpt) { WT_BLOCK_MODS *blk; WT_ITEM bitstring; @@ -818,9 +799,9 @@ __wt_ckpt_blkmod_to_meta(WT_SESSION_IMPL *session, WT_ITEM *buf, WT_CKPT *ckpt) WT_RET(__wt_raw_to_hex(session, blk->bitstring.data, blk->bitstring.size, &bitstring)); WT_RET(__wt_buf_catfmt(session, buf, "%s\"%s\"=(id=%" PRIu32 ",granularity=%" PRIu64 ",nbits=%" PRIu64 ",offset=%" PRIu64 - ",rename=%d,blocks=%.*s)", + ",blocks=%.*s)", i == 0 ? "" : ",", blk->id_str, i, blk->granularity, blk->nbits, blk->offset, - F_ISSET(blk, WT_BLOCK_MODS_RENAME) ? 1 : 0, (int)bitstring.size, (char *)bitstring.data)); + (int)bitstring.size, (char *)bitstring.data)); /* The hex string length should match the appropriate number of bits. */ WT_ASSERT(session, (blk->nbits >> 2) <= bitstring.size); __wt_buf_free(session, &bitstring); @@ -848,7 +829,7 @@ __wt_meta_ckptlist_set( /* Add backup block modifications for any added checkpoint. */ WT_CKPT_FOREACH (ckptbase, ckpt) if (F_ISSET(ckpt, WT_CKPT_ADD)) - WT_ERR(__wt_ckpt_blkmod_to_meta(session, buf, ckpt)); + WT_ERR(__ckpt_blkmod_to_meta(session, buf, ckpt)); has_lsn = ckptlsn != NULL; if (ckptlsn != NULL) diff --git a/src/third_party/wiredtiger/src/schema/schema_rename.c b/src/third_party/wiredtiger/src/schema/schema_rename.c index 53c006ed226..3dfc99493c4 100644 --- a/src/third_party/wiredtiger/src/schema/schema_rename.c +++ b/src/third_party/wiredtiger/src/schema/schema_rename.c @@ -8,30 +8,6 @@ #include "wt_internal.h" -/* - * __rename_blkmod -- - * Reset the incremental backup information for a rename. - */ -static int -__rename_blkmod(WT_SESSION_IMPL *session, const char *oldvalue, WT_ITEM *buf) -{ - WT_CKPT ckpt; - WT_DECL_RET; - - WT_CLEAR(ckpt); - /* - * Replace the old file entries with new file entries. We need to recreate the incremental - * backup information to indicate copying the entire file in its bitmap. - */ - /* First load any existing backup information into a temp checkpoint structure. */ - WT_RET(__wt_meta_blk_mods_load(session, oldvalue, &ckpt, true)); - - /* Take the checkpoint structure and generate the metadata string. */ - ret = __wt_ckpt_blkmod_to_meta(session, buf, &ckpt); - __wt_meta_checkpoint_free(session, &ckpt); - return (ret); -} - /* * __rename_file -- * WT_SESSION::rename for a file. @@ -39,10 +15,8 @@ __rename_blkmod(WT_SESSION_IMPL *session, const char *oldvalue, WT_ITEM *buf) static int __rename_file(WT_SESSION_IMPL *session, const char *uri, const char *newuri) { - WT_DECL_ITEM(buf); WT_DECL_RET; char *newvalue, *oldvalue; - const char *filecfg[3] = {NULL, NULL, NULL}; const char *filename, *newfile; bool exist; @@ -59,7 +33,6 @@ __rename_file(WT_SESSION_IMPL *session, const char *uri, const char *newuri) WT_WITH_HANDLE_LIST_WRITE_LOCK( session, ret = __wt_conn_dhandle_close_all(session, uri, true, false)); WT_ERR(ret); - WT_ERR(__wt_scr_alloc(session, 1024, &buf)); /* * First, check if the file being renamed exists in the system. Doing this check first matches @@ -81,20 +54,13 @@ __rename_file(WT_SESSION_IMPL *session, const char *uri, const char *newuri) default: WT_ERR(ret); } - __wt_free(session, newvalue); WT_ERR(__wt_fs_exist(session, newfile, &exist)); if (exist) WT_ERR_MSG(session, EEXIST, "%s", newfile); + /* Replace the old file entries with new file entries. */ WT_ERR(__wt_metadata_remove(session, uri)); - filecfg[0] = oldvalue; - if (F_ISSET(S2C(session), WT_CONN_INCR_BACKUP)) { - WT_ERR(__rename_blkmod(session, oldvalue, buf)); - filecfg[1] = buf->mem; - } else - filecfg[1] = NULL; - WT_ERR(__wt_config_collapse(session, filecfg, &newvalue)); - WT_ERR(__wt_metadata_insert(session, newuri, newvalue)); + WT_ERR(__wt_metadata_insert(session, newuri, oldvalue)); /* Rename the underlying file. */ WT_ERR(__wt_fs_rename(session, filename, newfile, false)); @@ -102,7 +68,6 @@ __rename_file(WT_SESSION_IMPL *session, const char *uri, const char *newuri) WT_ERR(__wt_meta_track_fileop(session, uri, newuri)); err: - __wt_scr_free(session, &buf); __wt_free(session, newvalue); __wt_free(session, oldvalue); return (ret); diff --git a/src/third_party/wiredtiger/test/csuite/incr_backup/main.c b/src/third_party/wiredtiger/test/csuite/incr_backup/main.c index 0dd403be91e..792e10275b2 100644 --- a/src/third_party/wiredtiger/test/csuite/incr_backup/main.c +++ b/src/third_party/wiredtiger/test/csuite/incr_backup/main.c @@ -59,7 +59,7 @@ static void usage(void) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static bool slow_incremental = false; static bool do_drop = true; -static bool do_rename = true; +static bool do_rename = false; #define VERBOSE(level, fmt, ...) \ do { \ -- cgit v1.2.1