diff options
author | Michael Cahill <michael.cahill@mongodb.com> | 2016-02-23 14:35:42 +1100 |
---|---|---|
committer | Michael Cahill <michael.cahill@mongodb.com> | 2016-02-23 14:35:42 +1100 |
commit | 59e0ed7c0a00edcd3449cf7f124b4e08b51fe42d (patch) | |
tree | ccae980ef1f9ba0d7e2982c212a9dcd9964a4554 | |
parent | 86e6246932285fd5217a841b80e3297f56e62f58 (diff) | |
download | mongo-59e0ed7c0a00edcd3449cf7f124b4e08b51fe42d.tar.gz |
WT-2420 Gather backup metadata in a single pass.
Previously, we did one pass to gather entries for the backup file, and a
separate pass to gather handles. Since WT-2395, those two passes could
be out-of-sync, leading to files missing from a backup.
-rw-r--r-- | src/conn/conn_dhandle.c | 60 | ||||
-rw-r--r-- | src/conn/conn_stat.c | 2 | ||||
-rw-r--r-- | src/cursor/cur_backup.c | 143 | ||||
-rw-r--r-- | src/include/extern.h | 5 | ||||
-rw-r--r-- | src/include/misc.h | 4 | ||||
-rw-r--r-- | src/lsm/lsm_tree.c | 3 | ||||
-rw-r--r-- | src/meta/meta_apply.c | 34 | ||||
-rw-r--r-- | src/txn/txn_ckpt.c | 4 |
8 files changed, 111 insertions, 144 deletions
diff --git a/src/conn/conn_dhandle.c b/src/conn/conn_dhandle.c index 3bea24be508..e3f9871878b 100644 --- a/src/conn/conn_dhandle.c +++ b/src/conn/conn_dhandle.c @@ -355,13 +355,25 @@ err: F_CLR(btree, WT_BTREE_SPECIAL_FLAGS); /* * __conn_btree_apply_internal -- - * Apply a function to the open btree handles. + * Apply a function an open data handle. */ static int __conn_btree_apply_internal(WT_SESSION_IMPL *session, WT_DATA_HANDLE *dhandle, - int (*func)(WT_SESSION_IMPL *, const char *[]), const char *cfg[]) + int (*file_func)(WT_SESSION_IMPL *, const char *[]), + int (*name_func)(WT_SESSION_IMPL *, const char *, bool *), + const char *cfg[]) { WT_DECL_RET; + bool skip; + + /* Always apply the name function, if supplied. */ + skip = false; + if (name_func != NULL) + WT_RET(name_func(session, dhandle->name, &skip)); + + /* If there is no file function, don't bother locking the handle */ + if (file_func == NULL || skip) + return (0); /* * We need to pull the handle into the session handle cache and make @@ -372,7 +384,7 @@ __conn_btree_apply_internal(WT_SESSION_IMPL *session, WT_DATA_HANDLE *dhandle, dhandle->name, dhandle->checkpoint, NULL, 0)) != 0) return (ret == EBUSY ? 0 : ret); - WT_SAVE_DHANDLE(session, ret = func(session, cfg)); + WT_SAVE_DHANDLE(session, ret = file_func(session, cfg)); if (WT_META_TRACKING(session)) WT_TRET(__wt_meta_track_handle_lock(session, false)); else @@ -385,9 +397,10 @@ __conn_btree_apply_internal(WT_SESSION_IMPL *session, WT_DATA_HANDLE *dhandle, * Apply a function to all open btree handles with the given URI. */ int -__wt_conn_btree_apply(WT_SESSION_IMPL *session, - bool apply_checkpoints, const char *uri, - int (*func)(WT_SESSION_IMPL *, const char *[]), const char *cfg[]) +__wt_conn_btree_apply(WT_SESSION_IMPL *session, const char *uri, + int (*file_func)(WT_SESSION_IMPL *, const char *[]), + int (*name_func)(WT_SESSION_IMPL *, const char *, bool *), + const char *cfg[]) { WT_CONNECTION_IMPL *conn; WT_DATA_HANDLE *dhandle; @@ -404,23 +417,26 @@ __wt_conn_btree_apply(WT_SESSION_IMPL *session, if (uri != NULL) { bucket = __wt_hash_city64(uri, strlen(uri)) % WT_HASH_ARRAY_SIZE; - TAILQ_FOREACH(dhandle, &conn->dhhash[bucket], hashq) - if (F_ISSET(dhandle, WT_DHANDLE_OPEN) && - !F_ISSET(dhandle, WT_DHANDLE_DEAD) && - strcmp(uri, dhandle->name) == 0 && - (apply_checkpoints || dhandle->checkpoint == NULL)) - WT_RET(__conn_btree_apply_internal( - session, dhandle, func, cfg)); + TAILQ_FOREACH(dhandle, &conn->dhhash[bucket], hashq) { + if (!F_ISSET(dhandle, WT_DHANDLE_OPEN) || + F_ISSET(dhandle, WT_DHANDLE_DEAD) || + dhandle->checkpoint != NULL || + strcmp(uri, dhandle->name) != 0) + continue; + WT_RET(__conn_btree_apply_internal( + session, dhandle, file_func, name_func, cfg)); + } } else { - TAILQ_FOREACH(dhandle, &conn->dhqh, q) - if (F_ISSET(dhandle, WT_DHANDLE_OPEN) && - !F_ISSET(dhandle, WT_DHANDLE_DEAD) && - (apply_checkpoints || - dhandle->checkpoint == NULL) && - WT_PREFIX_MATCH(dhandle->name, "file:") && - !WT_IS_METADATA(session, dhandle)) - WT_RET(__conn_btree_apply_internal( - session, dhandle, func, cfg)); + TAILQ_FOREACH(dhandle, &conn->dhqh, q) { + if (!F_ISSET(dhandle, WT_DHANDLE_OPEN) || + F_ISSET(dhandle, WT_DHANDLE_DEAD) || + dhandle->checkpoint != NULL || + !WT_PREFIX_MATCH(dhandle->name, "file:") || + WT_IS_METADATA(session, dhandle)) + continue; + WT_RET(__conn_btree_apply_internal( + session, dhandle, file_func, name_func, cfg)); + } } return (0); diff --git a/src/conn/conn_stat.c b/src/conn/conn_stat.c index e3493ba5eeb..50f8a8b0ca9 100644 --- a/src/conn/conn_stat.c +++ b/src/conn/conn_stat.c @@ -350,7 +350,7 @@ __statlog_log_one(WT_SESSION_IMPL *session, WT_ITEM *path, WT_ITEM *tmp) if (conn->stat_sources != NULL) { WT_WITH_HANDLE_LIST_LOCK(session, ret = __wt_conn_btree_apply( - session, false, NULL, __statlog_apply, NULL)); + session, NULL, __statlog_apply, NULL, NULL)); WT_RET(ret); } diff --git a/src/cursor/cur_backup.c b/src/cursor/cur_backup.c index 63c4ac6a4ff..b097a8c08aa 100644 --- a/src/cursor/cur_backup.c +++ b/src/cursor/cur_backup.c @@ -8,12 +8,12 @@ #include "wt_internal.h" -static int __backup_all(WT_SESSION_IMPL *, WT_CURSOR_BACKUP *); +static int __backup_all(WT_SESSION_IMPL *); static int __backup_cleanup_handles(WT_SESSION_IMPL *, WT_CURSOR_BACKUP *); static int __backup_file_create(WT_SESSION_IMPL *, WT_CURSOR_BACKUP *, bool); -static int __backup_list_all_append(WT_SESSION_IMPL *, const char *[]); static int __backup_list_append( WT_SESSION_IMPL *, WT_CURSOR_BACKUP *, const char *); +static int __backup_list_uri_append(WT_SESSION_IMPL *, const char *, bool *); static int __backup_start( WT_SESSION_IMPL *, WT_CURSOR_BACKUP *, const char *[]); static int __backup_stop(WT_SESSION_IMPL *); @@ -241,7 +241,7 @@ __backup_start( if (!target_list) { WT_ERR(__backup_log_append(session, cb, true)); - WT_ERR(__backup_all(session, cb)); + WT_ERR(__backup_all(session)); } /* Add the hot backup and standard WiredTiger files to the list. */ @@ -332,55 +332,14 @@ __backup_stop(WT_SESSION_IMPL *session) * Backup all objects in the database. */ static int -__backup_all(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb) +__backup_all(WT_SESSION_IMPL *session) { - WT_CONFIG_ITEM cval; - WT_CURSOR *cursor; WT_DECL_RET; - const char *key, *value; - - cursor = NULL; - - /* Copy all of the metadata entries to the hot backup file. */ - WT_RET(__wt_metadata_cursor(session, &cursor)); - while ((ret = cursor->next(cursor)) == 0) { - WT_ERR(cursor->get_key(cursor, &key)); - WT_ERR(cursor->get_value(cursor, &value)); - WT_ERR(__wt_fprintf(cb->bfp, "%s\n%s\n", key, value)); - - /* - * While reading the metadata file, check there are no "sources" - * or "types" which can't support hot backup. This checks for - * a data source that's non-standard, which can't be backed up, - * but is also sanity checking: if there's an entry backed by - * anything other than a file or lsm entry, we're confused. - */ - if ((ret = __wt_config_getones( - session, value, "type", &cval)) == 0 && - !WT_PREFIX_MATCH_LEN(cval.str, cval.len, "file") && - !WT_PREFIX_MATCH_LEN(cval.str, cval.len, "lsm")) - WT_ERR_MSG(session, ENOTSUP, - "hot backup is not supported for objects of " - "type %.*s", (int)cval.len, cval.str); - WT_ERR_NOTFOUND_OK(ret); - if ((ret =__wt_config_getones( - session, value, "source", &cval)) == 0 && - !WT_PREFIX_MATCH_LEN(cval.str, cval.len, "file:") && - !WT_PREFIX_MATCH_LEN(cval.str, cval.len, "lsm:")) - WT_ERR_MSG(session, ENOTSUP, - "hot backup is not supported for objects of " - "source %.*s", (int)cval.len, cval.str); - WT_ERR_NOTFOUND_OK(ret); - } - WT_ERR_NOTFOUND_OK(ret); - - WT_ERR(__wt_metadata_cursor_release(session, &cursor)); /* Build a list of the file objects that need to be copied. */ WT_WITH_HANDLE_LIST_LOCK(session, ret = - __wt_meta_btree_apply(session, __backup_list_all_append, NULL)); + __wt_meta_apply_all(session, NULL, __backup_list_uri_append, NULL)); -err: WT_TRET(__wt_metadata_cursor_release(session, &cursor)); return (ret); } @@ -430,11 +389,11 @@ __backup_uri(WT_SESSION_IMPL *session, */ if (WT_PREFIX_MATCH(uri, "log:")) { *log_only = !target_list; - WT_ERR(__wt_backup_list_uri_append(session, uri, NULL)); + WT_ERR(__backup_list_uri_append(session, uri, NULL)); } else { *log_only = false; WT_ERR(__wt_schema_worker(session, - uri, NULL, __wt_backup_list_uri_append, cfg, 0)); + uri, NULL, __backup_list_uri_append, cfg, 0)); } } WT_ERR_NOTFOUND_OK(ret); @@ -471,12 +430,12 @@ __wt_backup_file_remove(WT_SESSION_IMPL *session) } /* - * __wt_backup_list_uri_append -- + * __backup_list_uri_append -- * Append a new file name to the list, allocate space as necessary. * Called via the schema_worker function. */ -int -__wt_backup_list_uri_append( +static int +__backup_list_uri_append( WT_SESSION_IMPL *session, const char *name, bool *skip) { WT_CURSOR_BACKUP *cb; @@ -485,11 +444,31 @@ __wt_backup_list_uri_append( cb = session->bkp_cursor; WT_UNUSED(skip); + /* + * While reading the metadata file, check there are no data sources + * that can't support hot backup. This checks for a data source that's + * non-standard, which can't be backed up, but is also sanity checking: + * if there's an entry backed by anything other than a file or lsm + * entry, we're confused. + */ if (WT_PREFIX_MATCH(name, "log:")) { WT_RET(__backup_log_append(session, cb, false)); return (0); } + if (!WT_PREFIX_MATCH(name, "file:") && + !WT_PREFIX_MATCH(name, "colgroup:") && + !WT_PREFIX_MATCH(name, "index:") && + !WT_PREFIX_MATCH(name, "lsm:") && + !WT_PREFIX_MATCH(name, "table:")) + WT_RET_MSG(session, ENOTSUP, + "hot backup is not supported for objects of type %s", + name); + + /* Ignore the lookaside table. */ + if (strcmp(name, WT_LAS_URI) == 0) + return (0); + /* Add the metadata entry to the backup file. */ WT_RET(__wt_metadata_search(session, name, &value)); WT_RET(__wt_fprintf(cb->bfp, "%s\n%s\n", name, value)); @@ -503,34 +482,6 @@ __wt_backup_list_uri_append( } /* - * __backup_list_all_append -- - * Append a new file name to the list, allocate space as necessary. - * Called via the __wt_meta_btree_apply function. - */ -static int -__backup_list_all_append(WT_SESSION_IMPL *session, const char *cfg[]) -{ - WT_CURSOR_BACKUP *cb; - const char *name; - - WT_UNUSED(cfg); - - cb = session->bkp_cursor; - name = session->dhandle->name; - - /* Ignore files in the process of being bulk-loaded. */ - if (F_ISSET(S2BT(session), WT_BTREE_BULK)) - return (0); - - /* Ignore the lookaside table. */ - if (strcmp(name, WT_LAS_URI) == 0) - return (0); - - /* Add the file to the list of files to be copied. */ - return (__backup_list_append(session, cb, name)); -} - -/* * __backup_list_append -- * Append a new file name to the list, allocate space as necessary. */ @@ -541,7 +492,6 @@ __backup_list_append( WT_CURSOR_BACKUP_ENTRY *p; WT_DATA_HANDLE *old_dhandle; WT_DECL_RET; - bool need_handle; const char *name; /* Leave a NULL at the end to mark the end of the list. */ @@ -551,11 +501,26 @@ __backup_list_append( p[0].name = p[1].name = NULL; p[0].handle = p[1].handle = NULL; - need_handle = false; name = uri; + + /* + * If it's a file in the database, get a handle for the underlying + * object (this handle blocks schema level operations, for example + * WT_SESSION.drop or an LSM file discard after level merging). + * + * If the handle is busy (e.g., it is being bulk-loaded), silently skip + * it. We have a special fake checkpoint in the metadata, and recovery + * will recreate an empty file. + */ if (WT_PREFIX_MATCH(uri, "file:")) { - need_handle = true; name += strlen("file:"); + + old_dhandle = session->dhandle; + ret = __wt_session_get_btree(session, uri, NULL, NULL, 0); + p->handle = session->dhandle; + session->dhandle = old_dhandle; + if (ret != 0) + return (ret == EBUSY ? 0 : ret); } /* @@ -569,20 +534,6 @@ __backup_list_append( */ WT_RET(__wt_strdup(session, name, &p->name)); - /* - * If it's a file in the database, get a handle for the underlying - * object (this handle blocks schema level operations, for example - * WT_SESSION.drop or an LSM file discard after level merging). - */ - if (need_handle) { - old_dhandle = session->dhandle; - if ((ret = - __wt_session_get_btree(session, uri, NULL, NULL, 0)) == 0) - p->handle = session->dhandle; - session->dhandle = old_dhandle; - WT_RET(ret); - } - ++cb->list_next; return (0); } diff --git a/src/include/extern.h b/src/include/extern.h index b4018eb1191..40b6314aff7 100644 --- a/src/include/extern.h +++ b/src/include/extern.h @@ -252,7 +252,7 @@ extern int __wt_checkpoint_signal(WT_SESSION_IMPL *session, wt_off_t logsize); extern int __wt_conn_dhandle_find( WT_SESSION_IMPL *session, const char *uri, const char *checkpoint); extern int __wt_conn_btree_sync_and_close(WT_SESSION_IMPL *session, bool final, bool force); extern int __wt_conn_btree_open( WT_SESSION_IMPL *session, const char *cfg[], uint32_t flags); -extern int __wt_conn_btree_apply(WT_SESSION_IMPL *session, bool apply_checkpoints, const char *uri, int (*func)(WT_SESSION_IMPL *, const char *[]), const char *cfg[]); +extern int __wt_conn_btree_apply(WT_SESSION_IMPL *session, const char *uri, int (*file_func)(WT_SESSION_IMPL *, const char *[]), int (*name_func)(WT_SESSION_IMPL *, const char *, bool *), const char *cfg[]); extern int __wt_conn_dhandle_close_all( WT_SESSION_IMPL *session, const char *uri, bool force); extern int __wt_conn_dhandle_discard_single( WT_SESSION_IMPL *session, bool final, bool force); extern int __wt_conn_dhandle_discard(WT_SESSION_IMPL *session); @@ -276,7 +276,6 @@ extern int __wt_sweep_create(WT_SESSION_IMPL *session); extern int __wt_sweep_destroy(WT_SESSION_IMPL *session); extern int __wt_curbackup_open(WT_SESSION_IMPL *session, const char *uri, const char *cfg[], WT_CURSOR **cursorp); extern int __wt_backup_file_remove(WT_SESSION_IMPL *session); -extern int __wt_backup_list_uri_append( WT_SESSION_IMPL *session, const char *name, bool *skip); extern int __wt_curbulk_init(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk, bool bitmap, bool skip_sort_check); extern int __wt_curconfig_open(WT_SESSION_IMPL *session, const char *uri, const char *cfg[], WT_CURSOR **cursorp); extern int __wt_curds_open( WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, const char *cfg[], WT_DATA_SOURCE *dsrc, WT_CURSOR **cursorp); @@ -446,7 +445,7 @@ extern int __wt_lsm_work_bloom(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree); extern int __wt_lsm_checkpoint_chunk(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, WT_LSM_CHUNK *chunk); extern int __wt_lsm_free_chunks(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree); extern int __wt_lsm_worker_start(WT_SESSION_IMPL *session, WT_LSM_WORKER_ARGS *args); -extern int __wt_meta_btree_apply(WT_SESSION_IMPL *session, int (*func)(WT_SESSION_IMPL *, const char *[]), const char *cfg[]); +extern int __wt_meta_apply_all(WT_SESSION_IMPL *session, int (*file_func)(WT_SESSION_IMPL *, const char *[]), int (*name_func)(WT_SESSION_IMPL *, const char *, bool *), const char *cfg[]); extern int __wt_meta_checkpoint(WT_SESSION_IMPL *session, const char *fname, const char *checkpoint, WT_CKPT *ckpt); extern int __wt_meta_checkpoint_last_name( WT_SESSION_IMPL *session, const char *fname, const char **namep); extern int __wt_meta_checkpoint_clear(WT_SESSION_IMPL *session, const char *fname); diff --git a/src/include/misc.h b/src/include/misc.h index 5dadb1b1484..4d3ca758dc7 100644 --- a/src/include/misc.h +++ b/src/include/misc.h @@ -201,10 +201,6 @@ (((const char *)str)[0] == ((const char *)pfx)[0] && \ strncmp((str), (pfx), strlen(pfx)) == 0) -/* Check if a non-nul-terminated string matches a prefix. */ -#define WT_PREFIX_MATCH_LEN(str, len, pfx) \ - ((len) >= strlen(pfx) && WT_PREFIX_MATCH(str, pfx)) - /* Check if a string matches a prefix, and move past it. */ #define WT_PREFIX_SKIP(str, pfx) \ (WT_PREFIX_MATCH(str, pfx) ? ((str) += strlen(pfx), 1) : 0) diff --git a/src/lsm/lsm_tree.c b/src/lsm/lsm_tree.c index 33cfb242a1e..7c188bf3dc7 100644 --- a/src/lsm/lsm_tree.c +++ b/src/lsm/lsm_tree.c @@ -1461,8 +1461,7 @@ __wt_lsm_tree_worker(WT_SESSION_IMPL *session, continue; WT_ERR(__wt_schema_worker(session, chunk->uri, file_func, name_func, cfg, open_flags)); - if (name_func == __wt_backup_list_uri_append && - F_ISSET(chunk, WT_LSM_CHUNK_BLOOM)) + if (F_ISSET(chunk, WT_LSM_CHUNK_BLOOM)) WT_ERR(__wt_schema_worker(session, chunk->bloom_uri, file_func, name_func, cfg, open_flags)); } diff --git a/src/meta/meta_apply.c b/src/meta/meta_apply.c index 7722cd55fbd..fb483c21dd9 100644 --- a/src/meta/meta_apply.c +++ b/src/meta/meta_apply.c @@ -15,22 +15,26 @@ */ static inline int __meta_btree_apply(WT_SESSION_IMPL *session, WT_CURSOR *cursor, - int (*func)(WT_SESSION_IMPL *, const char *[]), const char *cfg[]) + int (*file_func)(WT_SESSION_IMPL *, const char *[]), + int (*name_func)(WT_SESSION_IMPL *, const char *, bool *), + const char *cfg[]) { WT_DECL_RET; const char *uri; - int cmp; + bool skip; - cursor->set_key(cursor, "file:"); - if ((ret = cursor->search_near(cursor, &cmp)) == 0 && cmp < 0) - ret = cursor->next(cursor); - for (; ret == 0; ret = cursor->next(cursor)) { + while ((ret = cursor->next(cursor)) == 0) { WT_RET(cursor->get_key(cursor, &uri)); - if (!WT_PREFIX_MATCH(uri, "file:")) - break; if (strcmp(uri, WT_METAFILE_URI) == 0) continue; + skip = false; + if (name_func != NULL) + WT_RET(name_func(session, uri, &skip)); + + if (file_func == NULL || skip || !WT_PREFIX_MATCH(uri, "file:")) + continue; + /* * We need to pull the handle into the session handle cache * and make sure it's referenced to stop other internal code @@ -40,7 +44,7 @@ __meta_btree_apply(WT_SESSION_IMPL *session, WT_CURSOR *cursor, if ((ret = __wt_session_get_btree( session, uri, NULL, NULL, 0)) != 0) return (ret == EBUSY ? 0 : ret); - WT_SAVE_DHANDLE(session, ret = func(session, cfg)); + WT_SAVE_DHANDLE(session, ret = file_func(session, cfg)); if (WT_META_TRACKING(session)) WT_TRET(__wt_meta_track_handle_lock( session, false)); @@ -54,20 +58,22 @@ __meta_btree_apply(WT_SESSION_IMPL *session, WT_CURSOR *cursor, } /* - * __wt_meta_btree_apply -- + * __wt_meta_apply_all -- * Apply a function to all files listed in the metadata, apart from the * metadata file. */ int -__wt_meta_btree_apply(WT_SESSION_IMPL *session, - int (*func)(WT_SESSION_IMPL *, const char *[]), const char *cfg[]) +__wt_meta_apply_all(WT_SESSION_IMPL *session, + int (*file_func)(WT_SESSION_IMPL *, const char *[]), + int (*name_func)(WT_SESSION_IMPL *, const char *, bool *), + const char *cfg[]) { WT_CURSOR *cursor; WT_DECL_RET; WT_RET(__wt_metadata_cursor(session, &cursor)); - WT_SAVE_DHANDLE(session, - ret = __meta_btree_apply(session, cursor, func, cfg)); + WT_SAVE_DHANDLE(session, ret = + __meta_btree_apply(session, cursor, file_func, name_func, cfg)); WT_TRET(__wt_metadata_cursor_release(session, &cursor)); return (ret); diff --git a/src/txn/txn_ckpt.c b/src/txn/txn_ckpt.c index 85102ae8cfe..4bb8ccdc6f0 100644 --- a/src/txn/txn_ckpt.c +++ b/src/txn/txn_ckpt.c @@ -155,8 +155,8 @@ __checkpoint_apply_all(WT_SESSION_IMPL *session, const char *cfg[], ckpt_closed = cval.len != 0; } WT_ERR(ckpt_closed ? - __wt_meta_btree_apply(session, op, cfg) : - __wt_conn_btree_apply(session, false, NULL, op, cfg)); + __wt_meta_apply_all(session, op, NULL, cfg) : + __wt_conn_btree_apply(session, NULL, op, NULL, cfg)); } if (fullp != NULL) |