diff options
| author | Luke Chen <luke.chen@mongodb.com> | 2021-02-22 15:03:10 +1100 |
|---|---|---|
| committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2021-02-22 04:23:20 +0000 |
| commit | 567b7b59849dc2ecd10d6899b9b31c1aba5f7bd1 (patch) | |
| tree | 2438425f413000b458c835858109c51af92e3ef8 | |
| parent | 85a8c709a0baaffcd60fe3a0c04d5f7621a805a0 (diff) | |
| download | mongo-567b7b59849dc2ecd10d6899b9b31c1aba5f7bd1.tar.gz | |
Import wiredtiger: 9d20694ad13222211f2afb3b2328163b5156fb6c from branch mongodb-4.2
ref: 7e416a7dd9..9d20694ad1
for: 4.2.13
WT-6309 Add support for start/stop arguments to wt printlog command
WT-6430 Move WT_CONN_SERVER flags into their own field
WT-6844 Make force stop durable
WT-7026 Atomically read and set ref->addr in case of race
WT-7027 Run the metadata checkpoint for force_stop at read-committed isolation for incremental backup
WT-7028 Sweep thread shouldn't lock during checkpoint gathering handles
WT-7104 Redact user data from printlog output
37 files changed, 684 insertions, 336 deletions
diff --git a/src/third_party/wiredtiger/dist/log.py b/src/third_party/wiredtiger/dist/log.py index b2c5b5d1af9..694dbb57d07 100644 --- a/src/third_party/wiredtiger/dist/log.py +++ b/src/third_party/wiredtiger/dist/log.py @@ -86,6 +86,19 @@ def printf_setup(f, i, nl_indent): def n_setup(f): return len(field_types[f[0]][4]) +# Check for an operation that has a file id type. Redact any user data +# if the redact flag is set, but print operations for file id 0, known +# to be the metadata. +def check_redact(optype): + for f in optype.fields: + if f[0] == 'uint32_id': + redact_str = '\tif (!FLD_ISSET(args->flags, WT_TXN_PRINTLOG_UNREDACT) && ' + redact_str += '%s != WT_METAFILE_ID) {\n' % (f[1]) + redact_str += '\t\tWT_RET(__wt_fprintf(session, args->fs, " REDACTED"));\n' + redact_str += '\t\treturn (0);\n\t}\n' + return redact_str + return '' + # Create a printf line, with an optional setup function. # ishex indicates that the the field name in the output is modified # (to add "-hex"), and that the setup and printf are conditional @@ -287,6 +300,7 @@ __wt_logop_%(name)s_print(WT_SESSION_IMPL *session, \t%(arg_init)sWT_RET(__wt_logop_%(name)s_unpack( \t session, pp, end%(arg_addrs)s)); +\t%(redact)s \tWT_RET(__wt_fprintf(session, args->fs, \t " \\"optype\\": \\"%(name)s\\",\\n")); %(print_args)s @@ -303,6 +317,7 @@ __wt_logop_%(name)s_print(WT_SESSION_IMPL *session, 'arg_fini' : ('\nerr:\t__wt_free(session, escaped);\n\treturn (ret);' if has_escape(optype.fields) else '\treturn (0);'), 'arg_addrs' : ''.join(', &%s' % f[1] for f in optype.fields), + 'redact' : check_redact(optype), 'print_args' : ('\t' + '\n\t'.join(printf_line(f, optype, i, s) for i,f in enumerate(optype.fields) for s in range(0, n_setup(f))) if optype.fields else ''), diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index 0a17906a297..123cf19449b 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -1017,6 +1017,7 @@ multithreaded munmap mutex mutexes +mux mytable mytxn namespace @@ -1197,6 +1198,7 @@ rwlock sH sHQ sT +sanitizer sanitizers scalability sched diff --git a/src/third_party/wiredtiger/dist/stat_data.py b/src/third_party/wiredtiger/dist/stat_data.py index 8ca4327861f..b0e9dfb6596 100644 --- a/src/third_party/wiredtiger/dist/stat_data.py +++ b/src/third_party/wiredtiger/dist/stat_data.py @@ -381,6 +381,7 @@ connection_stats = [ DhandleStat('dh_sweep_close', 'connection sweep dhandles closed'), DhandleStat('dh_sweep_ref', 'connection sweep candidate became referenced'), DhandleStat('dh_sweep_remove', 'connection sweep dhandles removed from hash list'), + DhandleStat('dh_sweep_skip_ckpt', 'connection sweeps skipped due to checkpoint gathering handles'), DhandleStat('dh_sweep_tod', 'connection sweep time-of-death sets'), DhandleStat('dh_sweeps', 'connection sweeps'), diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index b9c0cb846c3..f25a44554c1 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-4.2", - "commit": "7e416a7dd98de324bba158b087197124df70c5a9" + "commit": "9d20694ad13222211f2afb3b2328163b5156fb6c" } diff --git a/src/third_party/wiredtiger/src/async/async_api.c b/src/third_party/wiredtiger/src/async/async_api.c index 17054e0b852..6f0d42ffaef 100644 --- a/src/third_party/wiredtiger/src/async/async_api.c +++ b/src/third_party/wiredtiger/src/async/async_api.c @@ -238,7 +238,7 @@ __async_start(WT_SESSION_IMPL *session) /* * Start up the worker threads. */ - F_SET(conn, WT_CONN_SERVER_ASYNC); + FLD_SET(conn->server_flags, WT_CONN_SERVER_ASYNC); for (i = 0; i < conn->async_workers; i++) { /* * Each worker has its own session. We set both a general server flag in the connection and @@ -414,7 +414,7 @@ __wt_async_destroy(WT_SESSION_IMPL *session) if (!conn->async_cfg) return (0); - F_CLR(conn, WT_CONN_SERVER_ASYNC); + FLD_CLR(conn->server_flags, WT_CONN_SERVER_ASYNC); for (i = 0; i < conn->async_workers; i++) WT_TRET(__wt_thread_join(session, &async->worker_tids[i])); __wt_cond_destroy(session, &async->flush_cond); diff --git a/src/third_party/wiredtiger/src/async/async_worker.c b/src/third_party/wiredtiger/src/async/async_worker.c index 23098c5b8c5..4174364cc1e 100644 --- a/src/third_party/wiredtiger/src/async/async_worker.c +++ b/src/third_party/wiredtiger/src/async/async_worker.c @@ -51,7 +51,7 @@ retry: } if (!F_ISSET(session, WT_SESSION_SERVER_ASYNC)) return (0); - if (!F_ISSET(conn, WT_CONN_SERVER_ASYNC)) + if (!FLD_ISSET(conn->server_flags, WT_CONN_SERVER_ASYNC)) return (0); WT_ORDERED_READ(last_consume, async->alloc_tail); } @@ -278,7 +278,8 @@ __wt_async_worker(void *arg) worker.num_cursors = 0; TAILQ_INIT(&worker.cursorqh); - while (F_ISSET(conn, WT_CONN_SERVER_ASYNC) && F_ISSET(session, WT_SESSION_SERVER_ASYNC)) { + while (FLD_ISSET(conn->server_flags, WT_CONN_SERVER_ASYNC) && + F_ISSET(session, WT_SESSION_SERVER_ASYNC)) { WT_ERR(__async_op_dequeue(conn, session, &op)); if (op != NULL && op != &async->flush_op) { /* diff --git a/src/third_party/wiredtiger/src/btree/bt_split.c b/src/third_party/wiredtiger/src/btree/bt_split.c index 329ee184b2f..1d54a775259 100644 --- a/src/third_party/wiredtiger/src/btree/bt_split.c +++ b/src/third_party/wiredtiger/src/btree/bt_split.c @@ -1008,8 +1008,8 @@ __split_internal(WT_SESSION_IMPL *session, WT_PAGE *parent, WT_PAGE *page) * The newly allocated child's page index references the same structures as the parent. (We * cannot move WT_REF structures, threads may be underneath us right now changing the * structure state.) However, if the WT_REF structures reference on-page information, we - * have to fix that, because the disk image for the page that has an page index entry for - * the WT_REF is about to be discarded. + * have to fix that, because the disk image for the page that has a page index entry for the + * WT_REF is about to be discarded. */ child_pindex = WT_INTL_INDEX_GET_SAFE(child); child_incr = 0; diff --git a/src/third_party/wiredtiger/src/conn/conn_capacity.c b/src/third_party/wiredtiger/src/conn/conn_capacity.c index 31e5ab78478..38817d3e485 100644 --- a/src/third_party/wiredtiger/src/conn/conn_capacity.c +++ b/src/third_party/wiredtiger/src/conn/conn_capacity.c @@ -72,7 +72,7 @@ __capacity_config(WT_SESSION_IMPL *session, const char *cfg[]) static bool __capacity_server_run_chk(WT_SESSION_IMPL *session) { - return (F_ISSET(S2C(session), WT_CONN_SERVER_CAPACITY)); + return (FLD_ISSET(S2C(session)->server_flags, WT_CONN_SERVER_CAPACITY)); } /* @@ -129,7 +129,7 @@ __capacity_server_start(WT_CONNECTION_IMPL *conn) { WT_SESSION_IMPL *session; - F_SET(conn, WT_CONN_SERVER_CAPACITY); + FLD_SET(conn->server_flags, WT_CONN_SERVER_CAPACITY); /* * The capacity server gets its own session. @@ -197,7 +197,7 @@ __wt_capacity_server_destroy(WT_SESSION_IMPL *session) conn = S2C(session); - F_CLR(conn, WT_CONN_SERVER_CAPACITY); + FLD_CLR(conn->server_flags, WT_CONN_SERVER_CAPACITY); if (conn->capacity_tid_set) { __wt_cond_signal(session, conn->capacity_cond); WT_TRET(__wt_thread_join(session, &conn->capacity_tid)); diff --git a/src/third_party/wiredtiger/src/conn/conn_ckpt.c b/src/third_party/wiredtiger/src/conn/conn_ckpt.c index 7a6cd33c728..b726e6bb9f9 100644 --- a/src/third_party/wiredtiger/src/conn/conn_ckpt.c +++ b/src/third_party/wiredtiger/src/conn/conn_ckpt.c @@ -63,7 +63,7 @@ __ckpt_server_config(WT_SESSION_IMPL *session, const char **cfg, bool *startp) static bool __ckpt_server_run_chk(WT_SESSION_IMPL *session) { - return (F_ISSET(S2C(session), WT_CONN_SERVER_CHECKPOINT)); + return (FLD_ISSET(S2C(session)->server_flags, WT_CONN_SERVER_CHECKPOINT)); } /* @@ -134,7 +134,7 @@ __ckpt_server_start(WT_CONNECTION_IMPL *conn) if (conn->ckpt_session != NULL) return (0); - F_SET(conn, WT_CONN_SERVER_CHECKPOINT); + FLD_SET(conn->server_flags, WT_CONN_SERVER_CHECKPOINT); /* * The checkpoint server gets its own session. @@ -202,7 +202,7 @@ __wt_checkpoint_server_destroy(WT_SESSION_IMPL *session) conn = S2C(session); - F_CLR(conn, WT_CONN_SERVER_CHECKPOINT); + FLD_CLR(conn->server_flags, WT_CONN_SERVER_CHECKPOINT); if (conn->ckpt_tid_set) { __wt_cond_signal(session, conn->ckpt_cond); WT_TRET(__wt_thread_join(session, &conn->ckpt_tid)); diff --git a/src/third_party/wiredtiger/src/conn/conn_dhandle.c b/src/third_party/wiredtiger/src/conn/conn_dhandle.c index 53c11e32516..a756d152fa3 100644 --- a/src/third_party/wiredtiger/src/conn/conn_dhandle.c +++ b/src/third_party/wiredtiger/src/conn/conn_dhandle.c @@ -597,6 +597,7 @@ __wt_conn_btree_apply(WT_SESSION_IMPL *session, const char *uri, conn->ckpt_apply = conn->ckpt_skip = 0; conn->ckpt_apply_time = conn->ckpt_skip_time = 0; time_start = __wt_clock(session); + F_SET(conn, WT_CONN_CKPT_GATHER); } for (dhandle = NULL;;) { WT_WITH_HANDLE_LIST_READ_LOCK( @@ -612,6 +613,7 @@ __wt_conn_btree_apply(WT_SESSION_IMPL *session, const char *uri, } done: if (WT_SESSION_IS_CHECKPOINT(session)) { + F_CLR(conn, WT_CONN_CKPT_GATHER); time_stop = __wt_clock(session); time_diff = WT_CLOCKDIFF_US(time_stop, time_start); WT_STAT_CONN_SET(session, txn_checkpoint_handle_applied, conn->ckpt_apply); @@ -625,6 +627,7 @@ done: } err: + F_CLR(conn, WT_CONN_CKPT_GATHER); WT_DHANDLE_RELEASE(dhandle); return (ret); } diff --git a/src/third_party/wiredtiger/src/conn/conn_log.c b/src/third_party/wiredtiger/src/conn/conn_log.c index 845d1ab5a7b..0ef424f7e2d 100644 --- a/src/third_party/wiredtiger/src/conn/conn_log.c +++ b/src/third_party/wiredtiger/src/conn/conn_log.c @@ -486,7 +486,7 @@ __wt_log_truncate_files(WT_SESSION_IMPL *session, WT_CURSOR *cursor, bool force) conn = S2C(session); if (!FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED)) return (0); - if (!force && F_ISSET(conn, WT_CONN_SERVER_LOG) && + if (!force && FLD_ISSET(conn->server_flags, WT_CONN_SERVER_LOG) && FLD_ISSET(conn->log_flags, WT_CONN_LOG_ARCHIVE)) WT_RET_MSG(session, EINVAL, "Attempt to archive manually while a server is running"); @@ -530,7 +530,7 @@ __log_file_server(void *arg) log = conn->log; locked = false; yield_count = 0; - while (F_ISSET(conn, WT_CONN_SERVER_LOG)) { + while (FLD_ISSET(conn->server_flags, WT_CONN_SERVER_LOG)) { /* * If there is a log file to close, make sure any outstanding write operations have * completed, then fsync and close it. @@ -802,7 +802,7 @@ __log_wrlsn_server(void *arg) log = conn->log; yield = 0; WT_INIT_LSN(&prev); - while (F_ISSET(conn, WT_CONN_SERVER_LOG)) { + while (FLD_ISSET(conn->server_flags, WT_CONN_SERVER_LOG)) { /* * Write out any log record buffers if anything was done since last time. Only call the * function to walk the slots if the system is not idle. On an idle system the alloc_lsn @@ -872,7 +872,7 @@ __log_server(void *arg) * records sitting in the buffer over the time it takes to sync out an earlier file. */ did_work = true; - while (F_ISSET(conn, WT_CONN_SERVER_LOG)) { + while (FLD_ISSET(conn->server_flags, WT_CONN_SERVER_LOG)) { /* * Slots depend on future activity. Force out buffered writes in case we are idle. This * cannot be part of the wrlsn thread because of interaction advancing the write_lsn and a @@ -1001,7 +1001,7 @@ __wt_logmgr_open(WT_SESSION_IMPL *session) if (!FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED)) return (0); - F_SET(conn, WT_CONN_SERVER_LOG); + FLD_SET(conn->server_flags, WT_CONN_SERVER_LOG); /* * Start the log close thread. It is not configurable. If logging is enabled, this thread runs. @@ -1070,7 +1070,7 @@ __wt_logmgr_destroy(WT_SESSION_IMPL *session) conn = S2C(session); - F_CLR(conn, WT_CONN_SERVER_LOG); + FLD_CLR(conn->server_flags, WT_CONN_SERVER_LOG); if (!FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED)) { /* diff --git a/src/third_party/wiredtiger/src/conn/conn_open.c b/src/third_party/wiredtiger/src/conn/conn_open.c index 2f6632ddf8a..b7a98811683 100644 --- a/src/third_party/wiredtiger/src/conn/conn_open.c +++ b/src/third_party/wiredtiger/src/conn/conn_open.c @@ -78,7 +78,7 @@ __wt_connection_close(WT_CONNECTION_IMPL *conn) * wiredtiger_open hits an error (as well as during normal shutdown). Assert they're not * running. */ - WT_ASSERT(session, !F_ISSET(conn, WT_CONN_SERVER_ASYNC | WT_CONN_SERVER_LSM)); + WT_ASSERT(session, !FLD_ISSET(conn->server_flags, WT_CONN_SERVER_ASYNC | WT_CONN_SERVER_LSM)); /* Shut down the subsystems, ensuring workers see the state change. */ F_SET(conn, WT_CONN_CLOSING); diff --git a/src/third_party/wiredtiger/src/conn/conn_stat.c b/src/third_party/wiredtiger/src/conn/conn_stat.c index 4d2704c7f6a..f60f5cde851 100644 --- a/src/third_party/wiredtiger/src/conn/conn_stat.c +++ b/src/third_party/wiredtiger/src/conn/conn_stat.c @@ -532,7 +532,7 @@ __statlog_on_close(WT_SESSION_IMPL *session) if (!FLD_ISSET(conn->stat_flags, WT_STAT_ON_CLOSE)) return (0); - if (F_ISSET(conn, WT_CONN_SERVER_STATISTICS)) + if (FLD_ISSET(conn->server_flags, WT_CONN_SERVER_STATISTICS)) WT_RET_MSG(session, EINVAL, "Attempt to log statistics while a server is running"); WT_RET(__wt_scr_alloc(session, strlen(conn->stat_path) + 128, &tmp)); @@ -551,7 +551,7 @@ err: static bool __statlog_server_run_chk(WT_SESSION_IMPL *session) { - return (F_ISSET(S2C(session), WT_CONN_SERVER_STATISTICS)); + return (FLD_ISSET(S2C(session)->server_flags, WT_CONN_SERVER_STATISTICS)); } /* @@ -618,7 +618,7 @@ __statlog_start(WT_CONNECTION_IMPL *conn) if (conn->stat_session != NULL) return (0); - F_SET(conn, WT_CONN_SERVER_STATISTICS); + FLD_SET(conn->server_flags, WT_CONN_SERVER_STATISTICS); /* The statistics log server gets its own session. */ WT_RET(__wt_open_internal_session(conn, "statlog-server", true, 0, &conn->stat_session)); @@ -690,7 +690,7 @@ __wt_statlog_destroy(WT_SESSION_IMPL *session, bool is_close) conn = S2C(session); /* Stop the server thread. */ - F_CLR(conn, WT_CONN_SERVER_STATISTICS); + FLD_CLR(conn->server_flags, WT_CONN_SERVER_STATISTICS); if (conn->stat_tid_set) { __wt_cond_signal(session, conn->stat_cond); WT_TRET(__wt_thread_join(session, &conn->stat_tid)); diff --git a/src/third_party/wiredtiger/src/conn/conn_sweep.c b/src/third_party/wiredtiger/src/conn/conn_sweep.c index 114a467629b..3aac8680993 100644 --- a/src/third_party/wiredtiger/src/conn/conn_sweep.c +++ b/src/third_party/wiredtiger/src/conn/conn_sweep.c @@ -252,7 +252,7 @@ __sweep_remove_handles(WT_SESSION_IMPL *session) static bool __sweep_server_run_chk(WT_SESSION_IMPL *session) { - return (F_ISSET(S2C(session), WT_CONN_SERVER_SWEEP)); + return (FLD_ISSET(S2C(session)->server_flags, WT_CONN_SERVER_SWEEP)); } /* @@ -319,10 +319,18 @@ __sweep_server(void *arg) /* * See if it is time to sweep the data handles. Those are swept less frequently than the - * lookaside table by default and the frequency is controlled by a user setting. + * history store table by default and the frequency is controlled by a user setting. We want + * to avoid sweeping while checkpoint is gathering handles. Both need to lock the dhandle + * list and sweep acquiring that lock can interfere with checkpoint and cause it to take + * longer. Sweep is an operation that typically has long intervals so skipping some for + * checkpoint should have little impact. */ if (!cv_signalled && (now - last < sweep_interval)) continue; + if (F_ISSET(conn, WT_CONN_CKPT_GATHER)) { + WT_STAT_CONN_INCR(session, dh_sweep_skip_ckpt); + continue; + } WT_STAT_CONN_INCR(session, dh_sweeps); /* * Mark handles with a time of death, and report whether any handles are marked dead. If @@ -399,7 +407,7 @@ __wt_sweep_create(WT_SESSION_IMPL *session) conn = S2C(session); /* Set first, the thread might run before we finish up. */ - F_SET(conn, WT_CONN_SERVER_SWEEP); + FLD_SET(conn->server_flags, WT_CONN_SERVER_SWEEP); /* * Handle sweep does enough I/O it may be called upon to perform slow operations for the block @@ -438,7 +446,7 @@ __wt_sweep_destroy(WT_SESSION_IMPL *session) conn = S2C(session); - F_CLR(conn, WT_CONN_SERVER_SWEEP); + FLD_CLR(conn->server_flags, WT_CONN_SERVER_SWEEP); if (conn->sweep_tid_set) { __wt_cond_signal(session, conn->sweep_cond); WT_TRET(__wt_thread_join(session, &conn->sweep_tid)); diff --git a/src/third_party/wiredtiger/src/cursor/cur_backup.c b/src/third_party/wiredtiger/src/cursor/cur_backup.c index 2fe3b7dfc4a..04d4ab8a18e 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_backup.c +++ b/src/third_party/wiredtiger/src/cursor/cur_backup.c @@ -194,6 +194,7 @@ __curbackup_close(WT_CURSOR *cursor) WT_CURSOR_BACKUP *cb; WT_DECL_RET; WT_SESSION_IMPL *session; + const char *cfg[3] = {NULL, NULL, NULL}; cb = (WT_CURSOR_BACKUP *)cursor; CURSOR_API_CALL_PREPARE_ALLOWED(cursor, session, close, NULL); @@ -203,6 +204,20 @@ err: __wt_verbose( session, WT_VERB_BACKUP, "%s", "Releasing resources from forced stop incremental"); __wt_backup_destroy(session); + /* + * We need to force a checkpoint to the metadata to make the force stop durable. Without it, + * the backup information could reappear if we crash and restart. + */ + cfg[0] = WT_CONFIG_BASE(session, WT_SESSION_checkpoint); + cfg[1] = "force=true"; + /* + * Metadata checkpoints rely on read-committed isolation. Use that here no matter what + * isolation the caller's session sets for isolation. + */ + WT_WITH_DHANDLE(session, WT_SESSION_META_DHANDLE(session), + WT_WITH_METADATA_LOCK(session, + WT_WITH_TXN_ISOLATION( + session, WT_ISO_READ_COMMITTED, ret = __wt_checkpoint(session, cfg)))); } /* diff --git a/src/third_party/wiredtiger/src/cursor/cur_log.c b/src/third_party/wiredtiger/src/cursor/cur_log.c index 5d7d82805ba..ced955e0c22 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_log.c +++ b/src/third_party/wiredtiger/src/cursor/cur_log.c @@ -208,7 +208,7 @@ __curlog_next(WT_CURSOR *cursor) */ if (cl->stepp == NULL || cl->stepp >= cl->stepp_end || !*cl->stepp) { cl->txnid = 0; - ret = __wt_log_scan(session, cl->next_lsn, WT_LOGSCAN_ONE, __curlog_logrec, cl); + ret = __wt_log_scan(session, cl->next_lsn, NULL, WT_LOGSCAN_ONE, __curlog_logrec, cl); if (ret == ENOENT) ret = WT_NOTFOUND; WT_ERR(ret); @@ -247,7 +247,7 @@ __curlog_search(WT_CURSOR *cursor) */ WT_ERR(__wt_cursor_get_key(cursor, &key_file, &key_offset, &counter)); WT_SET_LSN(&key, key_file, key_offset); - ret = __wt_log_scan(session, &key, WT_LOGSCAN_ONE, __curlog_logrec, cl); + ret = __wt_log_scan(session, &key, NULL, WT_LOGSCAN_ONE, __curlog_logrec, cl); if (ret == ENOENT) ret = WT_NOTFOUND; WT_ERR(ret); diff --git a/src/third_party/wiredtiger/src/include/btree.i b/src/third_party/wiredtiger/src/include/btree.i index bb09570dc70..af0c53e341a 100644 --- a/src/third_party/wiredtiger/src/include/btree.i +++ b/src/third_party/wiredtiger/src/include/btree.i @@ -639,14 +639,23 @@ __wt_off_page(WT_PAGE *page, const void *p) static inline void __wt_ref_addr_free(WT_SESSION_IMPL *session, WT_REF *ref) { - if (ref->addr == NULL) - return; + void *ref_addr; - if (ref->home == NULL || __wt_off_page(ref->home, ref->addr)) { - __wt_free(session, ((WT_ADDR *)ref->addr)->addr); - __wt_free(session, ref->addr); + /* + * The page being discarded may be the child of a page being split, where the WT_REF.addr field + * is being instantiated (as it can no longer reference the on-disk image). Loop until we read + * and clear the address without a race, then free the read address as necessary. + */ + do { + WT_ORDERED_READ(ref_addr, ref->addr); + if (ref_addr == NULL) + return; + } while (!__wt_atomic_cas_ptr(&ref->addr, ref_addr, NULL)); + + if (ref->home == NULL || __wt_off_page(ref->home, ref_addr)) { + __wt_free(session, ((WT_ADDR *)ref_addr)->addr); + __wt_free(session, ref_addr); } - ref->addr = NULL; } /* diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h index f1f9f52506d..1e49fc32170 100644 --- a/src/third_party/wiredtiger/src/include/connection.h +++ b/src/third_party/wiredtiger/src/include/connection.h @@ -515,39 +515,47 @@ struct __wt_connection_impl { */ WT_FILE_SYSTEM *file_system; +/* + * Server subsystem flags. + */ +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_CONN_SERVER_ASYNC 0x01u +#define WT_CONN_SERVER_CAPACITY 0x02u +#define WT_CONN_SERVER_CHECKPOINT 0x04u +#define WT_CONN_SERVER_LOG 0x08u +#define WT_CONN_SERVER_LSM 0x10u +#define WT_CONN_SERVER_STATISTICS 0x20u +#define WT_CONN_SERVER_SWEEP 0x40u + /* AUTOMATIC FLAG VALUE GENERATION STOP */ + uint32_t server_flags; + /* AUTOMATIC FLAG VALUE GENERATION START */ -#define WT_CONN_CACHE_CURSORS 0x00000001u -#define WT_CONN_CACHE_POOL 0x00000002u -#define WT_CONN_CKPT_SYNC 0x00000004u -#define WT_CONN_CLOSING 0x00000008u -#define WT_CONN_CLOSING_NO_MORE_OPENS 0x00000010u -#define WT_CONN_CLOSING_TIMESTAMP 0x00000020u -#define WT_CONN_COMPATIBILITY 0x00000040u -#define WT_CONN_DATA_CORRUPTION 0x00000080u -#define WT_CONN_DEBUG_CURSOR_COPY 0x00000100u -#define WT_CONN_DEBUG_REALLOC_EXACT 0x00000200u -#define WT_CONN_DEBUG_SLOW_CKPT 0x00000400u -#define WT_CONN_EVICTION_NO_LOOKASIDE 0x00000800u -#define WT_CONN_EVICTION_RUN 0x00001000u -#define WT_CONN_INCR_BACKUP 0x00002000u -#define WT_CONN_IN_MEMORY 0x00004000u -#define WT_CONN_LEAK_MEMORY 0x00008000u -#define WT_CONN_LOOKASIDE_OPEN 0x00010000u -#define WT_CONN_LSM_MERGE 0x00020000u -#define WT_CONN_OPTRACK 0x00040000u -#define WT_CONN_PANIC 0x00080000u -#define WT_CONN_READONLY 0x00100000u -#define WT_CONN_RECONFIGURING 0x00200000u -#define WT_CONN_RECOVERING 0x00400000u -#define WT_CONN_SALVAGE 0x00800000u -#define WT_CONN_SERVER_ASYNC 0x01000000u -#define WT_CONN_SERVER_CAPACITY 0x02000000u -#define WT_CONN_SERVER_CHECKPOINT 0x04000000u -#define WT_CONN_SERVER_LOG 0x08000000u -#define WT_CONN_SERVER_LSM 0x10000000u -#define WT_CONN_SERVER_STATISTICS 0x20000000u -#define WT_CONN_SERVER_SWEEP 0x40000000u -#define WT_CONN_WAS_BACKUP 0x80000000u +#define WT_CONN_CACHE_CURSORS 0x0000001u +#define WT_CONN_CACHE_POOL 0x0000002u +#define WT_CONN_CKPT_GATHER 0x0000004u +#define WT_CONN_CKPT_SYNC 0x0000008u +#define WT_CONN_CLOSING 0x0000010u +#define WT_CONN_CLOSING_NO_MORE_OPENS 0x0000020u +#define WT_CONN_CLOSING_TIMESTAMP 0x0000040u +#define WT_CONN_COMPATIBILITY 0x0000080u +#define WT_CONN_DATA_CORRUPTION 0x0000100u +#define WT_CONN_DEBUG_CURSOR_COPY 0x0000200u +#define WT_CONN_DEBUG_REALLOC_EXACT 0x0000400u +#define WT_CONN_DEBUG_SLOW_CKPT 0x0000800u +#define WT_CONN_EVICTION_NO_LOOKASIDE 0x0001000u +#define WT_CONN_EVICTION_RUN 0x0002000u +#define WT_CONN_INCR_BACKUP 0x0004000u +#define WT_CONN_IN_MEMORY 0x0008000u +#define WT_CONN_LEAK_MEMORY 0x0010000u +#define WT_CONN_LOOKASIDE_OPEN 0x0020000u +#define WT_CONN_LSM_MERGE 0x0040000u +#define WT_CONN_OPTRACK 0x0080000u +#define WT_CONN_PANIC 0x0100000u +#define WT_CONN_READONLY 0x0200000u +#define WT_CONN_RECONFIGURING 0x0400000u +#define WT_CONN_RECOVERING 0x0800000u +#define WT_CONN_SALVAGE 0x1000000u +#define WT_CONN_WAS_BACKUP 0x2000000u /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t flags; }; diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index da6d2a17677..abd6bedf95f 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -822,7 +822,8 @@ extern int __wt_log_remove(WT_SESSION_IMPL *session, const char *file_prefix, ui WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_log_reset(WT_SESSION_IMPL *session, uint32_t lognum) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_log_scan(WT_SESSION_IMPL *session, WT_LSN *lsnp, uint32_t flags, +extern int __wt_log_scan(WT_SESSION_IMPL *session, WT_LSN *start_lsnp, WT_LSN *end_lsnp, + uint32_t flags, int (*func)(WT_SESSION_IMPL *session, WT_ITEM *record, WT_LSN *lsnp, WT_LSN *next_lsnp, void *cookie, int firstrecord), void *cookie) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -1465,9 +1466,9 @@ extern int __wt_txn_parse_timestamp_raw(WT_SESSION_IMPL *session, const char *na wt_timestamp_t *timestamp, WT_CONFIG_ITEM *cval) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_txn_prepare(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_txn_printlog(WT_SESSION *wt_session, const char *ofile, uint32_t flags) - WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) - WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_txn_printlog(WT_SESSION *wt_session, const char *ofile, uint32_t flags, + WT_LSN *start_lsn, WT_LSN *end_lsn) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_txn_query_timestamp(WT_SESSION_IMPL *session, char *hex_timestamp, const char *cfg[], bool global_txn) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_txn_reconfigure(WT_SESSION_IMPL *session, const char *config) diff --git a/src/third_party/wiredtiger/src/include/log.h b/src/third_party/wiredtiger/src/include/log.h index cf3f356f1a7..f726cd2d72a 100644 --- a/src/third_party/wiredtiger/src/include/log.h +++ b/src/third_party/wiredtiger/src/include/log.h @@ -50,8 +50,11 @@ union __wt_lsn { #define WT_LOG_ALIGN 128 /* - * Atomically set the two components of the LSN. + * Atomically set the LSN. There are two forms. We need WT_ASSIGN_LSN because some compilers (at + * least clang address sanitizer) does not do atomic 64-bit structure assignment so we need to + * explicitly assign the 64-bit field. And WT_SET_LSN atomically sets the LSN given a file/offset. */ +#define WT_ASSIGN_LSN(dstl, srcl) (dstl)->file_offset = (srcl)->file_offset #define WT_SET_LSN(l, f, o) (l)->file_offset = (((uint64_t)(f) << 32) + (o)) #define WT_INIT_LSN(l) WT_SET_LSN((l), 1, 0) @@ -395,8 +398,10 @@ struct __wt_txn_printlog_args { WT_FSTREAM *fs; /* AUTOMATIC FLAG VALUE GENERATION START */ -#define WT_TXN_PRINTLOG_HEX 0x1u /* Add hex output */ - /* AUTOMATIC FLAG VALUE GENERATION STOP */ +#define WT_TXN_PRINTLOG_HEX 0x1u /* Add hex output */ +#define WT_TXN_PRINTLOG_MSG 0x2u /* Messages only */ +#define WT_TXN_PRINTLOG_UNREDACT 0x4u /* Don't redact user data from output */ + /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t flags; }; diff --git a/src/third_party/wiredtiger/src/include/stat.h b/src/third_party/wiredtiger/src/include/stat.h index 56f4125c1f8..aed97004726 100644 --- a/src/third_party/wiredtiger/src/include/stat.h +++ b/src/third_party/wiredtiger/src/include/stat.h @@ -505,6 +505,7 @@ struct __wt_connection_stats { int64_t dh_sweep_remove; int64_t dh_sweep_tod; int64_t dh_sweeps; + int64_t dh_sweep_skip_ckpt; int64_t dh_session_handles; int64_t dh_session_sweeps; int64_t lock_checkpoint_count; diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index 61cbdf9e9e9..879198b01da 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -5390,471 +5390,476 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_CONN_DH_SWEEP_TOD 1208 /*! data-handle: connection sweeps */ #define WT_STAT_CONN_DH_SWEEPS 1209 +/*! + * data-handle: connection sweeps skipped due to checkpoint gathering + * handles + */ +#define WT_STAT_CONN_DH_SWEEP_SKIP_CKPT 1210 /*! data-handle: session dhandles swept */ -#define WT_STAT_CONN_DH_SESSION_HANDLES 1210 +#define WT_STAT_CONN_DH_SESSION_HANDLES 1211 /*! data-handle: session sweep attempts */ -#define WT_STAT_CONN_DH_SESSION_SWEEPS 1211 +#define WT_STAT_CONN_DH_SESSION_SWEEPS 1212 /*! lock: checkpoint lock acquisitions */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_COUNT 1212 +#define WT_STAT_CONN_LOCK_CHECKPOINT_COUNT 1213 /*! lock: checkpoint lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_APPLICATION 1213 +#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_APPLICATION 1214 /*! lock: checkpoint lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_INTERNAL 1214 +#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_INTERNAL 1215 /*! lock: dhandle lock application thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_APPLICATION 1215 +#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_APPLICATION 1216 /*! lock: dhandle lock internal thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_INTERNAL 1216 +#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_INTERNAL 1217 /*! lock: dhandle read lock acquisitions */ -#define WT_STAT_CONN_LOCK_DHANDLE_READ_COUNT 1217 +#define WT_STAT_CONN_LOCK_DHANDLE_READ_COUNT 1218 /*! lock: dhandle write lock acquisitions */ -#define WT_STAT_CONN_LOCK_DHANDLE_WRITE_COUNT 1218 +#define WT_STAT_CONN_LOCK_DHANDLE_WRITE_COUNT 1219 /*! * lock: durable timestamp queue lock application thread time waiting * (usecs) */ -#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WAIT_APPLICATION 1219 +#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WAIT_APPLICATION 1220 /*! * lock: durable timestamp queue lock internal thread time waiting * (usecs) */ -#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WAIT_INTERNAL 1220 +#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WAIT_INTERNAL 1221 /*! lock: durable timestamp queue read lock acquisitions */ -#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_READ_COUNT 1221 +#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_READ_COUNT 1222 /*! lock: durable timestamp queue write lock acquisitions */ -#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WRITE_COUNT 1222 +#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WRITE_COUNT 1223 /*! lock: metadata lock acquisitions */ -#define WT_STAT_CONN_LOCK_METADATA_COUNT 1223 +#define WT_STAT_CONN_LOCK_METADATA_COUNT 1224 /*! lock: metadata lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_METADATA_WAIT_APPLICATION 1224 +#define WT_STAT_CONN_LOCK_METADATA_WAIT_APPLICATION 1225 /*! lock: metadata lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_METADATA_WAIT_INTERNAL 1225 +#define WT_STAT_CONN_LOCK_METADATA_WAIT_INTERNAL 1226 /*! * lock: read timestamp queue lock application thread time waiting * (usecs) */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_APPLICATION 1226 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_APPLICATION 1227 /*! lock: read timestamp queue lock internal thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_INTERNAL 1227 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_INTERNAL 1228 /*! lock: read timestamp queue read lock acquisitions */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_READ_COUNT 1228 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_READ_COUNT 1229 /*! lock: read timestamp queue write lock acquisitions */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WRITE_COUNT 1229 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WRITE_COUNT 1230 /*! lock: schema lock acquisitions */ -#define WT_STAT_CONN_LOCK_SCHEMA_COUNT 1230 +#define WT_STAT_CONN_LOCK_SCHEMA_COUNT 1231 /*! lock: schema lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_APPLICATION 1231 +#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_APPLICATION 1232 /*! lock: schema lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_INTERNAL 1232 +#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_INTERNAL 1233 /*! * lock: table lock application thread time waiting for the table lock * (usecs) */ -#define WT_STAT_CONN_LOCK_TABLE_WAIT_APPLICATION 1233 +#define WT_STAT_CONN_LOCK_TABLE_WAIT_APPLICATION 1234 /*! * lock: table lock internal thread time waiting for the table lock * (usecs) */ -#define WT_STAT_CONN_LOCK_TABLE_WAIT_INTERNAL 1234 +#define WT_STAT_CONN_LOCK_TABLE_WAIT_INTERNAL 1235 /*! lock: table read lock acquisitions */ -#define WT_STAT_CONN_LOCK_TABLE_READ_COUNT 1235 +#define WT_STAT_CONN_LOCK_TABLE_READ_COUNT 1236 /*! lock: table write lock acquisitions */ -#define WT_STAT_CONN_LOCK_TABLE_WRITE_COUNT 1236 +#define WT_STAT_CONN_LOCK_TABLE_WRITE_COUNT 1237 /*! lock: txn global lock application thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_APPLICATION 1237 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_APPLICATION 1238 /*! lock: txn global lock internal thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_INTERNAL 1238 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_INTERNAL 1239 /*! lock: txn global read lock acquisitions */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_READ_COUNT 1239 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_READ_COUNT 1240 /*! lock: txn global write lock acquisitions */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WRITE_COUNT 1240 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WRITE_COUNT 1241 /*! log: busy returns attempting to switch slots */ -#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1241 +#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1242 /*! log: force archive time sleeping (usecs) */ -#define WT_STAT_CONN_LOG_FORCE_ARCHIVE_SLEEP 1242 +#define WT_STAT_CONN_LOG_FORCE_ARCHIVE_SLEEP 1243 /*! log: log bytes of payload data */ -#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1243 +#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1244 /*! log: log bytes written */ -#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1244 +#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1245 /*! log: log files manually zero-filled */ -#define WT_STAT_CONN_LOG_ZERO_FILLS 1245 +#define WT_STAT_CONN_LOG_ZERO_FILLS 1246 /*! log: log flush operations */ -#define WT_STAT_CONN_LOG_FLUSH 1246 +#define WT_STAT_CONN_LOG_FLUSH 1247 /*! log: log force write operations */ -#define WT_STAT_CONN_LOG_FORCE_WRITE 1247 +#define WT_STAT_CONN_LOG_FORCE_WRITE 1248 /*! log: log force write operations skipped */ -#define WT_STAT_CONN_LOG_FORCE_WRITE_SKIP 1248 +#define WT_STAT_CONN_LOG_FORCE_WRITE_SKIP 1249 /*! log: log records compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1249 +#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1250 /*! log: log records not compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1250 +#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1251 /*! log: log records too small to compress */ -#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1251 +#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1252 /*! log: log release advances write LSN */ -#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1252 +#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1253 /*! log: log scan operations */ -#define WT_STAT_CONN_LOG_SCANS 1253 +#define WT_STAT_CONN_LOG_SCANS 1254 /*! log: log scan records requiring two reads */ -#define WT_STAT_CONN_LOG_SCAN_REREADS 1254 +#define WT_STAT_CONN_LOG_SCAN_REREADS 1255 /*! log: log server thread advances write LSN */ -#define WT_STAT_CONN_LOG_WRITE_LSN 1255 +#define WT_STAT_CONN_LOG_WRITE_LSN 1256 /*! log: log server thread write LSN walk skipped */ -#define WT_STAT_CONN_LOG_WRITE_LSN_SKIP 1256 +#define WT_STAT_CONN_LOG_WRITE_LSN_SKIP 1257 /*! log: log sync operations */ -#define WT_STAT_CONN_LOG_SYNC 1257 +#define WT_STAT_CONN_LOG_SYNC 1258 /*! log: log sync time duration (usecs) */ -#define WT_STAT_CONN_LOG_SYNC_DURATION 1258 +#define WT_STAT_CONN_LOG_SYNC_DURATION 1259 /*! log: log sync_dir operations */ -#define WT_STAT_CONN_LOG_SYNC_DIR 1259 +#define WT_STAT_CONN_LOG_SYNC_DIR 1260 /*! log: log sync_dir time duration (usecs) */ -#define WT_STAT_CONN_LOG_SYNC_DIR_DURATION 1260 +#define WT_STAT_CONN_LOG_SYNC_DIR_DURATION 1261 /*! log: log write operations */ -#define WT_STAT_CONN_LOG_WRITES 1261 +#define WT_STAT_CONN_LOG_WRITES 1262 /*! log: logging bytes consolidated */ -#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1262 +#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1263 /*! log: maximum log file size */ -#define WT_STAT_CONN_LOG_MAX_FILESIZE 1263 +#define WT_STAT_CONN_LOG_MAX_FILESIZE 1264 /*! log: number of pre-allocated log files to create */ -#define WT_STAT_CONN_LOG_PREALLOC_MAX 1264 +#define WT_STAT_CONN_LOG_PREALLOC_MAX 1265 /*! log: pre-allocated log files not ready and missed */ -#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1265 +#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1266 /*! log: pre-allocated log files prepared */ -#define WT_STAT_CONN_LOG_PREALLOC_FILES 1266 +#define WT_STAT_CONN_LOG_PREALLOC_FILES 1267 /*! log: pre-allocated log files used */ -#define WT_STAT_CONN_LOG_PREALLOC_USED 1267 +#define WT_STAT_CONN_LOG_PREALLOC_USED 1268 /*! log: records processed by log scan */ -#define WT_STAT_CONN_LOG_SCAN_RECORDS 1268 +#define WT_STAT_CONN_LOG_SCAN_RECORDS 1269 /*! log: slot close lost race */ -#define WT_STAT_CONN_LOG_SLOT_CLOSE_RACE 1269 +#define WT_STAT_CONN_LOG_SLOT_CLOSE_RACE 1270 /*! log: slot close unbuffered waits */ -#define WT_STAT_CONN_LOG_SLOT_CLOSE_UNBUF 1270 +#define WT_STAT_CONN_LOG_SLOT_CLOSE_UNBUF 1271 /*! log: slot closures */ -#define WT_STAT_CONN_LOG_SLOT_CLOSES 1271 +#define WT_STAT_CONN_LOG_SLOT_CLOSES 1272 /*! log: slot join atomic update races */ -#define WT_STAT_CONN_LOG_SLOT_RACES 1272 +#define WT_STAT_CONN_LOG_SLOT_RACES 1273 /*! log: slot join calls atomic updates raced */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_RACE 1273 +#define WT_STAT_CONN_LOG_SLOT_YIELD_RACE 1274 /*! log: slot join calls did not yield */ -#define WT_STAT_CONN_LOG_SLOT_IMMEDIATE 1274 +#define WT_STAT_CONN_LOG_SLOT_IMMEDIATE 1275 /*! log: slot join calls found active slot closed */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_CLOSE 1275 +#define WT_STAT_CONN_LOG_SLOT_YIELD_CLOSE 1276 /*! log: slot join calls slept */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_SLEEP 1276 +#define WT_STAT_CONN_LOG_SLOT_YIELD_SLEEP 1277 /*! log: slot join calls yielded */ -#define WT_STAT_CONN_LOG_SLOT_YIELD 1277 +#define WT_STAT_CONN_LOG_SLOT_YIELD 1278 /*! log: slot join found active slot closed */ -#define WT_STAT_CONN_LOG_SLOT_ACTIVE_CLOSED 1278 +#define WT_STAT_CONN_LOG_SLOT_ACTIVE_CLOSED 1279 /*! log: slot joins yield time (usecs) */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_DURATION 1279 +#define WT_STAT_CONN_LOG_SLOT_YIELD_DURATION 1280 /*! log: slot transitions unable to find free slot */ -#define WT_STAT_CONN_LOG_SLOT_NO_FREE_SLOTS 1280 +#define WT_STAT_CONN_LOG_SLOT_NO_FREE_SLOTS 1281 /*! log: slot unbuffered writes */ -#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1281 +#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1282 /*! log: total in-memory size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_MEM 1282 +#define WT_STAT_CONN_LOG_COMPRESS_MEM 1283 /*! log: total log buffer size */ -#define WT_STAT_CONN_LOG_BUFFER_SIZE 1283 +#define WT_STAT_CONN_LOG_BUFFER_SIZE 1284 /*! log: total size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_LEN 1284 +#define WT_STAT_CONN_LOG_COMPRESS_LEN 1285 /*! log: written slots coalesced */ -#define WT_STAT_CONN_LOG_SLOT_COALESCED 1285 +#define WT_STAT_CONN_LOG_SLOT_COALESCED 1286 /*! log: yields waiting for previous log file close */ -#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1286 +#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1287 /*! perf: file system read latency histogram (bucket 1) - 10-49ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT50 1287 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT50 1288 /*! perf: file system read latency histogram (bucket 2) - 50-99ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT100 1288 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT100 1289 /*! perf: file system read latency histogram (bucket 3) - 100-249ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT250 1289 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT250 1290 /*! perf: file system read latency histogram (bucket 4) - 250-499ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT500 1290 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT500 1291 /*! perf: file system read latency histogram (bucket 5) - 500-999ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT1000 1291 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT1000 1292 /*! perf: file system read latency histogram (bucket 6) - 1000ms+ */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_GT1000 1292 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_GT1000 1293 /*! perf: file system write latency histogram (bucket 1) - 10-49ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT50 1293 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT50 1294 /*! perf: file system write latency histogram (bucket 2) - 50-99ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT100 1294 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT100 1295 /*! perf: file system write latency histogram (bucket 3) - 100-249ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT250 1295 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT250 1296 /*! perf: file system write latency histogram (bucket 4) - 250-499ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT500 1296 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT500 1297 /*! perf: file system write latency histogram (bucket 5) - 500-999ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT1000 1297 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT1000 1298 /*! perf: file system write latency histogram (bucket 6) - 1000ms+ */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_GT1000 1298 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_GT1000 1299 /*! perf: operation read latency histogram (bucket 1) - 100-249us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT250 1299 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT250 1300 /*! perf: operation read latency histogram (bucket 2) - 250-499us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT500 1300 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT500 1301 /*! perf: operation read latency histogram (bucket 3) - 500-999us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT1000 1301 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT1000 1302 /*! perf: operation read latency histogram (bucket 4) - 1000-9999us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT10000 1302 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT10000 1303 /*! perf: operation read latency histogram (bucket 5) - 10000us+ */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_GT10000 1303 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_GT10000 1304 /*! perf: operation write latency histogram (bucket 1) - 100-249us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT250 1304 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT250 1305 /*! perf: operation write latency histogram (bucket 2) - 250-499us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT500 1305 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT500 1306 /*! perf: operation write latency histogram (bucket 3) - 500-999us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT1000 1306 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT1000 1307 /*! perf: operation write latency histogram (bucket 4) - 1000-9999us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT10000 1307 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT10000 1308 /*! perf: operation write latency histogram (bucket 5) - 10000us+ */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_GT10000 1308 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_GT10000 1309 /*! reconciliation: fast-path pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1309 +#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1310 /*! reconciliation: page reconciliation calls */ -#define WT_STAT_CONN_REC_PAGES 1310 +#define WT_STAT_CONN_REC_PAGES 1311 /*! reconciliation: page reconciliation calls for eviction */ -#define WT_STAT_CONN_REC_PAGES_EVICTION 1311 +#define WT_STAT_CONN_REC_PAGES_EVICTION 1312 /*! reconciliation: pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE 1312 +#define WT_STAT_CONN_REC_PAGE_DELETE 1313 /*! reconciliation: split bytes currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1313 +#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1314 /*! reconciliation: split objects currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1314 +#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1315 /*! session: open session count */ -#define WT_STAT_CONN_SESSION_OPEN 1315 +#define WT_STAT_CONN_SESSION_OPEN 1316 /*! session: session query timestamp calls */ -#define WT_STAT_CONN_SESSION_QUERY_TS 1316 +#define WT_STAT_CONN_SESSION_QUERY_TS 1317 /*! session: table alter failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_FAIL 1317 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_FAIL 1318 /*! session: table alter successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_SUCCESS 1318 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_SUCCESS 1319 /*! session: table alter unchanged and skipped */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_SKIP 1319 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_SKIP 1320 /*! session: table compact failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_FAIL 1320 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_FAIL 1321 /*! session: table compact successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SUCCESS 1321 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SUCCESS 1322 /*! session: table create failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_CREATE_FAIL 1322 +#define WT_STAT_CONN_SESSION_TABLE_CREATE_FAIL 1323 /*! session: table create successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_CREATE_SUCCESS 1323 +#define WT_STAT_CONN_SESSION_TABLE_CREATE_SUCCESS 1324 /*! session: table drop failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_DROP_FAIL 1324 +#define WT_STAT_CONN_SESSION_TABLE_DROP_FAIL 1325 /*! session: table drop successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_DROP_SUCCESS 1325 +#define WT_STAT_CONN_SESSION_TABLE_DROP_SUCCESS 1326 /*! session: table import failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_IMPORT_FAIL 1326 +#define WT_STAT_CONN_SESSION_TABLE_IMPORT_FAIL 1327 /*! session: table import successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_IMPORT_SUCCESS 1327 +#define WT_STAT_CONN_SESSION_TABLE_IMPORT_SUCCESS 1328 /*! session: table rebalance failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_FAIL 1328 +#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_FAIL 1329 /*! session: table rebalance successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_SUCCESS 1329 +#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_SUCCESS 1330 /*! session: table rename failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1330 +#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1331 /*! session: table rename successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1331 +#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1332 /*! session: table salvage failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1332 +#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1333 /*! session: table salvage successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1333 +#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1334 /*! session: table truncate failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1334 +#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1335 /*! session: table truncate successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1335 +#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1336 /*! session: table verify failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1336 +#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1337 /*! session: table verify successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1337 +#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1338 /*! thread-state: active filesystem fsync calls */ -#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1338 +#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1339 /*! thread-state: active filesystem read calls */ -#define WT_STAT_CONN_THREAD_READ_ACTIVE 1339 +#define WT_STAT_CONN_THREAD_READ_ACTIVE 1340 /*! thread-state: active filesystem write calls */ -#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1340 +#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1341 /*! thread-yield: application thread time evicting (usecs) */ -#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1341 +#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1342 /*! thread-yield: application thread time waiting for cache (usecs) */ -#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1342 +#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1343 /*! * thread-yield: connection close blocked waiting for transaction state * stabilization */ -#define WT_STAT_CONN_TXN_RELEASE_BLOCKED 1343 +#define WT_STAT_CONN_TXN_RELEASE_BLOCKED 1344 /*! thread-yield: connection close yielded for lsm manager shutdown */ -#define WT_STAT_CONN_CONN_CLOSE_BLOCKED_LSM 1344 +#define WT_STAT_CONN_CONN_CLOSE_BLOCKED_LSM 1345 /*! thread-yield: data handle lock yielded */ -#define WT_STAT_CONN_DHANDLE_LOCK_BLOCKED 1345 +#define WT_STAT_CONN_DHANDLE_LOCK_BLOCKED 1346 /*! * thread-yield: get reference for page index and slot time sleeping * (usecs) */ -#define WT_STAT_CONN_PAGE_INDEX_SLOT_REF_BLOCKED 1346 +#define WT_STAT_CONN_PAGE_INDEX_SLOT_REF_BLOCKED 1347 /*! thread-yield: log server sync yielded for log write */ -#define WT_STAT_CONN_LOG_SERVER_SYNC_BLOCKED 1347 +#define WT_STAT_CONN_LOG_SERVER_SYNC_BLOCKED 1348 /*! thread-yield: page access yielded due to prepare state change */ -#define WT_STAT_CONN_PREPARED_TRANSITION_BLOCKED_PAGE 1348 +#define WT_STAT_CONN_PREPARED_TRANSITION_BLOCKED_PAGE 1349 /*! thread-yield: page acquire busy blocked */ -#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1349 +#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1350 /*! thread-yield: page acquire eviction blocked */ -#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1350 +#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1351 /*! thread-yield: page acquire locked blocked */ -#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1351 +#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1352 /*! thread-yield: page acquire read blocked */ -#define WT_STAT_CONN_PAGE_READ_BLOCKED 1352 +#define WT_STAT_CONN_PAGE_READ_BLOCKED 1353 /*! thread-yield: page acquire time sleeping (usecs) */ -#define WT_STAT_CONN_PAGE_SLEEP 1353 +#define WT_STAT_CONN_PAGE_SLEEP 1354 /*! * thread-yield: page delete rollback time sleeping for state change * (usecs) */ -#define WT_STAT_CONN_PAGE_DEL_ROLLBACK_BLOCKED 1354 +#define WT_STAT_CONN_PAGE_DEL_ROLLBACK_BLOCKED 1355 /*! thread-yield: page reconciliation yielded due to child modification */ -#define WT_STAT_CONN_CHILD_MODIFY_BLOCKED_PAGE 1355 +#define WT_STAT_CONN_CHILD_MODIFY_BLOCKED_PAGE 1356 /*! transaction: Number of prepared updates */ -#define WT_STAT_CONN_TXN_PREPARED_UPDATES_COUNT 1356 +#define WT_STAT_CONN_TXN_PREPARED_UPDATES_COUNT 1357 /*! transaction: Number of prepared updates added to cache overflow */ -#define WT_STAT_CONN_TXN_PREPARED_UPDATES_LOOKASIDE_INSERTS 1357 +#define WT_STAT_CONN_TXN_PREPARED_UPDATES_LOOKASIDE_INSERTS 1358 /*! transaction: durable timestamp queue entries walked */ -#define WT_STAT_CONN_TXN_DURABLE_QUEUE_WALKED 1358 +#define WT_STAT_CONN_TXN_DURABLE_QUEUE_WALKED 1359 /*! transaction: durable timestamp queue insert to empty */ -#define WT_STAT_CONN_TXN_DURABLE_QUEUE_EMPTY 1359 +#define WT_STAT_CONN_TXN_DURABLE_QUEUE_EMPTY 1360 /*! transaction: durable timestamp queue inserts to head */ -#define WT_STAT_CONN_TXN_DURABLE_QUEUE_HEAD 1360 +#define WT_STAT_CONN_TXN_DURABLE_QUEUE_HEAD 1361 /*! transaction: durable timestamp queue inserts total */ -#define WT_STAT_CONN_TXN_DURABLE_QUEUE_INSERTS 1361 +#define WT_STAT_CONN_TXN_DURABLE_QUEUE_INSERTS 1362 /*! transaction: durable timestamp queue length */ -#define WT_STAT_CONN_TXN_DURABLE_QUEUE_LEN 1362 +#define WT_STAT_CONN_TXN_DURABLE_QUEUE_LEN 1363 /*! transaction: number of named snapshots created */ -#define WT_STAT_CONN_TXN_SNAPSHOTS_CREATED 1363 +#define WT_STAT_CONN_TXN_SNAPSHOTS_CREATED 1364 /*! transaction: number of named snapshots dropped */ -#define WT_STAT_CONN_TXN_SNAPSHOTS_DROPPED 1364 +#define WT_STAT_CONN_TXN_SNAPSHOTS_DROPPED 1365 /*! transaction: prepared transactions */ -#define WT_STAT_CONN_TXN_PREPARE 1365 +#define WT_STAT_CONN_TXN_PREPARE 1366 /*! transaction: prepared transactions committed */ -#define WT_STAT_CONN_TXN_PREPARE_COMMIT 1366 +#define WT_STAT_CONN_TXN_PREPARE_COMMIT 1367 /*! transaction: prepared transactions currently active */ -#define WT_STAT_CONN_TXN_PREPARE_ACTIVE 1367 +#define WT_STAT_CONN_TXN_PREPARE_ACTIVE 1368 /*! transaction: prepared transactions rolled back */ -#define WT_STAT_CONN_TXN_PREPARE_ROLLBACK 1368 +#define WT_STAT_CONN_TXN_PREPARE_ROLLBACK 1369 /*! transaction: query timestamp calls */ -#define WT_STAT_CONN_TXN_QUERY_TS 1369 +#define WT_STAT_CONN_TXN_QUERY_TS 1370 /*! transaction: read timestamp queue entries walked */ -#define WT_STAT_CONN_TXN_READ_QUEUE_WALKED 1370 +#define WT_STAT_CONN_TXN_READ_QUEUE_WALKED 1371 /*! transaction: read timestamp queue insert to empty */ -#define WT_STAT_CONN_TXN_READ_QUEUE_EMPTY 1371 +#define WT_STAT_CONN_TXN_READ_QUEUE_EMPTY 1372 /*! transaction: read timestamp queue inserts to head */ -#define WT_STAT_CONN_TXN_READ_QUEUE_HEAD 1372 +#define WT_STAT_CONN_TXN_READ_QUEUE_HEAD 1373 /*! transaction: read timestamp queue inserts total */ -#define WT_STAT_CONN_TXN_READ_QUEUE_INSERTS 1373 +#define WT_STAT_CONN_TXN_READ_QUEUE_INSERTS 1374 /*! transaction: read timestamp queue length */ -#define WT_STAT_CONN_TXN_READ_QUEUE_LEN 1374 +#define WT_STAT_CONN_TXN_READ_QUEUE_LEN 1375 /*! transaction: rollback to stable calls */ -#define WT_STAT_CONN_TXN_ROLLBACK_TO_STABLE 1375 +#define WT_STAT_CONN_TXN_ROLLBACK_TO_STABLE 1376 /*! transaction: rollback to stable updates aborted */ -#define WT_STAT_CONN_TXN_ROLLBACK_UPD_ABORTED 1376 +#define WT_STAT_CONN_TXN_ROLLBACK_UPD_ABORTED 1377 /*! transaction: rollback to stable updates removed from cache overflow */ -#define WT_STAT_CONN_TXN_ROLLBACK_LAS_REMOVED 1377 +#define WT_STAT_CONN_TXN_ROLLBACK_LAS_REMOVED 1378 /*! transaction: set timestamp calls */ -#define WT_STAT_CONN_TXN_SET_TS 1378 +#define WT_STAT_CONN_TXN_SET_TS 1379 /*! transaction: set timestamp durable calls */ -#define WT_STAT_CONN_TXN_SET_TS_DURABLE 1379 +#define WT_STAT_CONN_TXN_SET_TS_DURABLE 1380 /*! transaction: set timestamp durable updates */ -#define WT_STAT_CONN_TXN_SET_TS_DURABLE_UPD 1380 +#define WT_STAT_CONN_TXN_SET_TS_DURABLE_UPD 1381 /*! transaction: set timestamp oldest calls */ -#define WT_STAT_CONN_TXN_SET_TS_OLDEST 1381 +#define WT_STAT_CONN_TXN_SET_TS_OLDEST 1382 /*! transaction: set timestamp oldest updates */ -#define WT_STAT_CONN_TXN_SET_TS_OLDEST_UPD 1382 +#define WT_STAT_CONN_TXN_SET_TS_OLDEST_UPD 1383 /*! transaction: set timestamp stable calls */ -#define WT_STAT_CONN_TXN_SET_TS_STABLE 1383 +#define WT_STAT_CONN_TXN_SET_TS_STABLE 1384 /*! transaction: set timestamp stable updates */ -#define WT_STAT_CONN_TXN_SET_TS_STABLE_UPD 1384 +#define WT_STAT_CONN_TXN_SET_TS_STABLE_UPD 1385 /*! transaction: transaction begins */ -#define WT_STAT_CONN_TXN_BEGIN 1385 +#define WT_STAT_CONN_TXN_BEGIN 1386 /*! transaction: transaction checkpoint currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1386 +#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1387 /*! transaction: transaction checkpoint generation */ -#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1387 +#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1388 /*! transaction: transaction checkpoint max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1388 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1389 /*! transaction: transaction checkpoint min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1389 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1390 /*! * transaction: transaction checkpoint most recent duration for gathering * all handles (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION 1390 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION 1391 /*! * transaction: transaction checkpoint most recent duration for gathering * applied handles (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_APPLY 1391 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_APPLY 1392 /*! * transaction: transaction checkpoint most recent duration for gathering * skipped handles (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_SKIP 1392 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_SKIP 1393 /*! transaction: transaction checkpoint most recent handles applied */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_APPLIED 1393 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_APPLIED 1394 /*! transaction: transaction checkpoint most recent handles skipped */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_SKIPPED 1394 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_SKIPPED 1395 /*! transaction: transaction checkpoint most recent handles walked */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_WALKED 1395 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_WALKED 1396 /*! transaction: transaction checkpoint most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1396 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1397 /*! transaction: transaction checkpoint scrub dirty target */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1397 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1398 /*! transaction: transaction checkpoint scrub time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1398 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1399 /*! transaction: transaction checkpoint total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1399 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1400 /*! transaction: transaction checkpoints */ -#define WT_STAT_CONN_TXN_CHECKPOINT 1400 +#define WT_STAT_CONN_TXN_CHECKPOINT 1401 /*! * transaction: transaction checkpoints skipped because database was * clean */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1401 +#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1402 /*! transaction: transaction failures due to cache overflow */ -#define WT_STAT_CONN_TXN_FAIL_CACHE 1402 +#define WT_STAT_CONN_TXN_FAIL_CACHE 1403 /*! * transaction: transaction fsync calls for checkpoint after allocating * the transaction ID */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1403 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1404 /*! * transaction: transaction fsync duration for checkpoint after * allocating the transaction ID (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1404 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1405 /*! transaction: transaction range of IDs currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_RANGE 1405 +#define WT_STAT_CONN_TXN_PINNED_RANGE 1406 /*! transaction: transaction range of IDs currently pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1406 +#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1407 /*! * transaction: transaction range of IDs currently pinned by named * snapshots */ -#define WT_STAT_CONN_TXN_PINNED_SNAPSHOT_RANGE 1407 +#define WT_STAT_CONN_TXN_PINNED_SNAPSHOT_RANGE 1408 /*! transaction: transaction range of timestamps currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP 1408 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP 1409 /*! transaction: transaction range of timestamps pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_CHECKPOINT 1409 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_CHECKPOINT 1410 /*! * transaction: transaction range of timestamps pinned by the oldest * active read timestamp */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_READER 1410 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_READER 1411 /*! * transaction: transaction range of timestamps pinned by the oldest * timestamp */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_OLDEST 1411 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_OLDEST 1412 /*! transaction: transaction read timestamp of the oldest active reader */ -#define WT_STAT_CONN_TXN_TIMESTAMP_OLDEST_ACTIVE_READ 1412 +#define WT_STAT_CONN_TXN_TIMESTAMP_OLDEST_ACTIVE_READ 1413 /*! transaction: transaction sync calls */ -#define WT_STAT_CONN_TXN_SYNC 1413 +#define WT_STAT_CONN_TXN_SYNC 1414 /*! transaction: transactions committed */ -#define WT_STAT_CONN_TXN_COMMIT 1414 +#define WT_STAT_CONN_TXN_COMMIT 1415 /*! transaction: transactions rolled back */ -#define WT_STAT_CONN_TXN_ROLLBACK 1415 +#define WT_STAT_CONN_TXN_ROLLBACK 1416 /*! transaction: update conflicts */ -#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1416 +#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1417 /*! * @} diff --git a/src/third_party/wiredtiger/src/log/log.c b/src/third_party/wiredtiger/src/log/log.c index 8c1bbfd9222..7f35ec77fba 100644 --- a/src/third_party/wiredtiger/src/log/log.c +++ b/src/third_party/wiredtiger/src/log/log.c @@ -2018,7 +2018,7 @@ __log_salvage_message( * Scan the logs, calling a function on each record found. */ int -__wt_log_scan(WT_SESSION_IMPL *session, WT_LSN *lsnp, uint32_t flags, +__wt_log_scan(WT_SESSION_IMPL *session, WT_LSN *start_lsnp, WT_LSN *end_lsnp, uint32_t flags, int (*func)(WT_SESSION_IMPL *session, WT_ITEM *record, WT_LSN *lsnp, WT_LSN *next_lsnp, void *cookie, int firstrecord), void *cookie) @@ -2056,7 +2056,7 @@ __wt_log_scan(WT_SESSION_IMPL *session, WT_LSN *lsnp, uint32_t flags, if (func == NULL) return (0); - if (lsnp != NULL && LF_ISSET(WT_LOGSCAN_FIRST | WT_LOGSCAN_FROM_CKP)) + if (start_lsnp != NULL && LF_ISSET(WT_LOGSCAN_FIRST | WT_LOGSCAN_FROM_CKP)) WT_RET_MSG(session, WT_ERROR, "choose either a start LSN or a start flag"); /* * Set up the allocation size, starting and ending LSNs. The values for those depend on whether @@ -2065,9 +2065,9 @@ __wt_log_scan(WT_SESSION_IMPL *session, WT_LSN *lsnp, uint32_t flags, lastlog = 0; if (log != NULL) { allocsize = log->allocsize; - end_lsn = log->alloc_lsn; - start_lsn = log->first_lsn; - if (lsnp == NULL) { + WT_ASSIGN_LSN(&end_lsn, &log->alloc_lsn); + WT_ASSIGN_LSN(&start_lsn, &log->first_lsn); + if (start_lsnp == NULL) { if (LF_ISSET(WT_LOGSCAN_FROM_CKP)) start_lsn = log->ckpt_lsn; else if (!LF_ISSET(WT_LOGSCAN_FIRST)) @@ -2097,16 +2097,17 @@ __wt_log_scan(WT_SESSION_IMPL *session, WT_LSN *lsnp, uint32_t flags, WT_SET_LSN(&end_lsn, lastlog, 0); WT_ERR(__wt_fs_directory_list_free(session, &logfiles, logcount)); } - if (lsnp != NULL) { + + if (start_lsnp != NULL) { /* * Offsets must be on allocation boundaries. An invalid LSN from a user should just return * WT_NOTFOUND. It is not an error. But if it is from recovery, we expect valid LSNs so give * more information about that. */ - if (lsnp->l.offset % allocsize != 0) { + if (start_lsnp->l.offset % allocsize != 0) { if (LF_ISSET(WT_LOGSCAN_RECOVER | WT_LOGSCAN_RECOVER_METADATA)) WT_ERR_MSG(session, WT_NOTFOUND, "__wt_log_scan unaligned LSN %" PRIu32 "/%" PRIu32, - lsnp->l.file, lsnp->l.offset); + start_lsnp->l.file, start_lsnp->l.offset); else WT_ERR(WT_NOTFOUND); } @@ -2115,11 +2116,11 @@ __wt_log_scan(WT_SESSION_IMPL *session, WT_LSN *lsnp, uint32_t flags, * return WT_NOTFOUND. It is not an error. But if it is from recovery, we expect valid LSNs * so give more information about that. */ - if (lsnp->l.file > lastlog) { + if (start_lsnp->l.file > lastlog) { if (LF_ISSET(WT_LOGSCAN_RECOVER | WT_LOGSCAN_RECOVER_METADATA)) WT_ERR_MSG(session, WT_NOTFOUND, "__wt_log_scan LSN %" PRIu32 "/%" PRIu32 " larger than biggest log file %" PRIu32, - lsnp->l.file, lsnp->l.offset, lastlog); + start_lsnp->l.file, start_lsnp->l.offset, lastlog); else WT_ERR(WT_NOTFOUND); } @@ -2127,8 +2128,8 @@ __wt_log_scan(WT_SESSION_IMPL *session, WT_LSN *lsnp, uint32_t flags, * Log cursors may not know the starting LSN. If an LSN is passed in that it is equal to the * smallest LSN, start from the beginning of the log. */ - if (!WT_IS_INIT_LSN(lsnp)) - start_lsn = *lsnp; + if (!WT_IS_INIT_LSN(start_lsnp)) + WT_ASSIGN_LSN(&start_lsn, start_lsnp); } WT_ERR(__log_open_verify(session, start_lsn.l.file, &log_fh, &prev_lsn, NULL, &need_salvage)); if (need_salvage) @@ -2363,7 +2364,14 @@ advance: if (LF_ISSET(WT_LOGSCAN_ONE)) break; } - rd_lsn = next_lsn; + + /* + * Exit the scanning loop if the next LSN seen is greater than our user set end range LSN. + */ + if (end_lsnp != NULL && __wt_log_cmp(&next_lsn, end_lsnp) > 0) + break; + + WT_ASSIGN_LSN(&rd_lsn, &next_lsn); } /* Truncate if we're in recovery. */ diff --git a/src/third_party/wiredtiger/src/log/log_auto.c b/src/third_party/wiredtiger/src/log/log_auto.c index 615f4238aa3..0531fff0c11 100644 --- a/src/third_party/wiredtiger/src/log/log_auto.c +++ b/src/third_party/wiredtiger/src/log/log_auto.c @@ -136,6 +136,11 @@ __wt_logop_col_modify_print( escaped = NULL; WT_RET(__wt_logop_col_modify_unpack(session, pp, end, &fileid, &recno, &value)); + if (!FLD_ISSET(args->flags, WT_TXN_PRINTLOG_UNREDACT) && fileid != WT_METAFILE_ID) { + WT_RET(__wt_fprintf(session, args->fs, " REDACTED")); + return (0); + } + WT_RET(__wt_fprintf(session, args->fs, " \"optype\": \"col_modify\",\n")); WT_ERR(__wt_fprintf( session, args->fs, " \"fileid\": %" PRIu32 " 0x%" PRIx32 ",\n", fileid, fileid)); @@ -203,6 +208,11 @@ __wt_logop_col_put_print( escaped = NULL; WT_RET(__wt_logop_col_put_unpack(session, pp, end, &fileid, &recno, &value)); + if (!FLD_ISSET(args->flags, WT_TXN_PRINTLOG_UNREDACT) && fileid != WT_METAFILE_ID) { + WT_RET(__wt_fprintf(session, args->fs, " REDACTED")); + return (0); + } + WT_RET(__wt_fprintf(session, args->fs, " \"optype\": \"col_put\",\n")); WT_ERR(__wt_fprintf( session, args->fs, " \"fileid\": %" PRIu32 " 0x%" PRIx32 ",\n", fileid, fileid)); @@ -266,6 +276,11 @@ __wt_logop_col_remove_print( WT_RET(__wt_logop_col_remove_unpack(session, pp, end, &fileid, &recno)); + if (!FLD_ISSET(args->flags, WT_TXN_PRINTLOG_UNREDACT) && fileid != WT_METAFILE_ID) { + WT_RET(__wt_fprintf(session, args->fs, " REDACTED")); + return (0); + } + WT_RET(__wt_fprintf(session, args->fs, " \"optype\": \"col_remove\",\n")); WT_RET(__wt_fprintf( session, args->fs, " \"fileid\": %" PRIu32 " 0x%" PRIx32 ",\n", fileid, fileid)); @@ -321,6 +336,11 @@ __wt_logop_col_truncate_print( WT_RET(__wt_logop_col_truncate_unpack(session, pp, end, &fileid, &start, &stop)); + if (!FLD_ISSET(args->flags, WT_TXN_PRINTLOG_UNREDACT) && fileid != WT_METAFILE_ID) { + WT_RET(__wt_fprintf(session, args->fs, " REDACTED")); + return (0); + } + WT_RET(__wt_fprintf(session, args->fs, " \"optype\": \"col_truncate\",\n")); WT_RET(__wt_fprintf( session, args->fs, " \"fileid\": %" PRIu32 " 0x%" PRIx32 ",\n", fileid, fileid)); @@ -380,6 +400,11 @@ __wt_logop_row_modify_print( escaped = NULL; WT_RET(__wt_logop_row_modify_unpack(session, pp, end, &fileid, &key, &value)); + if (!FLD_ISSET(args->flags, WT_TXN_PRINTLOG_UNREDACT) && fileid != WT_METAFILE_ID) { + WT_RET(__wt_fprintf(session, args->fs, " REDACTED")); + return (0); + } + WT_RET(__wt_fprintf(session, args->fs, " \"optype\": \"row_modify\",\n")); WT_ERR(__wt_fprintf( session, args->fs, " \"fileid\": %" PRIu32 " 0x%" PRIx32 ",\n", fileid, fileid)); @@ -452,6 +477,11 @@ __wt_logop_row_put_print( escaped = NULL; WT_RET(__wt_logop_row_put_unpack(session, pp, end, &fileid, &key, &value)); + if (!FLD_ISSET(args->flags, WT_TXN_PRINTLOG_UNREDACT) && fileid != WT_METAFILE_ID) { + WT_RET(__wt_fprintf(session, args->fs, " REDACTED")); + return (0); + } + WT_RET(__wt_fprintf(session, args->fs, " \"optype\": \"row_put\",\n")); WT_ERR(__wt_fprintf( session, args->fs, " \"fileid\": %" PRIu32 " 0x%" PRIx32 ",\n", fileid, fileid)); @@ -522,6 +552,11 @@ __wt_logop_row_remove_print( escaped = NULL; WT_RET(__wt_logop_row_remove_unpack(session, pp, end, &fileid, &key)); + if (!FLD_ISSET(args->flags, WT_TXN_PRINTLOG_UNREDACT) && fileid != WT_METAFILE_ID) { + WT_RET(__wt_fprintf(session, args->fs, " REDACTED")); + return (0); + } + WT_RET(__wt_fprintf(session, args->fs, " \"optype\": \"row_remove\",\n")); WT_ERR(__wt_fprintf( session, args->fs, " \"fileid\": %" PRIu32 " 0x%" PRIx32 ",\n", fileid, fileid)); @@ -589,6 +624,11 @@ __wt_logop_row_truncate_print( escaped = NULL; WT_RET(__wt_logop_row_truncate_unpack(session, pp, end, &fileid, &start, &stop, &mode)); + if (!FLD_ISSET(args->flags, WT_TXN_PRINTLOG_UNREDACT) && fileid != WT_METAFILE_ID) { + WT_RET(__wt_fprintf(session, args->fs, " REDACTED")); + return (0); + } + WT_RET(__wt_fprintf(session, args->fs, " \"optype\": \"row_truncate\",\n")); WT_ERR(__wt_fprintf( session, args->fs, " \"fileid\": %" PRIu32 " 0x%" PRIx32 ",\n", fileid, fileid)); diff --git a/src/third_party/wiredtiger/src/lsm/lsm_manager.c b/src/third_party/wiredtiger/src/lsm/lsm_manager.c index dff03d2663d..2cbde2fbf90 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_manager.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_manager.c @@ -213,7 +213,7 @@ __wt_lsm_manager_start(WT_SESSION_IMPL *session) manager->lsm_worker_cookies[i].session = worker_session; } - F_SET(conn, WT_CONN_SERVER_LSM); + FLD_SET(conn->server_flags, WT_CONN_SERVER_LSM); /* Start the LSM manager thread. */ WT_ERR(__wt_thread_create(session, &manager->lsm_worker_cookies[0].tid, __lsm_worker_manager, @@ -270,7 +270,7 @@ __wt_lsm_manager_destroy(WT_SESSION_IMPL *session) removed = 0; /* Clear the LSM server flag. */ - F_CLR(conn, WT_CONN_SERVER_LSM); + FLD_CLR(conn->server_flags, WT_CONN_SERVER_LSM); WT_ASSERT(session, !F_ISSET(conn, WT_CONN_READONLY) || manager->lsm_workers == 0); if (manager->lsm_workers > 0) { @@ -354,7 +354,7 @@ __lsm_manager_run_server(WT_SESSION_IMPL *session) conn = S2C(session); dhandle_locked = false; - while (F_ISSET(conn, WT_CONN_SERVER_LSM)) { + while (FLD_ISSET(conn->server_flags, WT_CONN_SERVER_LSM)) { __wt_sleep(0, 10000); if (TAILQ_EMPTY(&conn->lsmqh)) continue; diff --git a/src/third_party/wiredtiger/src/session/session_api.c b/src/third_party/wiredtiger/src/session/session_api.c index 24b969c1a3b..6f829be0b7b 100644 --- a/src/third_party/wiredtiger/src/session/session_api.c +++ b/src/third_party/wiredtiger/src/session/session_api.c @@ -1828,7 +1828,7 @@ __transaction_sync_run_chk(WT_SESSION_IMPL *session) conn = S2C(session); - return (FLD_ISSET(conn->flags, WT_CONN_SERVER_LOG)); + return (FLD_ISSET(conn->server_flags, WT_CONN_SERVER_LOG)); } /* diff --git a/src/third_party/wiredtiger/src/support/stat.c b/src/third_party/wiredtiger/src/support/stat.c index 2981efc27ec..83ca498d486 100644 --- a/src/third_party/wiredtiger/src/support/stat.c +++ b/src/third_party/wiredtiger/src/support/stat.c @@ -899,6 +899,7 @@ static const char *const __stats_connection_desc[] = { "data-handle: connection sweep dhandles removed from hash list", "data-handle: connection sweep time-of-death sets", "data-handle: connection sweeps", + "data-handle: connection sweeps skipped due to checkpoint gathering handles", "data-handle: session dhandles swept", "data-handle: session sweep attempts", "lock: checkpoint lock acquisitions", @@ -1357,6 +1358,7 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->dh_sweep_remove = 0; stats->dh_sweep_tod = 0; stats->dh_sweeps = 0; + stats->dh_sweep_skip_ckpt = 0; stats->dh_session_handles = 0; stats->dh_session_sweeps = 0; stats->lock_checkpoint_count = 0; @@ -1811,6 +1813,7 @@ __wt_stat_connection_aggregate(WT_CONNECTION_STATS **from, WT_CONNECTION_STATS * to->dh_sweep_remove += WT_STAT_READ(from, dh_sweep_remove); to->dh_sweep_tod += WT_STAT_READ(from, dh_sweep_tod); to->dh_sweeps += WT_STAT_READ(from, dh_sweeps); + to->dh_sweep_skip_ckpt += WT_STAT_READ(from, dh_sweep_skip_ckpt); to->dh_session_handles += WT_STAT_READ(from, dh_session_handles); to->dh_session_sweeps += WT_STAT_READ(from, dh_session_sweeps); to->lock_checkpoint_count += WT_STAT_READ(from, lock_checkpoint_count); diff --git a/src/third_party/wiredtiger/src/txn/txn_log.c b/src/third_party/wiredtiger/src/txn/txn_log.c index 73d3f32ed88..4c6e4faac39 100644 --- a/src/third_party/wiredtiger/src/txn/txn_log.c +++ b/src/third_party/wiredtiger/src/txn/txn_log.c @@ -717,8 +717,8 @@ __txn_printlog(WT_SESSION_IMPL *session, WT_ITEM *rawrec, WT_LSN *lsnp, WT_LSN * * Print the log in a human-readable format. */ int -__wt_txn_printlog(WT_SESSION *wt_session, const char *ofile, uint32_t flags) - WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) +__wt_txn_printlog(WT_SESSION *wt_session, const char *ofile, uint32_t flags, WT_LSN *start_lsn, + WT_LSN *end_lsn) WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { WT_DECL_RET; WT_FSTREAM *fs; @@ -735,8 +735,9 @@ __wt_txn_printlog(WT_SESSION *wt_session, const char *ofile, uint32_t flags) WT_ERR(__wt_fprintf(session, fs, "[\n")); args.fs = fs; args.flags = flags; - WT_ERR(__wt_log_scan(session, NULL, WT_LOGSCAN_FIRST, __txn_printlog, &args)); - ret = __wt_fprintf(session, fs, "\n]\n"); + WT_ERR(__wt_log_scan(session, start_lsn, end_lsn, 0x0, __txn_printlog, &args)); + if (!LF_ISSET(WT_TXN_PRINTLOG_MSG)) + ret = __wt_fprintf(session, fs, "\n]\n"); err: if (ofile != NULL) diff --git a/src/third_party/wiredtiger/src/txn/txn_recover.c b/src/third_party/wiredtiger/src/txn/txn_recover.c index f546d8f2e43..03a88afd466 100644 --- a/src/third_party/wiredtiger/src/txn/txn_recover.c +++ b/src/third_party/wiredtiger/src/txn/txn_recover.c @@ -595,15 +595,15 @@ __wt_txn_recover(WT_SESSION_IMPL *session) WT_ERR_MSG(session, WT_RUN_RECOVERY, "Read-only database needs recovery"); } if (WT_IS_INIT_LSN(&metafile->ckpt_lsn)) - ret = __wt_log_scan(session, NULL, WT_LOGSCAN_FIRST, __txn_log_recover, &r); + ret = __wt_log_scan(session, NULL, NULL, WT_LOGSCAN_FIRST, __txn_log_recover, &r); else { /* * Start at the last checkpoint LSN referenced in the metadata. If we see the end of a * checkpoint while scanning, we will change the full scan to start from there. */ - r.ckpt_lsn = metafile->ckpt_lsn; - ret = __wt_log_scan( - session, &metafile->ckpt_lsn, WT_LOGSCAN_RECOVER_METADATA, __txn_log_recover, &r); + WT_ASSIGN_LSN(&r.ckpt_lsn, &metafile->ckpt_lsn); + ret = __wt_log_scan(session, &metafile->ckpt_lsn, NULL, WT_LOGSCAN_RECOVER_METADATA, + __txn_log_recover, &r); } if (F_ISSET(conn, WT_CONN_SALVAGE)) ret = 0; @@ -674,9 +674,9 @@ __wt_txn_recover(WT_SESSION_IMPL *session) FLD_SET(conn->log_flags, WT_CONN_LOG_RECOVER_DIRTY); if (WT_IS_INIT_LSN(&r.ckpt_lsn)) ret = __wt_log_scan( - session, NULL, WT_LOGSCAN_FIRST | WT_LOGSCAN_RECOVER, __txn_log_recover, &r); + session, NULL, NULL, WT_LOGSCAN_FIRST | WT_LOGSCAN_RECOVER, __txn_log_recover, &r); else - ret = __wt_log_scan(session, &r.ckpt_lsn, WT_LOGSCAN_RECOVER, __txn_log_recover, &r); + ret = __wt_log_scan(session, &r.ckpt_lsn, NULL, WT_LOGSCAN_RECOVER, __txn_log_recover, &r); if (F_ISSET(conn, WT_CONN_SALVAGE)) ret = 0; WT_ERR(ret); diff --git a/src/third_party/wiredtiger/src/utilities/util.h b/src/third_party/wiredtiger/src/utilities/util.h index d0078edc45f..fc3e3b93128 100644 --- a/src/third_party/wiredtiger/src/utilities/util.h +++ b/src/third_party/wiredtiger/src/utilities/util.h @@ -53,5 +53,6 @@ int util_str2num(WT_SESSION *, const char *, bool, uint64_t *); int util_truncate(WT_SESSION *, int, char *[]); int util_upgrade(WT_SESSION *, int, char *[]); char *util_uri(WT_SESSION *, const char *, const char *); +void util_usage(const char *, const char *, const char *[]); int util_verify(WT_SESSION *, int, char *[]); int util_write(WT_SESSION *, int, char *[]); diff --git a/src/third_party/wiredtiger/src/utilities/util_misc.c b/src/third_party/wiredtiger/src/utilities/util_misc.c index 6ea5e688fae..d0fff3519e9 100644 --- a/src/third_party/wiredtiger/src/utilities/util_misc.c +++ b/src/third_party/wiredtiger/src/utilities/util_misc.c @@ -152,3 +152,21 @@ util_flush(WT_SESSION *session, const char *uri) (void)util_err(session, ret, "%s: session.drop", uri); return (1); } + +/* + * util_usage -- + * Display a usage statement. + */ +void +util_usage(const char *usage, const char *tag, const char *list[]) +{ + const char **p; + + if (usage != NULL) + fprintf(stderr, "usage: %s %s %s\n", progname, usage_prefix, usage); + if (tag != NULL) + fprintf(stderr, "%s\n", tag); + if (list != NULL) + for (p = list; *p != NULL; p += 2) + fprintf(stderr, " %s%s%s\n", p[0], strlen(p[0]) > 2 ? "\n " : " ", p[1]); +} diff --git a/src/third_party/wiredtiger/src/utilities/util_printlog.c b/src/third_party/wiredtiger/src/utilities/util_printlog.c index b9ce0a59435..8581e896313 100644 --- a/src/third_party/wiredtiger/src/utilities/util_printlog.c +++ b/src/third_party/wiredtiger/src/utilities/util_printlog.c @@ -8,23 +8,67 @@ #include "util.h" -static int usage(void); +static int +usage(void) +{ + static const char *options[] = {"-f", "output to the specified file", "-x", + "display key and value items in hexadecimal format", "-l", + "the start LSN from which the log will be printed, optionally the end LSN can also be " + "specified", + "-m", "output log message records only", "-u", "print user data, don't redact", NULL, NULL}; + + util_usage( + "printlog [-mux] [-f output-file] [-l start-file,start-offset]|[-l " + "start-file,start-offset,end-file,end-offset]", + "options:", options); + return (1); +} int util_printlog(WT_SESSION *session, int argc, char *argv[]) { WT_DECL_RET; + WT_LSN end_lsn, start_lsn; + uint32_t end_lsnfile, end_lsnoffset, start_lsnfile, start_lsnoffset; uint32_t flags; int ch; - char *ofile; + int n_args; + char *ofile, *start_str; + bool end_set, start_set; + end_set = start_set = false; flags = 0; ofile = NULL; - while ((ch = __wt_getopt(progname, argc, argv, "f:x")) != EOF) + /* + * By default redact user data. This way if any support people are using this on customer data, + * it is redacted unless they make the effort to keep it in. It lessens the risk of doing the + * wrong command. + */ + while ((ch = __wt_getopt(progname, argc, argv, "f:l:mux")) != EOF) switch (ch) { case 'f': /* output file */ ofile = __wt_optarg; break; + case 'l': + start_str = __wt_optarg; + n_args = sscanf(start_str, "%" SCNu32 ",%" SCNu32 ",%" SCNu32 ",%" SCNu32, + &start_lsnfile, &start_lsnoffset, &end_lsnfile, &end_lsnoffset); + if (n_args == 2) { + WT_SET_LSN(&start_lsn, start_lsnfile, start_lsnoffset); + start_set = true; + } else if (n_args == 4) { + WT_SET_LSN(&start_lsn, start_lsnfile, start_lsnoffset); + WT_SET_LSN(&end_lsn, end_lsnfile, end_lsnoffset); + end_set = start_set = true; + } else + return (usage()); + break; + case 'm': /* messages only */ + LF_SET(WT_TXN_PRINTLOG_MSG); + break; + case 'u': /* print user data, don't redact */ + LF_SET(WT_TXN_PRINTLOG_UNREDACT); + break; case 'x': /* hex output */ LF_SET(WT_TXN_PRINTLOG_HEX); break; @@ -38,15 +82,9 @@ util_printlog(WT_SESSION *session, int argc, char *argv[]) if (argc != 0) return (usage()); - if ((ret = __wt_txn_printlog(session, ofile, flags)) != 0) + if ((ret = __wt_txn_printlog(session, ofile, flags, start_set == true ? &start_lsn : NULL, + end_set == true ? &end_lsn : NULL)) != 0) (void)util_err(session, ret, "printlog"); return (ret); } - -static int -usage(void) -{ - (void)fprintf(stderr, "usage: %s %s printlog [-x] [-f output-file]\n", progname, usage_prefix); - return (1); -} diff --git a/src/third_party/wiredtiger/test/suite/suite_subprocess.py b/src/third_party/wiredtiger/test/suite/suite_subprocess.py index d586f2a7997..a1bdeb24241 100755 --- a/src/third_party/wiredtiger/test/suite/suite_subprocess.py +++ b/src/third_party/wiredtiger/test/suite/suite_subprocess.py @@ -93,7 +93,9 @@ class suite_subprocess: got = f.read(len(expect) + 100) self.assertEqual(got, expect, filename + ': does not contain expected:\n\'' + expect + '\', but contains:\n\'' + got + '\'.') - def check_file_contains_one_of(self, filename, expectlist): + # Check contents of the file against a provided checklist. Expected is used as a bool to either + # ensure checklist is included or ensure the checklist is not included in the file. + def check_file_contains_one_of(self, filename, checklist, expected): """ Check that the file contains the expected string in the first 100K bytes """ @@ -101,21 +103,27 @@ class suite_subprocess: with open(filename, 'r') as f: got = f.read(maxbytes) found = False - for expect in expectlist: + for expect in checklist: pat = self.convert_to_pattern(expect) if pat == None: if expect in got: found = True - break + if expected: + break + else: + self.fail("Did not expect: " + got) else: if re.search(pat, got): found = True - break - if not found: - if len(expectlist) == 1: - expect = '\'' + expectlist[0] + '\'' + if expected: + break + else: + self.fail("Did not expect: " + got) + if not found and expected: + if len(checklist) == 1: + expect = '\'' + checklist[0] + '\'' else: - expect = str(expectlist) + expect = str(checklist) gotstr = '\'' + \ (got if len(got) < 1000 else (got[0:1000] + '...')) + '\'' if len(got) >= maxbytes: @@ -123,8 +131,11 @@ class suite_subprocess: else: self.fail(filename + ': does not contain expected ' + expect + ', got ' + gotstr) - def check_file_contains(self, filename, expect): - self.check_file_contains_one_of(filename, [expect]) + def check_file_contains(self, filename, content): + self.check_file_contains_one_of(filename, [content], True) + + def check_file_not_contains(self, filename, content): + self.check_file_contains_one_of(filename, [content], False) def check_empty_file(self, filename): """ diff --git a/src/third_party/wiredtiger/test/suite/test_backup13.py b/src/third_party/wiredtiger/test/suite/test_backup13.py index 46d360c09bb..cbf7439322d 100644 --- a/src/third_party/wiredtiger/test/suite/test_backup13.py +++ b/src/third_party/wiredtiger/test/suite/test_backup13.py @@ -39,15 +39,40 @@ class test_backup13(wttest.WiredTigerTestCase, suite_subprocess): conn_config='cache_size=1G,log=(enabled,file_max=100K)' dir='backup.dir' # Backup directory name logmax="100K" - uri="table:test" - nops=1000 mult=0 + nops=1000 + uri="table:test" + + scenarios = make_scenarios([ + ('default', dict(sess_cfg='')), + ('read-committed', dict(sess_cfg='isolation=read-committed')), + ('read-uncommitted', dict(sess_cfg='isolation=read-uncommitted')), + ('snapshot', dict(sess_cfg='isolation=snapshot')), + ]) pfx = 'test_backup' # Set the key and value big enough that we modify a few blocks. bigkey = 'Key' * 100 bigval = 'Value' * 100 + def simulate_crash_restart(self, olddir, newdir): + ''' Simulate a crash from olddir and restart in newdir. ''' + # with the connection still open, copy files to new directory + shutil.rmtree(newdir, ignore_errors=True) + os.mkdir(newdir) + for fname in os.listdir(olddir): + fullname = os.path.join(olddir, fname) + # Skip lock file on Windows since it is locked + if os.path.isfile(fullname) and \ + "WiredTiger.lock" not in fullname and \ + "Tmplog" not in fullname and \ + "Preplog" not in fullname: + shutil.copy(fullname, newdir) + # close the original connection and open to new directory + self.close_conn() + self.conn = self.setUpConnectionOpen(newdir) + self.session = self.setUpSessionOpen(self.conn) + def add_data(self, uri): c = self.session.open_cursor(uri) for i in range(0, self.nops): @@ -60,8 +85,10 @@ class test_backup13(wttest.WiredTigerTestCase, suite_subprocess): # Increase the multiplier so that later calls insert unique items. self.mult += 1 - def test_backup13(self): + def session_config(self): + return self.sess_cfg + def test_backup13(self): self.session.create(self.uri, "key_format=S,value_format=S") self.add_data(self.uri) @@ -156,8 +183,15 @@ class test_backup13(wttest.WiredTigerTestCase, suite_subprocess): # Make sure after a force stop we cannot access old backup info. config = 'incremental=(src_id="ID1",this_id="ID3")' + + self.assertRaises(wiredtiger.WiredTigerError, + lambda: self.session.open_cursor('backup:', None, config)) + + # Make sure after a crash we cannot access old backup info. + self.simulate_crash_restart(".", "RESTART") self.assertRaises(wiredtiger.WiredTigerError, lambda: self.session.open_cursor('backup:', None, config)) + self.reopen_conn() # Make sure after a restart we cannot access old backup info. self.assertRaises(wiredtiger.WiredTigerError, diff --git a/src/third_party/wiredtiger/test/suite/test_backup20.py b/src/third_party/wiredtiger/test/suite/test_backup20.py new file mode 100644 index 00000000000..81deb6026bb --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_backup20.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2020 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +import os, shutil +from helper import compare_files +from suite_subprocess import suite_subprocess +from wtdataset import simple_key +from wtscenario import make_scenarios + +# test_backup20.py +# Test cursor backup force stop without a checkpoint. +# This reproduces the issue from WT-7027 where we hit an assertion +# because the session was created with snapshot isolation. +class test_backup20(wttest.WiredTigerTestCase, suite_subprocess): + conn_config='cache_size=1G,log=(enabled,file_max=100K)' + dir='backup.dir' # Backup directory name + logmax="100K" + uri="table:test" + nops=1000 + mult=0 + + pfx = 'test_backup' + + scenarios = make_scenarios([ + ('default', dict(sess_cfg='')), + ('read-committed', dict(sess_cfg='isolation=read-committed')), + ('read-uncommitted', dict(sess_cfg='isolation=read-uncommitted')), + ('snapshot', dict(sess_cfg='isolation=snapshot')), + ]) + + def session_config(self): + return self.sess_cfg + + def test_backup20(self): + self.session.create(self.uri, "key_format=S,value_format=S") + + # Open up the backup cursor. This causes a new log file to be created. + # That log file is not part of the list returned. This is a full backup + # primary cursor with incremental configured. + config = 'incremental=(enabled,granularity=1M,this_id="ID1")' + bkup_c = self.session.open_cursor('backup:', None, config) + bkup_c.close() + + # Do a force stop to release resources and reset the system. + config = 'incremental=(force_stop=true)' + bkup_c = self.session.open_cursor('backup:', None, config) + bkup_c.close() + + self.session.close() + self.conn.close() + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_txn08.py b/src/third_party/wiredtiger/test/suite/test_txn08.py index 7116aa4eb75..f750f0d8dae 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn08.py +++ b/src/third_party/wiredtiger/test/suite/test_txn08.py @@ -64,14 +64,55 @@ class test_txn08(wttest.WiredTigerTestCase, suite_subprocess): # # Run printlog and make sure it exits with zero status. # - self.runWt(['printlog'], outfilename='printlog.out') + self.runWt(['printlog', '-u'], outfilename='printlog.out') self.check_file_contains('printlog.out', '\\u0001\\u0002abcd\\u0003\\u0004') - self.runWt(['printlog', '-x'], outfilename='printlog-hex.out') + self.runWt(['printlog', '-u','-x'], outfilename='printlog-hex.out') self.check_file_contains('printlog-hex.out', '\\u0001\\u0002abcd\\u0003\\u0004') self.check_file_contains('printlog-hex.out', '0102616263640304') + # Check the printlog start LSN and stop LSN feature. + self.runWt(['printlog', '-l 2,128'], outfilename='printlog-range01.out') + self.check_file_contains('printlog-range01.out', + '"lsn" : [2,128],') + self.check_file_contains('printlog-range01.out', + '"lsn" : [2,256],') + self.check_file_not_contains('printlog-range01.out', + '"lsn" : [1,128],') + self.runWt(['printlog', '-l 2,128,3,128'], outfilename='printlog-range02.out') + self.check_file_contains('printlog-range02.out', + '"lsn" : [2,128],') + self.check_file_not_contains('printlog-range02.out', + '"lsn" : [1,128],') + self.check_file_not_contains('printlog-range02.out', + '"lsn" : [3,256],') + # Test for invalid LSN, return WT_NOTFOUND + self.runWt(['printlog', '-l 2,300'], outfilename='printlog-range03.out', errfilename='printlog-range03.err', failure=True) + self.check_file_contains('printlog-range03.err','WT_NOTFOUND') + # Test for Start > end, print the start lsn and then stop + self.runWt(['printlog', '-l 3,128,2,128'], outfilename='printlog-range04.out') + self.check_file_contains('printlog-range04.out','"lsn" : [3,128],') + self.check_file_not_contains('printlog-range04.out','"lsn" : [3,256],') + # Test for usage error, print the usage message if arguments are invalid + self.runWt(['printlog', '-l'], outfilename='printlog-range05.out', errfilename='printlog-range05.err', failure=True) + self.check_file_contains('printlog-range05.err','wt: option requires an argument -- l') + # Test start and end offset of 0 + self.runWt(['printlog', '-l 2,0,3,0'], outfilename='printlog-range06.out') + self.check_file_contains('printlog-range06.out', + '"lsn" : [2,128],') + self.check_file_not_contains('printlog-range06.out', + '"lsn" : [1,128],') + self.check_file_not_contains('printlog-range06.out', + '"lsn" : [3,256],') + # Test for start == end + self.runWt(['printlog', '-l 1,256,1,256'], outfilename='printlog-range07.out') + self.check_file_contains('printlog-range07.out', + '"lsn" : [1,256],') + self.check_file_not_contains('printlog-range07.out', + '"lsn" : [1,128],') + self.check_file_not_contains('printlog-range07.out', + '"lsn" : [1,384],') if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_txn19.py b/src/third_party/wiredtiger/test/suite/test_txn19.py index 12484cd001a..0c8e59eaf54 100755 --- a/src/third_party/wiredtiger/test/suite/test_txn19.py +++ b/src/third_party/wiredtiger/test/suite/test_txn19.py @@ -519,7 +519,7 @@ class test_txn19_meta(wttest.WiredTigerTestCase, suite_subprocess): if expect_fail: self.check_file_contains_one_of(errfile, - ['WT_TRY_SALVAGE: database corruption detected']) + ['WT_TRY_SALVAGE: database corruption detected'], True) for salvagedir in [ newdir, newdir2 ]: # Removing the 'WiredTiger.turtle' file has weird behavior: |