diff options
| author | Luke Chen <luke.chen@mongodb.com> | 2018-12-11 10:34:04 +1100 |
|---|---|---|
| committer | Luke Chen <luke.chen@mongodb.com> | 2018-12-11 10:44:49 +1100 |
| commit | 61687c883a8df23c6740f16d9f448c352b46a6f5 (patch) | |
| tree | 1c90c24b18ec002e4b43e9c69d9577fb34c45643 /src/third_party/wiredtiger | |
| parent | e6a0068b690c1bfc53f090401e1f066be3574733 (diff) | |
| download | mongo-61687c883a8df23c6740f16d9f448c352b46a6f5.tar.gz | |
Import wiredtiger: 5812c92f5fb2ed3b58d0398e5cbe1cb33380d450 from branch mongodb-4.2
ref: fcb59a43a4..5812c92f5f
for: 4.1.7
WT-4383 Update session statistics to reflect the operation statistics.
WT-4384 Add documentation about session statistics cursor
WT-4438 Use more accurate statistics for cursor cache totals
WT-4454 Auto-generate Evergreen configuration when adding new 'make check' or csuite tests
WT-4470 Minimize testing that requires TESTUTIL_ENABLE_LONG_TESTS
WT-4479 Remove schema lock wait assertion from the test
WT-4480 Fix duplicate backup cursor and archiving check
WT-4481 Evergreen script cleanups
Diffstat (limited to 'src/third_party/wiredtiger')
43 files changed, 885 insertions, 332 deletions
diff --git a/src/third_party/wiredtiger/dist/s_all b/src/third_party/wiredtiger/dist/s_all index a0ea9918206..51ea98ab154 100755 --- a/src/third_party/wiredtiger/dist/s_all +++ b/src/third_party/wiredtiger/dist/s_all @@ -86,6 +86,7 @@ run "sh ./s_style" COMMANDS=" 2>&1 ./s_define > ${t_pfx}s_define 2>&1 ./s_docs > ${t_pfx}s_docs +2>&1 ./s_evergreen > ${t_pfx}s_evergreen 2>&1 ./s_export > ${t_pfx}s_export 2>&1 ./s_funcs > ${t_pfx}s_funcs 2>&1 ./s_function > ${t_pfx}s_function diff --git a/src/third_party/wiredtiger/dist/s_copyright.list b/src/third_party/wiredtiger/dist/s_copyright.list index e5014ef5921..934488e3d88 100644 --- a/src/third_party/wiredtiger/dist/s_copyright.list +++ b/src/third_party/wiredtiger/dist/s_copyright.list @@ -41,6 +41,7 @@ skip src/include/flags.h skip src/include/queue.h skip src/log/log_auto.c skip src/support/stat.c +skip test/evergreen/evg_cfg.py skip test/packing/intpack-test.c skip test/packing/intpack-test2.c skip test/packing/packing-test.c diff --git a/src/third_party/wiredtiger/dist/s_evergreen b/src/third_party/wiredtiger/dist/s_evergreen new file mode 100755 index 00000000000..9815d2e64a0 --- /dev/null +++ b/src/third_party/wiredtiger/dist/s_evergreen @@ -0,0 +1,24 @@ +#! /bin/sh + +t=__wt.$$ +trap 'rm -f $t' 0 1 2 3 13 15 + +toplevel_dir=$(git rev-parse --show-toplevel) +program=${toplevel_dir}/test/evergreen/evg_cfg.py + +# Run checking program to identify missing tests in Evergreen configuration +${program} check >$t 2>&1 +e=$? + +test -s $t && { + echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" + echo "$0: $program check" + cat $t + echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" + + # Don't exit non-zero unless the python script did, the script + # requires python modules that are commonly not installed, and + # in that case it exits 0. Post the complaint, but don't fail. + exit $e +} +exit 0 diff --git a/src/third_party/wiredtiger/dist/s_stat b/src/third_party/wiredtiger/dist/s_stat index 32cf16fd3d4..d0ea3ab9cce 100755 --- a/src/third_party/wiredtiger/dist/s_stat +++ b/src/third_party/wiredtiger/dist/s_stat @@ -30,6 +30,7 @@ lock_commit_timestamp_wait_application lock_commit_timestamp_wait_internal lock_commit_timestamp_write_count lock_dhandle_read_count +lock_dhandle_wait lock_dhandle_wait_application lock_dhandle_wait_internal lock_dhandle_write_count @@ -41,6 +42,7 @@ lock_read_timestamp_wait_application lock_read_timestamp_wait_internal lock_read_timestamp_write_count lock_schema_count +lock_schema_wait lock_schema_wait_application lock_schema_wait_internal lock_table_read_count diff --git a/src/third_party/wiredtiger/dist/stat_data.py b/src/third_party/wiredtiger/dist/stat_data.py index f2c61bea4b1..50e7be0039f 100644 --- a/src/third_party/wiredtiger/dist/stat_data.py +++ b/src/third_party/wiredtiger/dist/stat_data.py @@ -296,7 +296,8 @@ connection_stats = [ ########################################## # Cursor operations ########################################## - CursorStat('cursors_cached', 'cursors currently cached', 'no_clear,no_scale'), + CursorStat('cursor_open_count', 'open cursor count', 'no_clear,no_scale'), + CursorStat('cursor_cached_count', 'cached cursor count', 'no_clear,no_scale'), CursorStat('cursor_cache', 'cursor close calls that result in cache'), CursorStat('cursor_create', 'cursor create calls'), CursorStat('cursor_insert', 'cursor insert calls'), @@ -469,7 +470,6 @@ connection_stats = [ ########################################## # Session operations ########################################## - SessionOpStat('session_cursor_open', 'open cursor count', 'no_clear,no_scale'), SessionOpStat('session_open', 'open session count', 'no_clear,no_scale'), SessionOpStat('session_query_ts', 'session query timestamp calls'), SessionOpStat('session_table_alter_fail', 'table alter failed calls', 'no_clear,no_scale'), @@ -698,6 +698,7 @@ dsrc_stats = [ ########################################## # Cursor operations ########################################## + CursorStat('cursor_open_count', 'open cursor count', 'no_clear,no_scale'), CursorStat('cursor_cache', 'close calls that result in cache'), CursorStat('cursor_create', 'create calls'), CursorStat('cursor_insert', 'insert calls'), @@ -756,8 +757,6 @@ dsrc_stats = [ # Session operations ########################################## SessionOpStat('session_compact', 'object compaction'), - SessionOpStat('session_cursors_cached', 'cached cursor count', 'no_clear,no_scale'), - SessionOpStat('session_cursor_open', 'open cursor count', 'no_clear,no_scale'), ########################################## # Transaction statistics diff --git a/src/third_party/wiredtiger/examples/c/ex_all.c b/src/third_party/wiredtiger/examples/c/ex_all.c index 8a5efca4ab3..d9be2299833 100644 --- a/src/third_party/wiredtiger/examples/c/ex_all.c +++ b/src/third_party/wiredtiger/examples/c/ex_all.c @@ -578,6 +578,11 @@ cursor_statistics(WT_SESSION *session) "statistics:table:mytable", NULL, "statistics=(all,clear)", &cursor)); /*! [Statistics cursor clear configuration] */ + + /*! [Statistics cursor session] */ + error_check(session->open_cursor( + session, "statistics:session", NULL, NULL, &cursor)); + /*! [Statistics cursor session] */ } static void diff --git a/src/third_party/wiredtiger/examples/c/ex_stat.c b/src/third_party/wiredtiger/examples/c/ex_stat.c index bc13197038a..fb9e9d07c98 100644 --- a/src/third_party/wiredtiger/examples/c/ex_stat.c +++ b/src/third_party/wiredtiger/examples/c/ex_stat.c @@ -37,6 +37,7 @@ void print_derived_stats(WT_SESSION *); void print_file_stats(WT_SESSION *); void print_join_cursor_stats(WT_SESSION *); void print_overflow_pages(WT_SESSION *); +void print_session_stats(WT_SESSION *); static const char *home; @@ -114,6 +115,20 @@ print_join_cursor_stats(WT_SESSION *session) } void +print_session_stats(WT_SESSION *session) +{ + WT_CURSOR *stat_cursor; + + /*! [statistics session function] */ + error_check(session->open_cursor(session, + "statistics:session", NULL, NULL, &stat_cursor)); + + print_cursor(stat_cursor); + error_check(stat_cursor->close(stat_cursor)); + /*! [statistics session function] */ +} + +void print_overflow_pages(WT_SESSION *session) { /*! [statistics retrieve by key] */ @@ -217,6 +232,8 @@ main(int argc, char *argv[]) print_join_cursor_stats(session); + print_session_stats(session); + print_overflow_pages(session); print_derived_stats(session); diff --git a/src/third_party/wiredtiger/examples/java/com/wiredtiger/examples/ex_all.java b/src/third_party/wiredtiger/examples/java/com/wiredtiger/examples/ex_all.java index 6649e14bd05..6485aeff566 100644 --- a/src/third_party/wiredtiger/examples/java/com/wiredtiger/examples/ex_all.java +++ b/src/third_party/wiredtiger/examples/java/com/wiredtiger/examples/ex_all.java @@ -492,6 +492,11 @@ cursor_statistics(Session session) null, "statistics=(all,clear)"); /*! [Statistics cursor clear configuration] */ + /*! [Statistics cursor session] */ + cursor = session.open_cursor( + "statistics:session", null, null); + /*! [Statistics cursor session] */ + return (true); } diff --git a/src/third_party/wiredtiger/examples/java/com/wiredtiger/examples/ex_stat.java b/src/third_party/wiredtiger/examples/java/com/wiredtiger/examples/ex_stat.java index 9e7cc723bab..fb5082516cc 100644 --- a/src/third_party/wiredtiger/examples/java/com/wiredtiger/examples/ex_stat.java +++ b/src/third_party/wiredtiger/examples/java/com/wiredtiger/examples/ex_stat.java @@ -119,6 +119,22 @@ public class ex_stat { } int + print_session_stats(Session session) + throws WiredTigerException + { + Cursor cursor; + int ret; + + /*! [statistics session function] */ + cursor = session.open_cursor("statistics:session", null, null); + ret = print_cursor(cursor); + ret = cursor.close(); + /*! [statistics session function] */ + + return (ret); + } + + int print_overflow_pages(Session session) throws WiredTigerException { @@ -264,6 +280,8 @@ public class ex_stat { ret = print_join_cursor_stats(session); + ret = print_session_stats(session); + ret = print_overflow_pages(session); ret = print_derived_stats(session); diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index c727e51820e..49a7eb25cc7 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -1,5 +1,5 @@ { - "commit": "fcb59a43a44222716ddae6d94d45cdfd36b915f7", + "commit": "5812c92f5fb2ed3b58d0398e5cbe1cb33380d450", "github": "wiredtiger/wiredtiger.git", "vendor": "wiredtiger", "branch": "mongodb-4.2" diff --git a/src/third_party/wiredtiger/src/btree/bt_io.c b/src/third_party/wiredtiger/src/btree/bt_io.c index 7e7909eed9f..3a5dd48e392 100644 --- a/src/third_party/wiredtiger/src/btree/bt_io.c +++ b/src/third_party/wiredtiger/src/btree/bt_io.c @@ -145,6 +145,7 @@ __wt_bt_read(WT_SESSION_IMPL *session, WT_STAT_DATA_INCR(session, compress_read); WT_STAT_CONN_INCRV(session, cache_bytes_read, dsk->mem_size); WT_STAT_DATA_INCRV(session, cache_bytes_read, dsk->mem_size); + WT_STAT_SESSION_INCRV(session, bytes_read, dsk->mem_size); (void)__wt_atomic_add64( &S2C(session)->cache->bytes_read, dsk->mem_size); @@ -185,7 +186,7 @@ __wt_bt_write(WT_SESSION_IMPL *session, WT_ITEM *buf, WT_KEYED_ENCRYPTOR *kencryptor; WT_PAGE_HEADER *dsk; size_t dst_len, len, result_len, size, src_len; - uint64_t time_start, time_stop; + uint64_t time_diff, time_start, time_stop; uint8_t *dst, *src; int compression_failed; /* Extension API, so not a bool. */ bool data_checksum, encrypted, timer; @@ -388,15 +389,17 @@ __wt_bt_write(WT_SESSION_IMPL *session, WT_ITEM *buf, /* Update some statistics now that the write is done */ if (timer) { time_stop = __wt_clock(session); + time_diff = WT_CLOCKDIFF_US(time_stop, time_start); WT_STAT_CONN_INCR(session, cache_write_app_count); - WT_STAT_CONN_INCRV(session, cache_write_app_time, - WT_CLOCKDIFF_US(time_stop, time_start)); + WT_STAT_CONN_INCRV(session, cache_write_app_time, time_diff); + WT_STAT_SESSION_INCRV(session, write_time, time_diff); } WT_STAT_CONN_INCR(session, cache_write); WT_STAT_DATA_INCR(session, cache_write); WT_STAT_CONN_INCRV(session, cache_bytes_write, dsk->mem_size); WT_STAT_DATA_INCRV(session, cache_bytes_write, dsk->mem_size); + WT_STAT_SESSION_INCRV(session, bytes_write, dsk->mem_size); (void)__wt_atomic_add64( &S2C(session)->cache->bytes_written, dsk->mem_size); diff --git a/src/third_party/wiredtiger/src/btree/bt_read.c b/src/third_party/wiredtiger/src/btree/bt_read.c index f6f7212eb85..dd560b39c0b 100644 --- a/src/third_party/wiredtiger/src/btree/bt_read.c +++ b/src/third_party/wiredtiger/src/btree/bt_read.c @@ -415,7 +415,7 @@ __page_read(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags) WT_ITEM tmp; WT_PAGE *notused; size_t addr_size; - uint64_t time_start, time_stop; + uint64_t time_diff, time_start, time_stop; uint32_t page_flags, final_state, new_state, previous_state; const uint8_t *addr; bool timer; @@ -482,9 +482,10 @@ __page_read(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags) WT_ERR(__wt_bt_read(session, &tmp, addr, addr_size)); if (timer) { time_stop = __wt_clock(session); + time_diff = WT_CLOCKDIFF_US(time_stop, time_start); WT_STAT_CONN_INCR(session, cache_read_app_count); - WT_STAT_CONN_INCRV(session, cache_read_app_time, - WT_CLOCKDIFF_US(time_stop, time_start)); + WT_STAT_CONN_INCRV(session, cache_read_app_time, time_diff); + WT_STAT_SESSION_INCRV(session, read_time, time_diff); } /* diff --git a/src/third_party/wiredtiger/src/conn/conn_handle.c b/src/third_party/wiredtiger/src/conn/conn_handle.c index 42b8510cbd3..e8d5c41b167 100644 --- a/src/third_party/wiredtiger/src/conn/conn_handle.c +++ b/src/third_party/wiredtiger/src/conn/conn_handle.c @@ -57,11 +57,11 @@ __wt_connection_init(WT_CONNECTION_IMPL *conn) WT_RET(__wt_spin_init(session, &conn->fh_lock, "file list")); WT_SPIN_INIT_TRACKED(session, &conn->metadata_lock, metadata); WT_RET(__wt_spin_init(session, &conn->reconfig_lock, "reconfigure")); - WT_SPIN_INIT_TRACKED(session, &conn->schema_lock, schema); + WT_SPIN_INIT_SESSION_TRACKED(session, &conn->schema_lock, schema); WT_RET(__wt_spin_init(session, &conn->turtle_lock, "turtle file")); /* Read-write locks */ - WT_RWLOCK_INIT_TRACKED(session, &conn->dhandle_lock, dhandle); + WT_RWLOCK_INIT_SESSION_TRACKED(session, &conn->dhandle_lock, dhandle); WT_RET(__wt_rwlock_init(session, &conn->hot_backup_lock)); WT_RWLOCK_INIT_TRACKED(session, &conn->table_lock, table); diff --git a/src/third_party/wiredtiger/src/conn/conn_stat.c b/src/third_party/wiredtiger/src/conn/conn_stat.c index ffbc1caf2bb..741ea80f1cb 100644 --- a/src/third_party/wiredtiger/src/conn/conn_stat.c +++ b/src/third_party/wiredtiger/src/conn/conn_stat.c @@ -80,7 +80,7 @@ __wt_conn_stat_init(WT_SESSION_IMPL *session) WT_STAT_SET(session, stats, file_open, conn->open_file_count); WT_STAT_SET(session, - stats, session_cursor_open, conn->open_cursor_count); + stats, cursor_open_count, conn->open_cursor_count); WT_STAT_SET(session, stats, dh_conn_handle_count, conn->dhandle_count); WT_STAT_SET(session, stats, rec_split_stashed_objects, conn->stashed_objects); diff --git a/src/third_party/wiredtiger/src/cursor/cur_backup.c b/src/third_party/wiredtiger/src/cursor/cur_backup.c index ebb2e30a4dd..91bf3bb4931 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_backup.c +++ b/src/third_party/wiredtiger/src/cursor/cur_backup.c @@ -13,9 +13,10 @@ static int __backup_list_append( WT_SESSION_IMPL *, WT_CURSOR_BACKUP *, const char *); static int __backup_list_uri_append(WT_SESSION_IMPL *, const char *, bool *); static int __backup_start( - WT_SESSION_IMPL *, WT_CURSOR_BACKUP *, WT_CURSOR *, const char *[]); + WT_SESSION_IMPL *, WT_CURSOR_BACKUP *, bool, const char *[]); static int __backup_stop(WT_SESSION_IMPL *, WT_CURSOR_BACKUP *); -static int __backup_uri(WT_SESSION_IMPL *, const char *[], bool *, bool *); +static int __backup_uri( + WT_SESSION_IMPL *, const char *[], bool, bool *, bool *); /* * __curbackup_next -- @@ -171,7 +172,7 @@ __wt_curbackup_open(WT_SESSION_IMPL *session, const char *uri, */ WT_WITH_CHECKPOINT_LOCK(session, WT_WITH_SCHEMA_LOCK(session, - ret = __backup_start(session, cb, other, cfg))); + ret = __backup_start(session, cb, other != NULL, cfg))); WT_ERR(ret); WT_ERR(__wt_cursor_init(cursor, uri, NULL, cfg, cursorp)); @@ -217,7 +218,7 @@ err: WT_TRET(__wt_fs_directory_list_free(session, &logfiles, logcount)); */ static int __backup_start(WT_SESSION_IMPL *session, - WT_CURSOR_BACKUP *cb, WT_CURSOR *other, const char *cfg[]) + WT_CURSOR_BACKUP *cb, bool is_dup, const char *cfg[]) { WT_CONNECTION_IMPL *conn; WT_DECL_RET; @@ -239,15 +240,15 @@ __backup_start(WT_SESSION_IMPL *session, * Single thread hot backups: we're holding the schema lock, so we * know we'll serialize with other attempts to start a hot backup. */ - if (conn->hot_backup && other == NULL) + if (conn->hot_backup && !is_dup) WT_RET_MSG( session, EINVAL, "there is already a backup cursor open"); - if (F_ISSET(session, WT_SESSION_BACKUP_DUP) && other != NULL) + if (F_ISSET(session, WT_SESSION_BACKUP_DUP) && is_dup) WT_RET_MSG(session, EINVAL, "there is already a duplicate backup cursor open"); - if (other == NULL) { + if (!is_dup) { /* * The hot backup copy is done outside of WiredTiger, which * means file blocks can't be freed and re-allocated until the @@ -290,12 +291,13 @@ __backup_start(WT_SESSION_IMPL *session, * full backup, add all database objects and log files to the list. */ target_list = false; - WT_ERR(__backup_uri(session, cfg, &target_list, &log_only)); + WT_ERR(__backup_uri(session, + cfg, is_dup, &target_list, &log_only)); /* * For a duplicate cursor, all the work is done in backup_uri. The only * usage accepted is "target=("log:")" so error if not log only. */ - if (other != NULL) { + if (is_dup) { if (!log_only) WT_ERR_MSG(session, EINVAL, "duplicate backup cursor must be for logs only."); @@ -424,8 +426,8 @@ __backup_all(WT_SESSION_IMPL *session) * Backup a list of objects. */ static int -__backup_uri(WT_SESSION_IMPL *session, - const char *cfg[], bool *foundp, bool *log_only) +__backup_uri(WT_SESSION_IMPL *session, const char *cfg[], + bool is_dup, bool *foundp, bool *log_only) { WT_CONFIG targetconf; WT_CONFIG_ITEM cval, k, v; @@ -466,9 +468,10 @@ __backup_uri(WT_SESSION_IMPL *session, if (WT_PREFIX_MATCH(uri, "log:")) { /* * Log archive cannot mix with incremental backup, don't - * let that happen. + * let that happen. If we're a duplicate cursor + * archiving is already temporarily suspended. */ - if (FLD_ISSET( + if (!is_dup && FLD_ISSET( S2C(session)->log_flags, WT_CONN_LOG_ARCHIVE)) WT_ERR_MSG(session, EINVAL, "incremental backup not possible when " diff --git a/src/third_party/wiredtiger/src/cursor/cur_stat.c b/src/third_party/wiredtiger/src/cursor/cur_stat.c index a65043b138a..d49830c19b2 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_stat.c +++ b/src/third_party/wiredtiger/src/cursor/cur_stat.c @@ -270,6 +270,10 @@ __curstat_reset(WT_CURSOR *cursor) cst->notinitialized = cst->notpositioned = true; F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); + /* Reset the session statistics to zero. */ + if (strcmp(cursor->uri, "statistics:session") == 0) + __wt_stat_session_clear_single(&session->stats); + err: API_END_RET(session, ret); } @@ -520,15 +524,6 @@ __curstat_join_init(WT_SESSION_IMPL *session, static void __curstat_session_init(WT_SESSION_IMPL *session, WT_CURSOR_STAT *cst) { - /* This is a stub at the moment, initialize the session stats to 0. */ - session->stats.bytes_read = 0; - session->stats.bytes_write = 0; - session->stats.cache_time = 0; - session->stats.lock_dhandle_wait = 0; - session->stats.lock_schema_wait = 0; - session->stats.read_time = 0; - session->stats.write_time = 0; - /* * Copy stats from the session to the cursor. Optionally clear the * session's statistics. diff --git a/src/third_party/wiredtiger/src/cursor/cur_std.c b/src/third_party/wiredtiger/src/cursor/cur_std.c index 3217b55cfa4..bef7dc615a9 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_std.c +++ b/src/third_party/wiredtiger/src/cursor/cur_std.c @@ -620,9 +620,8 @@ __wt_cursor_cache(WT_CURSOR *cursor, WT_DATA_HANDLE *dhandle) TAILQ_INSERT_HEAD(&session->cursor_cache[bucket], cursor, q); (void)__wt_atomic_sub32(&S2C(session)->open_cursor_count, 1); - WT_STAT_DATA_DECR(session, session_cursor_open); - WT_STAT_DATA_INCR(session, session_cursors_cached); - WT_STAT_CONN_INCR(session, cursors_cached); + WT_STAT_CONN_INCR_ATOMIC(session, cursor_cached_count); + WT_STAT_DATA_DECR(session, cursor_open_count); F_SET(cursor, WT_CURSTD_CACHED); return (ret); } @@ -646,9 +645,8 @@ __wt_cursor_reopen(WT_CURSOR *cursor, WT_DATA_HANDLE *dhandle) WT_DHANDLE_RELEASE(dhandle); } (void)__wt_atomic_add32(&S2C(session)->open_cursor_count, 1); - WT_STAT_DATA_INCR(session, session_cursor_open); - WT_STAT_DATA_DECR(session, session_cursors_cached); - WT_STAT_CONN_DECR(session, cursors_cached); + WT_STAT_CONN_DECR_ATOMIC(session, cursor_cached_count); + WT_STAT_DATA_INCR(session, cursor_open_count); bucket = cursor->uri_hash % WT_HASH_ARRAY_SIZE; TAILQ_REMOVE(&session->cursor_cache[bucket], cursor, q); @@ -870,7 +868,7 @@ __wt_cursor_close(WT_CURSOR *cursor) TAILQ_REMOVE(&session->cursors, cursor, q); (void)__wt_atomic_sub32(&S2C(session)->open_cursor_count, 1); - WT_STAT_DATA_DECR(session, session_cursor_open); + WT_STAT_DATA_DECR(session, cursor_open_count); } __wt_buf_free(session, &cursor->key); __wt_buf_free(session, &cursor->value); @@ -1142,7 +1140,7 @@ __wt_cursor_init(WT_CURSOR *cursor, F_SET(cursor, WT_CURSTD_OPEN); (void)__wt_atomic_add32(&S2C(session)->open_cursor_count, 1); - WT_STAT_DATA_INCR(session, session_cursor_open); + WT_STAT_DATA_INCR(session, cursor_open_count); *cursorp = (cdump != NULL) ? cdump : cursor; return (0); diff --git a/src/third_party/wiredtiger/src/docs/data-sources.dox b/src/third_party/wiredtiger/src/docs/data-sources.dox index 7f1879e0ffe..aaee4adc04a 100644 --- a/src/third_party/wiredtiger/src/docs/data-sources.dox +++ b/src/third_party/wiredtiger/src/docs/data-sources.dox @@ -38,7 +38,7 @@ cursor types that give access to data managed by WiredTiger: key=<code>string</code>\, value=<code>string</code>\,<br> see @ref metadata for details} @row{<tt>statistics:[\<data source URI\>]</tt>, - database, data source or join statistics cursor, + database, data source, join or session statistics cursor, key=<code>int id</code>\,<br> value=<code>(string description\, string value\, uint64_t value)</code>\,<br> @@ -106,9 +106,10 @@ WiredTiger database as well as statistics for individual data sources. The statistics are at two levels: per-database and per-individual data source. Database-wide statistics are retrieved with the \c "statistics:" URI; individual data source statistics are available by specifying -\c "statistics:<data source URI>". Additionally, statistics about a -join cursor can be retrieved by specifying \c "statistics:join" and -supplying the join cursor as an argument in the SESSION::open_cursor call. +\c "statistics:<data source URI>". Statistics about a join cursor can be +retrieved by specifying \c "statistics:join" and supplying the join cursor as an +argument in the SESSION::open_cursor call. Statistics about a session can be +retrieved by specifying \c "statistics:session". The statistic key is an integer from the list of keys in @ref_single statistics_keys "Statistics Keys". Statistics cursors return @@ -118,7 +119,8 @@ unsigned 64-bit integral value, respectively. The cursor's statistics values are loaded when the cursor is opened and remain unchanged for the life of the cursor, unless the cursor is reset -by calling the WT_CURSOR::reset method, which reloads the values. +by calling the WT_CURSOR::reset method, which reloads the values. A reset call +on session statistics cursor would set the statistics values to zero. The following is an example of printing run-time statistics about the WiredTiger engine: @@ -133,7 +135,11 @@ The following is an example of printing statistics about a join cursor: @snippet ex_stat.c statistics join cursor function -These three examples can use a common display routine that iterates through the +The following is an example of printing statistics about a session cursor: + +@snippet ex_stat.c statistics session function + +These four examples can use a common display routine that iterates through the statistics until the cursor returns the end of the list. @snippet ex_stat.c statistics display function diff --git a/src/third_party/wiredtiger/src/docs/statistics.dox b/src/third_party/wiredtiger/src/docs/statistics.dox index 19b7b17257b..3735a009d56 100644 --- a/src/third_party/wiredtiger/src/docs/statistics.dox +++ b/src/third_party/wiredtiger/src/docs/statistics.dox @@ -74,8 +74,8 @@ when the cursor was opened), as well as statistics logging, when it is configured. The following example configures WiredTiger to maintain only relatively -inexpensive statistics, and to clears statistics after they are gathered -or logged: +inexpensive statistics, and to clear statistics after they are gathered or +logged: @snippet ex_all.c Statistics clear configuration @@ -88,6 +88,14 @@ in the join (see WT_SESSION::join); the uri of each reference cursor appears as a prefix in the description field returned as a value by the statistics cursor. +The following example opens a statistics cursor on an open WiredTiger session: + +@snippet ex_all.c Statistics cursor session + +A reset call on this cursor would reset the session statistics values to zero, +with or without the statistics configuration \c clear being specified to the +WT_SESSION::open_cursor method. + @section statistics_log Statistics logging WiredTiger will optionally log database statistics into files when the diff --git a/src/third_party/wiredtiger/src/evict/evict_lru.c b/src/third_party/wiredtiger/src/evict/evict_lru.c index 776c10d03e9..c7cff1bc54f 100644 --- a/src/third_party/wiredtiger/src/evict/evict_lru.c +++ b/src/third_party/wiredtiger/src/evict/evict_lru.c @@ -2444,6 +2444,7 @@ err: if (timer) { time_stop = __wt_clock(session); elapsed = WT_CLOCKDIFF_US(time_stop, time_start); WT_STAT_CONN_INCRV(session, application_cache_time, elapsed); + WT_STAT_SESSION_INCRV(session, cache_time, elapsed); session->cache_wait_us += elapsed; if (cache->cache_max_wait_us != 0 && session->cache_wait_us > cache->cache_max_wait_us) { diff --git a/src/third_party/wiredtiger/src/include/mutex.h b/src/third_party/wiredtiger/src/include/mutex.h index ba32d166f03..5c836526a21 100644 --- a/src/third_party/wiredtiger/src/include/mutex.h +++ b/src/third_party/wiredtiger/src/include/mutex.h @@ -53,6 +53,7 @@ struct __wt_rwlock { /* Read/write lock */ int16_t stat_write_count_off; /* write acquisitions offset */ int16_t stat_app_usecs_off; /* waiting application threads offset */ int16_t stat_int_usecs_off; /* waiting server threads offset */ + int16_t stat_session_usecs_off; /* waiting session offset */ WT_CONDVAR *cond_readers; /* Blocking readers */ WT_CONDVAR *cond_writers; /* Blocking writers */ @@ -65,16 +66,23 @@ struct __wt_rwlock { /* Read/write lock */ * Implemented as a macro so we can pass in a statistics field and convert * it into a statistics structure array offset. */ -#define WT_RWLOCK_INIT_TRACKED(session, l, name) do { \ - WT_RET(__wt_rwlock_init(session, l)); \ - (l)->stat_read_count_off = (int16_t)WT_STATS_FIELD_TO_OFFSET( \ - S2C(session)->stats, lock_##name##_read_count); \ - (l)->stat_write_count_off = (int16_t)WT_STATS_FIELD_TO_OFFSET( \ - S2C(session)->stats, lock_##name##_write_count); \ - (l)->stat_app_usecs_off = (int16_t)WT_STATS_FIELD_TO_OFFSET( \ - S2C(session)->stats, lock_##name##_wait_application); \ - (l)->stat_int_usecs_off = (int16_t)WT_STATS_FIELD_TO_OFFSET( \ - S2C(session)->stats, lock_##name##_wait_internal); \ +#define WT_RWLOCK_INIT_TRACKED(session, l, name) do { \ + WT_RET(__wt_rwlock_init(session, l)); \ + (l)->stat_read_count_off = (int16_t)WT_STATS_FIELD_TO_OFFSET( \ + S2C(session)->stats, lock_##name##_read_count); \ + (l)->stat_write_count_off = (int16_t)WT_STATS_FIELD_TO_OFFSET( \ + S2C(session)->stats, lock_##name##_write_count); \ + (l)->stat_app_usecs_off = (int16_t)WT_STATS_FIELD_TO_OFFSET( \ + S2C(session)->stats, lock_##name##_wait_application); \ + (l)->stat_int_usecs_off = (int16_t)WT_STATS_FIELD_TO_OFFSET( \ + S2C(session)->stats, lock_##name##_wait_internal); \ +} while (0) + +#define WT_RWLOCK_INIT_SESSION_TRACKED(session, l, name) do { \ + WT_RWLOCK_INIT_TRACKED(session, l, name); \ + (l)->stat_session_usecs_off = \ + (int16_t)WT_SESSION_STATS_FIELD_TO_OFFSET( \ + &(session->stats), lock_##name##_wait); \ } while (0) /* @@ -112,6 +120,7 @@ struct __wt_spinlock { int16_t stat_count_off; /* acquisitions offset */ int16_t stat_app_usecs_off; /* waiting application threads offset */ int16_t stat_int_usecs_off; /* waiting server threads offset */ + int16_t stat_session_usecs_off; /* waiting session offset */ int8_t initialized; /* Lock initialized, for cleanup */ diff --git a/src/third_party/wiredtiger/src/include/mutex.i b/src/third_party/wiredtiger/src/include/mutex.i index 8a2699f031d..d7f715e81e1 100644 --- a/src/third_party/wiredtiger/src/include/mutex.i +++ b/src/third_party/wiredtiger/src/include/mutex.i @@ -286,6 +286,13 @@ __wt_spin_unlock(WT_SESSION_IMPL *session, WT_SPINLOCK *t) S2C(session)->stats, lock_##name##_wait_internal); \ } while (0) +#define WT_SPIN_INIT_SESSION_TRACKED(session, t, name) do { \ + WT_SPIN_INIT_TRACKED(session, t, name); \ + (t)->stat_session_usecs_off = \ + (int16_t)WT_SESSION_STATS_FIELD_TO_OFFSET( \ + &(session->stats), lock_##name##_wait); \ +} while (0) + /* * __wt_spin_lock_track -- * Spinlock acquisition, with tracking. @@ -293,21 +300,25 @@ __wt_spin_unlock(WT_SESSION_IMPL *session, WT_SPINLOCK *t) static inline void __wt_spin_lock_track(WT_SESSION_IMPL *session, WT_SPINLOCK *t) { - uint64_t time_start, time_stop; - int64_t **stats; + uint64_t time_diff, time_start, time_stop; + int64_t *session_stats, **stats; if (t->stat_count_off != -1 && WT_STAT_ENABLED(session)) { time_start = __wt_clock(session); __wt_spin_lock(session, t); time_stop = __wt_clock(session); + time_diff = WT_CLOCKDIFF_US(time_stop, time_start); stats = (int64_t **)S2C(session)->stats; + session_stats = (int64_t *)&(session->stats); stats[session->stat_bucket][t->stat_count_off]++; if (F_ISSET(session, WT_SESSION_INTERNAL)) stats[session->stat_bucket][t->stat_int_usecs_off] += - (int64_t)WT_CLOCKDIFF_US(time_stop, time_start); - else + (int64_t)time_diff; + else { stats[session->stat_bucket][t->stat_app_usecs_off] += - (int64_t)WT_CLOCKDIFF_US(time_stop, time_start); + (int64_t)time_diff; + } + session_stats[t->stat_session_usecs_off] += (int64_t)time_diff; } else __wt_spin_lock(session, t); } diff --git a/src/third_party/wiredtiger/src/include/stat.h b/src/third_party/wiredtiger/src/include/stat.h index 9ac1e6c619a..c54a9fcc8b0 100644 --- a/src/third_party/wiredtiger/src/include/stat.h +++ b/src/third_party/wiredtiger/src/include/stat.h @@ -84,6 +84,9 @@ #define WT_STATS_FIELD_TO_OFFSET(stats, fld) \ (int)(&(stats)[0]->fld - (int64_t *)(stats)[0]) +#define WT_SESSION_STATS_FIELD_TO_OFFSET(stats, fld) \ + (int)(&(stats)->fld - (int64_t *)(stats)) + /* AUTOMATIC FLAG VALUE GENERATION START */ #define WT_STAT_CLEAR 0x01u #define WT_STAT_JSON 0x02u @@ -259,6 +262,12 @@ __wt_stats_clear(void *stats_arg, int slot) } while (0) /* + * Update per session statistics. + */ +#define WT_STAT_SESSION_INCRV(session, fld, value) \ + WT_STAT_INCRV_BASE(session, &session->stats, fld, value) + +/* * Construct histogram increment functions to put the passed value into the * right bucket. Bucket ranges, represented by various statistics, depend upon * whether the passed value is in milliseconds or microseconds. Also values @@ -455,6 +464,7 @@ struct __wt_connection_stats { int64_t fsync_io; int64_t read_io; int64_t write_io; + int64_t cursor_cached_count; int64_t cursor_cache; int64_t cursor_create; int64_t cursor_insert; @@ -472,8 +482,8 @@ struct __wt_connection_stats { int64_t cursor_sweep_examined; int64_t cursor_sweep; int64_t cursor_update; - int64_t cursors_cached; int64_t cursor_reopen; + int64_t cursor_open_count; int64_t cursor_truncate; int64_t dh_conn_handle_count; int64_t dh_sweep_ref; @@ -586,7 +596,6 @@ struct __wt_connection_stats { int64_t rec_page_delete; int64_t rec_split_stashed_bytes; int64_t rec_split_stashed_objects; - int64_t session_cursor_open; int64_t session_open; int64_t session_query_ts; int64_t session_table_alter_fail; @@ -802,6 +811,7 @@ struct __wt_dsrc_stats { int64_t cursor_insert; int64_t cursor_modify; int64_t cursor_next; + int64_t cursor_open_count; int64_t cursor_prev; int64_t cursor_remove; int64_t cursor_reserve; @@ -824,9 +834,7 @@ struct __wt_dsrc_stats { int64_t rec_pages; int64_t rec_pages_eviction; int64_t rec_page_delete; - int64_t session_cursors_cached; int64_t session_compact; - int64_t session_cursor_open; int64_t txn_update_conflict; }; diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index bf96e953ec1..33d0b1ed74d 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -361,7 +361,8 @@ struct __wt_cursor { * with WT_CURSOR::prev will move to the last record. * * In the case of a statistics cursor, resetting the cursor refreshes - * the statistics information returned. + * the statistics information returned. Resetting a session statistics + * cursor resets all the session statistics values to zero. * * @snippet ex_all.c Reset the cursor * @@ -5232,282 +5233,282 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_CONN_READ_IO 1140 /*! connection: total write I/Os */ #define WT_STAT_CONN_WRITE_IO 1141 +/*! cursor: cached cursor count */ +#define WT_STAT_CONN_CURSOR_CACHED_COUNT 1142 /*! cursor: cursor close calls that result in cache */ -#define WT_STAT_CONN_CURSOR_CACHE 1142 +#define WT_STAT_CONN_CURSOR_CACHE 1143 /*! cursor: cursor create calls */ -#define WT_STAT_CONN_CURSOR_CREATE 1143 +#define WT_STAT_CONN_CURSOR_CREATE 1144 /*! cursor: cursor insert calls */ -#define WT_STAT_CONN_CURSOR_INSERT 1144 +#define WT_STAT_CONN_CURSOR_INSERT 1145 /*! cursor: cursor modify calls */ -#define WT_STAT_CONN_CURSOR_MODIFY 1145 +#define WT_STAT_CONN_CURSOR_MODIFY 1146 /*! cursor: cursor next calls */ -#define WT_STAT_CONN_CURSOR_NEXT 1146 +#define WT_STAT_CONN_CURSOR_NEXT 1147 /*! cursor: cursor operation restarted */ -#define WT_STAT_CONN_CURSOR_RESTART 1147 +#define WT_STAT_CONN_CURSOR_RESTART 1148 /*! cursor: cursor prev calls */ -#define WT_STAT_CONN_CURSOR_PREV 1148 +#define WT_STAT_CONN_CURSOR_PREV 1149 /*! cursor: cursor remove calls */ -#define WT_STAT_CONN_CURSOR_REMOVE 1149 +#define WT_STAT_CONN_CURSOR_REMOVE 1150 /*! cursor: cursor reserve calls */ -#define WT_STAT_CONN_CURSOR_RESERVE 1150 +#define WT_STAT_CONN_CURSOR_RESERVE 1151 /*! cursor: cursor reset calls */ -#define WT_STAT_CONN_CURSOR_RESET 1151 +#define WT_STAT_CONN_CURSOR_RESET 1152 /*! cursor: cursor search calls */ -#define WT_STAT_CONN_CURSOR_SEARCH 1152 +#define WT_STAT_CONN_CURSOR_SEARCH 1153 /*! cursor: cursor search near calls */ -#define WT_STAT_CONN_CURSOR_SEARCH_NEAR 1153 +#define WT_STAT_CONN_CURSOR_SEARCH_NEAR 1154 /*! cursor: cursor sweep buckets */ -#define WT_STAT_CONN_CURSOR_SWEEP_BUCKETS 1154 +#define WT_STAT_CONN_CURSOR_SWEEP_BUCKETS 1155 /*! cursor: cursor sweep cursors closed */ -#define WT_STAT_CONN_CURSOR_SWEEP_CLOSED 1155 +#define WT_STAT_CONN_CURSOR_SWEEP_CLOSED 1156 /*! cursor: cursor sweep cursors examined */ -#define WT_STAT_CONN_CURSOR_SWEEP_EXAMINED 1156 +#define WT_STAT_CONN_CURSOR_SWEEP_EXAMINED 1157 /*! cursor: cursor sweeps */ -#define WT_STAT_CONN_CURSOR_SWEEP 1157 +#define WT_STAT_CONN_CURSOR_SWEEP 1158 /*! cursor: cursor update calls */ -#define WT_STAT_CONN_CURSOR_UPDATE 1158 -/*! cursor: cursors currently cached */ -#define WT_STAT_CONN_CURSORS_CACHED 1159 +#define WT_STAT_CONN_CURSOR_UPDATE 1159 /*! cursor: cursors reused from cache */ #define WT_STAT_CONN_CURSOR_REOPEN 1160 +/*! cursor: open cursor count */ +#define WT_STAT_CONN_CURSOR_OPEN_COUNT 1161 /*! cursor: truncate calls */ -#define WT_STAT_CONN_CURSOR_TRUNCATE 1161 +#define WT_STAT_CONN_CURSOR_TRUNCATE 1162 /*! data-handle: connection data handles currently active */ -#define WT_STAT_CONN_DH_CONN_HANDLE_COUNT 1162 +#define WT_STAT_CONN_DH_CONN_HANDLE_COUNT 1163 /*! data-handle: connection sweep candidate became referenced */ -#define WT_STAT_CONN_DH_SWEEP_REF 1163 +#define WT_STAT_CONN_DH_SWEEP_REF 1164 /*! data-handle: connection sweep dhandles closed */ -#define WT_STAT_CONN_DH_SWEEP_CLOSE 1164 +#define WT_STAT_CONN_DH_SWEEP_CLOSE 1165 /*! data-handle: connection sweep dhandles removed from hash list */ -#define WT_STAT_CONN_DH_SWEEP_REMOVE 1165 +#define WT_STAT_CONN_DH_SWEEP_REMOVE 1166 /*! data-handle: connection sweep time-of-death sets */ -#define WT_STAT_CONN_DH_SWEEP_TOD 1166 +#define WT_STAT_CONN_DH_SWEEP_TOD 1167 /*! data-handle: connection sweeps */ -#define WT_STAT_CONN_DH_SWEEPS 1167 +#define WT_STAT_CONN_DH_SWEEPS 1168 /*! data-handle: session dhandles swept */ -#define WT_STAT_CONN_DH_SESSION_HANDLES 1168 +#define WT_STAT_CONN_DH_SESSION_HANDLES 1169 /*! data-handle: session sweep attempts */ -#define WT_STAT_CONN_DH_SESSION_SWEEPS 1169 +#define WT_STAT_CONN_DH_SESSION_SWEEPS 1170 /*! lock: checkpoint lock acquisitions */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_COUNT 1170 +#define WT_STAT_CONN_LOCK_CHECKPOINT_COUNT 1171 /*! lock: checkpoint lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_APPLICATION 1171 +#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_APPLICATION 1172 /*! lock: checkpoint lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_INTERNAL 1172 +#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_INTERNAL 1173 /*! * lock: commit timestamp queue lock application thread time waiting * (usecs) */ -#define WT_STAT_CONN_LOCK_COMMIT_TIMESTAMP_WAIT_APPLICATION 1173 +#define WT_STAT_CONN_LOCK_COMMIT_TIMESTAMP_WAIT_APPLICATION 1174 /*! lock: commit timestamp queue lock internal thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_COMMIT_TIMESTAMP_WAIT_INTERNAL 1174 +#define WT_STAT_CONN_LOCK_COMMIT_TIMESTAMP_WAIT_INTERNAL 1175 /*! lock: commit timestamp queue read lock acquisitions */ -#define WT_STAT_CONN_LOCK_COMMIT_TIMESTAMP_READ_COUNT 1175 +#define WT_STAT_CONN_LOCK_COMMIT_TIMESTAMP_READ_COUNT 1176 /*! lock: commit timestamp queue write lock acquisitions */ -#define WT_STAT_CONN_LOCK_COMMIT_TIMESTAMP_WRITE_COUNT 1176 +#define WT_STAT_CONN_LOCK_COMMIT_TIMESTAMP_WRITE_COUNT 1177 /*! lock: dhandle lock application thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_APPLICATION 1177 +#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_APPLICATION 1178 /*! lock: dhandle lock internal thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_INTERNAL 1178 +#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_INTERNAL 1179 /*! lock: dhandle read lock acquisitions */ -#define WT_STAT_CONN_LOCK_DHANDLE_READ_COUNT 1179 +#define WT_STAT_CONN_LOCK_DHANDLE_READ_COUNT 1180 /*! lock: dhandle write lock acquisitions */ -#define WT_STAT_CONN_LOCK_DHANDLE_WRITE_COUNT 1180 +#define WT_STAT_CONN_LOCK_DHANDLE_WRITE_COUNT 1181 /*! lock: metadata lock acquisitions */ -#define WT_STAT_CONN_LOCK_METADATA_COUNT 1181 +#define WT_STAT_CONN_LOCK_METADATA_COUNT 1182 /*! lock: metadata lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_METADATA_WAIT_APPLICATION 1182 +#define WT_STAT_CONN_LOCK_METADATA_WAIT_APPLICATION 1183 /*! lock: metadata lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_METADATA_WAIT_INTERNAL 1183 +#define WT_STAT_CONN_LOCK_METADATA_WAIT_INTERNAL 1184 /*! * lock: read timestamp queue lock application thread time waiting * (usecs) */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_APPLICATION 1184 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_APPLICATION 1185 /*! lock: read timestamp queue lock internal thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_INTERNAL 1185 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_INTERNAL 1186 /*! lock: read timestamp queue read lock acquisitions */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_READ_COUNT 1186 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_READ_COUNT 1187 /*! lock: read timestamp queue write lock acquisitions */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WRITE_COUNT 1187 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WRITE_COUNT 1188 /*! lock: schema lock acquisitions */ -#define WT_STAT_CONN_LOCK_SCHEMA_COUNT 1188 +#define WT_STAT_CONN_LOCK_SCHEMA_COUNT 1189 /*! lock: schema lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_APPLICATION 1189 +#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_APPLICATION 1190 /*! lock: schema lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_INTERNAL 1190 +#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_INTERNAL 1191 /*! * lock: table lock application thread time waiting for the table lock * (usecs) */ -#define WT_STAT_CONN_LOCK_TABLE_WAIT_APPLICATION 1191 +#define WT_STAT_CONN_LOCK_TABLE_WAIT_APPLICATION 1192 /*! * lock: table lock internal thread time waiting for the table lock * (usecs) */ -#define WT_STAT_CONN_LOCK_TABLE_WAIT_INTERNAL 1192 +#define WT_STAT_CONN_LOCK_TABLE_WAIT_INTERNAL 1193 /*! lock: table read lock acquisitions */ -#define WT_STAT_CONN_LOCK_TABLE_READ_COUNT 1193 +#define WT_STAT_CONN_LOCK_TABLE_READ_COUNT 1194 /*! lock: table write lock acquisitions */ -#define WT_STAT_CONN_LOCK_TABLE_WRITE_COUNT 1194 +#define WT_STAT_CONN_LOCK_TABLE_WRITE_COUNT 1195 /*! lock: txn global lock application thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_APPLICATION 1195 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_APPLICATION 1196 /*! lock: txn global lock internal thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_INTERNAL 1196 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_INTERNAL 1197 /*! lock: txn global read lock acquisitions */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_READ_COUNT 1197 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_READ_COUNT 1198 /*! lock: txn global write lock acquisitions */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WRITE_COUNT 1198 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WRITE_COUNT 1199 /*! log: busy returns attempting to switch slots */ -#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1199 +#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1200 /*! log: force archive time sleeping (usecs) */ -#define WT_STAT_CONN_LOG_FORCE_ARCHIVE_SLEEP 1200 +#define WT_STAT_CONN_LOG_FORCE_ARCHIVE_SLEEP 1201 /*! log: log bytes of payload data */ -#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1201 +#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1202 /*! log: log bytes written */ -#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1202 +#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1203 /*! log: log files manually zero-filled */ -#define WT_STAT_CONN_LOG_ZERO_FILLS 1203 +#define WT_STAT_CONN_LOG_ZERO_FILLS 1204 /*! log: log flush operations */ -#define WT_STAT_CONN_LOG_FLUSH 1204 +#define WT_STAT_CONN_LOG_FLUSH 1205 /*! log: log force write operations */ -#define WT_STAT_CONN_LOG_FORCE_WRITE 1205 +#define WT_STAT_CONN_LOG_FORCE_WRITE 1206 /*! log: log force write operations skipped */ -#define WT_STAT_CONN_LOG_FORCE_WRITE_SKIP 1206 +#define WT_STAT_CONN_LOG_FORCE_WRITE_SKIP 1207 /*! log: log records compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1207 +#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1208 /*! log: log records not compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1208 +#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1209 /*! log: log records too small to compress */ -#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1209 +#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1210 /*! log: log release advances write LSN */ -#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1210 +#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1211 /*! log: log scan operations */ -#define WT_STAT_CONN_LOG_SCANS 1211 +#define WT_STAT_CONN_LOG_SCANS 1212 /*! log: log scan records requiring two reads */ -#define WT_STAT_CONN_LOG_SCAN_REREADS 1212 +#define WT_STAT_CONN_LOG_SCAN_REREADS 1213 /*! log: log server thread advances write LSN */ -#define WT_STAT_CONN_LOG_WRITE_LSN 1213 +#define WT_STAT_CONN_LOG_WRITE_LSN 1214 /*! log: log server thread write LSN walk skipped */ -#define WT_STAT_CONN_LOG_WRITE_LSN_SKIP 1214 +#define WT_STAT_CONN_LOG_WRITE_LSN_SKIP 1215 /*! log: log sync operations */ -#define WT_STAT_CONN_LOG_SYNC 1215 +#define WT_STAT_CONN_LOG_SYNC 1216 /*! log: log sync time duration (usecs) */ -#define WT_STAT_CONN_LOG_SYNC_DURATION 1216 +#define WT_STAT_CONN_LOG_SYNC_DURATION 1217 /*! log: log sync_dir operations */ -#define WT_STAT_CONN_LOG_SYNC_DIR 1217 +#define WT_STAT_CONN_LOG_SYNC_DIR 1218 /*! log: log sync_dir time duration (usecs) */ -#define WT_STAT_CONN_LOG_SYNC_DIR_DURATION 1218 +#define WT_STAT_CONN_LOG_SYNC_DIR_DURATION 1219 /*! log: log write operations */ -#define WT_STAT_CONN_LOG_WRITES 1219 +#define WT_STAT_CONN_LOG_WRITES 1220 /*! log: logging bytes consolidated */ -#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1220 +#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1221 /*! log: maximum log file size */ -#define WT_STAT_CONN_LOG_MAX_FILESIZE 1221 +#define WT_STAT_CONN_LOG_MAX_FILESIZE 1222 /*! log: number of pre-allocated log files to create */ -#define WT_STAT_CONN_LOG_PREALLOC_MAX 1222 +#define WT_STAT_CONN_LOG_PREALLOC_MAX 1223 /*! log: pre-allocated log files not ready and missed */ -#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1223 +#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1224 /*! log: pre-allocated log files prepared */ -#define WT_STAT_CONN_LOG_PREALLOC_FILES 1224 +#define WT_STAT_CONN_LOG_PREALLOC_FILES 1225 /*! log: pre-allocated log files used */ -#define WT_STAT_CONN_LOG_PREALLOC_USED 1225 +#define WT_STAT_CONN_LOG_PREALLOC_USED 1226 /*! log: records processed by log scan */ -#define WT_STAT_CONN_LOG_SCAN_RECORDS 1226 +#define WT_STAT_CONN_LOG_SCAN_RECORDS 1227 /*! log: slot close lost race */ -#define WT_STAT_CONN_LOG_SLOT_CLOSE_RACE 1227 +#define WT_STAT_CONN_LOG_SLOT_CLOSE_RACE 1228 /*! log: slot close unbuffered waits */ -#define WT_STAT_CONN_LOG_SLOT_CLOSE_UNBUF 1228 +#define WT_STAT_CONN_LOG_SLOT_CLOSE_UNBUF 1229 /*! log: slot closures */ -#define WT_STAT_CONN_LOG_SLOT_CLOSES 1229 +#define WT_STAT_CONN_LOG_SLOT_CLOSES 1230 /*! log: slot join atomic update races */ -#define WT_STAT_CONN_LOG_SLOT_RACES 1230 +#define WT_STAT_CONN_LOG_SLOT_RACES 1231 /*! log: slot join calls atomic updates raced */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_RACE 1231 +#define WT_STAT_CONN_LOG_SLOT_YIELD_RACE 1232 /*! log: slot join calls did not yield */ -#define WT_STAT_CONN_LOG_SLOT_IMMEDIATE 1232 +#define WT_STAT_CONN_LOG_SLOT_IMMEDIATE 1233 /*! log: slot join calls found active slot closed */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_CLOSE 1233 +#define WT_STAT_CONN_LOG_SLOT_YIELD_CLOSE 1234 /*! log: slot join calls slept */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_SLEEP 1234 +#define WT_STAT_CONN_LOG_SLOT_YIELD_SLEEP 1235 /*! log: slot join calls yielded */ -#define WT_STAT_CONN_LOG_SLOT_YIELD 1235 +#define WT_STAT_CONN_LOG_SLOT_YIELD 1236 /*! log: slot join found active slot closed */ -#define WT_STAT_CONN_LOG_SLOT_ACTIVE_CLOSED 1236 +#define WT_STAT_CONN_LOG_SLOT_ACTIVE_CLOSED 1237 /*! log: slot joins yield time (usecs) */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_DURATION 1237 +#define WT_STAT_CONN_LOG_SLOT_YIELD_DURATION 1238 /*! log: slot transitions unable to find free slot */ -#define WT_STAT_CONN_LOG_SLOT_NO_FREE_SLOTS 1238 +#define WT_STAT_CONN_LOG_SLOT_NO_FREE_SLOTS 1239 /*! log: slot unbuffered writes */ -#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1239 +#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1240 /*! log: total in-memory size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_MEM 1240 +#define WT_STAT_CONN_LOG_COMPRESS_MEM 1241 /*! log: total log buffer size */ -#define WT_STAT_CONN_LOG_BUFFER_SIZE 1241 +#define WT_STAT_CONN_LOG_BUFFER_SIZE 1242 /*! log: total size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_LEN 1242 +#define WT_STAT_CONN_LOG_COMPRESS_LEN 1243 /*! log: written slots coalesced */ -#define WT_STAT_CONN_LOG_SLOT_COALESCED 1243 +#define WT_STAT_CONN_LOG_SLOT_COALESCED 1244 /*! log: yields waiting for previous log file close */ -#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1244 +#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1245 /*! perf: file system read latency histogram (bucket 1) - 10-49ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT50 1245 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT50 1246 /*! perf: file system read latency histogram (bucket 2) - 50-99ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT100 1246 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT100 1247 /*! perf: file system read latency histogram (bucket 3) - 100-249ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT250 1247 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT250 1248 /*! perf: file system read latency histogram (bucket 4) - 250-499ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT500 1248 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT500 1249 /*! perf: file system read latency histogram (bucket 5) - 500-999ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT1000 1249 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT1000 1250 /*! perf: file system read latency histogram (bucket 6) - 1000ms+ */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_GT1000 1250 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_GT1000 1251 /*! perf: file system write latency histogram (bucket 1) - 10-49ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT50 1251 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT50 1252 /*! perf: file system write latency histogram (bucket 2) - 50-99ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT100 1252 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT100 1253 /*! perf: file system write latency histogram (bucket 3) - 100-249ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT250 1253 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT250 1254 /*! perf: file system write latency histogram (bucket 4) - 250-499ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT500 1254 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT500 1255 /*! perf: file system write latency histogram (bucket 5) - 500-999ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT1000 1255 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT1000 1256 /*! perf: file system write latency histogram (bucket 6) - 1000ms+ */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_GT1000 1256 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_GT1000 1257 /*! perf: operation read latency histogram (bucket 1) - 100-249us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT250 1257 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT250 1258 /*! perf: operation read latency histogram (bucket 2) - 250-499us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT500 1258 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT500 1259 /*! perf: operation read latency histogram (bucket 3) - 500-999us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT1000 1259 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT1000 1260 /*! perf: operation read latency histogram (bucket 4) - 1000-9999us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT10000 1260 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT10000 1261 /*! perf: operation read latency histogram (bucket 5) - 10000us+ */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_GT10000 1261 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_GT10000 1262 /*! perf: operation write latency histogram (bucket 1) - 100-249us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT250 1262 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT250 1263 /*! perf: operation write latency histogram (bucket 2) - 250-499us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT500 1263 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT500 1264 /*! perf: operation write latency histogram (bucket 3) - 500-999us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT1000 1264 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT1000 1265 /*! perf: operation write latency histogram (bucket 4) - 1000-9999us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT10000 1265 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT10000 1266 /*! perf: operation write latency histogram (bucket 5) - 10000us+ */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_GT10000 1266 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_GT10000 1267 /*! reconciliation: fast-path pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1267 +#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1268 /*! reconciliation: page reconciliation calls */ -#define WT_STAT_CONN_REC_PAGES 1268 +#define WT_STAT_CONN_REC_PAGES 1269 /*! reconciliation: page reconciliation calls for eviction */ -#define WT_STAT_CONN_REC_PAGES_EVICTION 1269 +#define WT_STAT_CONN_REC_PAGES_EVICTION 1270 /*! reconciliation: pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE 1270 +#define WT_STAT_CONN_REC_PAGE_DELETE 1271 /*! reconciliation: split bytes currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1271 +#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1272 /*! reconciliation: split objects currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1272 -/*! session: open cursor count */ -#define WT_STAT_CONN_SESSION_CURSOR_OPEN 1273 +#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1273 /*! session: open session count */ #define WT_STAT_CONN_SESSION_OPEN 1274 /*! session: session query timestamp calls */ @@ -6058,61 +6059,59 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_DSRC_CURSOR_MODIFY 2113 /*! cursor: next calls */ #define WT_STAT_DSRC_CURSOR_NEXT 2114 +/*! cursor: open cursor count */ +#define WT_STAT_DSRC_CURSOR_OPEN_COUNT 2115 /*! cursor: prev calls */ -#define WT_STAT_DSRC_CURSOR_PREV 2115 +#define WT_STAT_DSRC_CURSOR_PREV 2116 /*! cursor: remove calls */ -#define WT_STAT_DSRC_CURSOR_REMOVE 2116 +#define WT_STAT_DSRC_CURSOR_REMOVE 2117 /*! cursor: reserve calls */ -#define WT_STAT_DSRC_CURSOR_RESERVE 2117 +#define WT_STAT_DSRC_CURSOR_RESERVE 2118 /*! cursor: reset calls */ -#define WT_STAT_DSRC_CURSOR_RESET 2118 +#define WT_STAT_DSRC_CURSOR_RESET 2119 /*! cursor: search calls */ -#define WT_STAT_DSRC_CURSOR_SEARCH 2119 +#define WT_STAT_DSRC_CURSOR_SEARCH 2120 /*! cursor: search near calls */ -#define WT_STAT_DSRC_CURSOR_SEARCH_NEAR 2120 +#define WT_STAT_DSRC_CURSOR_SEARCH_NEAR 2121 /*! cursor: truncate calls */ -#define WT_STAT_DSRC_CURSOR_TRUNCATE 2121 +#define WT_STAT_DSRC_CURSOR_TRUNCATE 2122 /*! cursor: update calls */ -#define WT_STAT_DSRC_CURSOR_UPDATE 2122 +#define WT_STAT_DSRC_CURSOR_UPDATE 2123 /*! reconciliation: dictionary matches */ -#define WT_STAT_DSRC_REC_DICTIONARY 2123 +#define WT_STAT_DSRC_REC_DICTIONARY 2124 /*! reconciliation: fast-path pages deleted */ -#define WT_STAT_DSRC_REC_PAGE_DELETE_FAST 2124 +#define WT_STAT_DSRC_REC_PAGE_DELETE_FAST 2125 /*! * reconciliation: internal page key bytes discarded using suffix * compression */ -#define WT_STAT_DSRC_REC_SUFFIX_COMPRESSION 2125 +#define WT_STAT_DSRC_REC_SUFFIX_COMPRESSION 2126 /*! reconciliation: internal page multi-block writes */ -#define WT_STAT_DSRC_REC_MULTIBLOCK_INTERNAL 2126 +#define WT_STAT_DSRC_REC_MULTIBLOCK_INTERNAL 2127 /*! reconciliation: internal-page overflow keys */ -#define WT_STAT_DSRC_REC_OVERFLOW_KEY_INTERNAL 2127 +#define WT_STAT_DSRC_REC_OVERFLOW_KEY_INTERNAL 2128 /*! reconciliation: leaf page key bytes discarded using prefix compression */ -#define WT_STAT_DSRC_REC_PREFIX_COMPRESSION 2128 +#define WT_STAT_DSRC_REC_PREFIX_COMPRESSION 2129 /*! reconciliation: leaf page multi-block writes */ -#define WT_STAT_DSRC_REC_MULTIBLOCK_LEAF 2129 +#define WT_STAT_DSRC_REC_MULTIBLOCK_LEAF 2130 /*! reconciliation: leaf-page overflow keys */ -#define WT_STAT_DSRC_REC_OVERFLOW_KEY_LEAF 2130 +#define WT_STAT_DSRC_REC_OVERFLOW_KEY_LEAF 2131 /*! reconciliation: maximum blocks required for a page */ -#define WT_STAT_DSRC_REC_MULTIBLOCK_MAX 2131 +#define WT_STAT_DSRC_REC_MULTIBLOCK_MAX 2132 /*! reconciliation: overflow values written */ -#define WT_STAT_DSRC_REC_OVERFLOW_VALUE 2132 +#define WT_STAT_DSRC_REC_OVERFLOW_VALUE 2133 /*! reconciliation: page checksum matches */ -#define WT_STAT_DSRC_REC_PAGE_MATCH 2133 +#define WT_STAT_DSRC_REC_PAGE_MATCH 2134 /*! reconciliation: page reconciliation calls */ -#define WT_STAT_DSRC_REC_PAGES 2134 +#define WT_STAT_DSRC_REC_PAGES 2135 /*! reconciliation: page reconciliation calls for eviction */ -#define WT_STAT_DSRC_REC_PAGES_EVICTION 2135 +#define WT_STAT_DSRC_REC_PAGES_EVICTION 2136 /*! reconciliation: pages deleted */ -#define WT_STAT_DSRC_REC_PAGE_DELETE 2136 -/*! session: cached cursor count */ -#define WT_STAT_DSRC_SESSION_CURSORS_CACHED 2137 +#define WT_STAT_DSRC_REC_PAGE_DELETE 2137 /*! session: object compaction */ #define WT_STAT_DSRC_SESSION_COMPACT 2138 -/*! session: open cursor count */ -#define WT_STAT_DSRC_SESSION_CURSOR_OPEN 2139 /*! transaction: update conflicts */ -#define WT_STAT_DSRC_TXN_UPDATE_CONFLICT 2140 +#define WT_STAT_DSRC_TXN_UPDATE_CONFLICT 2139 /*! * @} diff --git a/src/third_party/wiredtiger/src/support/mtx_rw.c b/src/third_party/wiredtiger/src/support/mtx_rw.c index 959405dee50..9b7c3d441bd 100644 --- a/src/third_party/wiredtiger/src/support/mtx_rw.c +++ b/src/third_party/wiredtiger/src/support/mtx_rw.c @@ -171,13 +171,14 @@ void __wt_readlock(WT_SESSION_IMPL *session, WT_RWLOCK *l) { WT_RWLOCK new, old; - uint64_t time_start, time_stop; - int64_t **stats; + uint64_t time_diff, time_start, time_stop; + int64_t *session_stats, **stats; int16_t writers_active; uint8_t ticket; int pause_cnt; bool set_stats; + session_stats = NULL; /* -Wconditional-uninitialized */ stats = NULL; /* -Wconditional-uninitialized */ time_start = time_stop = 0; /* -Wconditional-uninitialized */ @@ -243,6 +244,7 @@ stall: __wt_cond_wait(session, if (set_stats) { stats = (int64_t **)S2C(session)->stats; stats[session->stat_bucket][l->stat_read_count_off]++; + session_stats = (int64_t *)&(session->stats); time_start = __wt_clock(session); } /* Wait for our group to start. */ @@ -260,12 +262,15 @@ stall: __wt_cond_wait(session, } if (set_stats) { time_stop = __wt_clock(session); + time_diff = WT_CLOCKDIFF_US(time_stop, time_start); if (F_ISSET(session, WT_SESSION_INTERNAL)) stats[session->stat_bucket][l->stat_int_usecs_off] += - (int64_t)WT_CLOCKDIFF_US(time_stop, time_start); - else + (int64_t)time_diff; + else { stats[session->stat_bucket][l->stat_app_usecs_off] += - (int64_t)WT_CLOCKDIFF_US(time_stop, time_start); + (int64_t)time_diff; + } + session_stats[l->stat_session_usecs_off] += (int64_t)time_diff; } /* @@ -371,12 +376,13 @@ void __wt_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *l) { WT_RWLOCK new, old; - uint64_t time_start, time_stop; - int64_t **stats; + uint64_t time_diff, time_start, time_stop; + int64_t *session_stats, **stats; uint8_t ticket; int pause_cnt; bool set_stats; + session_stats = NULL; /* -Wconditional-uninitialized */ stats = NULL; /* -Wconditional-uninitialized */ time_start = time_stop = 0; /* -Wconditional-uninitialized */ @@ -407,6 +413,7 @@ __wt_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *l) if (set_stats) { stats = (int64_t **)S2C(session)->stats; stats[session->stat_bucket][l->stat_write_count_off]++; + session_stats = (int64_t *)&(session->stats); time_start = __wt_clock(session); } /* @@ -433,12 +440,14 @@ __wt_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *l) } if (set_stats) { time_stop = __wt_clock(session); + time_diff = WT_CLOCKDIFF_US(time_stop, time_start); if (F_ISSET(session, WT_SESSION_INTERNAL)) stats[session->stat_bucket][l->stat_int_usecs_off] += - (int64_t)WT_CLOCKDIFF_US(time_stop, time_start); + (int64_t)time_diff; else stats[session->stat_bucket][l->stat_app_usecs_off] += - (int64_t)WT_CLOCKDIFF_US(time_stop, time_start); + (int64_t)time_diff; + session_stats[l->stat_session_usecs_off] += (int64_t)time_diff; } /* diff --git a/src/third_party/wiredtiger/src/support/stat.c b/src/third_party/wiredtiger/src/support/stat.c index e46c4838063..cee1f270d6d 100644 --- a/src/third_party/wiredtiger/src/support/stat.c +++ b/src/third_party/wiredtiger/src/support/stat.c @@ -118,6 +118,7 @@ static const char * const __stats_dsrc_desc[] = { "cursor: insert calls", "cursor: modify calls", "cursor: next calls", + "cursor: open cursor count", "cursor: prev calls", "cursor: remove calls", "cursor: reserve calls", @@ -140,9 +141,7 @@ static const char * const __stats_dsrc_desc[] = { "reconciliation: page reconciliation calls", "reconciliation: page reconciliation calls for eviction", "reconciliation: pages deleted", - "session: cached cursor count", "session: object compaction", - "session: open cursor count", "transaction: update conflicts", }; @@ -301,6 +300,7 @@ __wt_stat_dsrc_clear_single(WT_DSRC_STATS *stats) stats->cursor_insert = 0; stats->cursor_modify = 0; stats->cursor_next = 0; + /* not clearing cursor_open_count */ stats->cursor_prev = 0; stats->cursor_remove = 0; stats->cursor_reserve = 0; @@ -323,9 +323,7 @@ __wt_stat_dsrc_clear_single(WT_DSRC_STATS *stats) stats->rec_pages = 0; stats->rec_pages_eviction = 0; stats->rec_page_delete = 0; - /* not clearing session_cursors_cached */ stats->session_compact = 0; - /* not clearing session_cursor_open */ stats->txn_update_conflict = 0; } @@ -485,6 +483,7 @@ __wt_stat_dsrc_aggregate_single( to->cursor_insert += from->cursor_insert; to->cursor_modify += from->cursor_modify; to->cursor_next += from->cursor_next; + to->cursor_open_count += from->cursor_open_count; to->cursor_prev += from->cursor_prev; to->cursor_remove += from->cursor_remove; to->cursor_reserve += from->cursor_reserve; @@ -508,9 +507,7 @@ __wt_stat_dsrc_aggregate_single( to->rec_pages += from->rec_pages; to->rec_pages_eviction += from->rec_pages_eviction; to->rec_page_delete += from->rec_page_delete; - to->session_cursors_cached += from->session_cursors_cached; to->session_compact += from->session_compact; - to->session_cursor_open += from->session_cursor_open; to->txn_update_conflict += from->txn_update_conflict; } @@ -702,6 +699,7 @@ __wt_stat_dsrc_aggregate( to->cursor_insert += WT_STAT_READ(from, cursor_insert); to->cursor_modify += WT_STAT_READ(from, cursor_modify); to->cursor_next += WT_STAT_READ(from, cursor_next); + to->cursor_open_count += WT_STAT_READ(from, cursor_open_count); to->cursor_prev += WT_STAT_READ(from, cursor_prev); to->cursor_remove += WT_STAT_READ(from, cursor_remove); to->cursor_reserve += WT_STAT_READ(from, cursor_reserve); @@ -731,10 +729,7 @@ __wt_stat_dsrc_aggregate( to->rec_pages += WT_STAT_READ(from, rec_pages); to->rec_pages_eviction += WT_STAT_READ(from, rec_pages_eviction); to->rec_page_delete += WT_STAT_READ(from, rec_page_delete); - to->session_cursors_cached += - WT_STAT_READ(from, session_cursors_cached); to->session_compact += WT_STAT_READ(from, session_compact); - to->session_cursor_open += WT_STAT_READ(from, session_cursor_open); to->txn_update_conflict += WT_STAT_READ(from, txn_update_conflict); } @@ -881,6 +876,7 @@ static const char * const __stats_connection_desc[] = { "connection: total fsync I/Os", "connection: total read I/Os", "connection: total write I/Os", + "cursor: cached cursor count", "cursor: cursor close calls that result in cache", "cursor: cursor create calls", "cursor: cursor insert calls", @@ -898,8 +894,8 @@ static const char * const __stats_connection_desc[] = { "cursor: cursor sweep cursors examined", "cursor: cursor sweeps", "cursor: cursor update calls", - "cursor: cursors currently cached", "cursor: cursors reused from cache", + "cursor: open cursor count", "cursor: truncate calls", "data-handle: connection data handles currently active", "data-handle: connection sweep candidate became referenced", @@ -1012,7 +1008,6 @@ static const char * const __stats_connection_desc[] = { "reconciliation: pages deleted", "reconciliation: split bytes currently awaiting free", "reconciliation: split objects currently awaiting free", - "session: open cursor count", "session: open session count", "session: session query timestamp calls", "session: table alter failed calls", @@ -1290,6 +1285,7 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->fsync_io = 0; stats->read_io = 0; stats->write_io = 0; + /* not clearing cursor_cached_count */ stats->cursor_cache = 0; stats->cursor_create = 0; stats->cursor_insert = 0; @@ -1307,8 +1303,8 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->cursor_sweep_examined = 0; stats->cursor_sweep = 0; stats->cursor_update = 0; - /* not clearing cursors_cached */ stats->cursor_reopen = 0; + /* not clearing cursor_open_count */ stats->cursor_truncate = 0; /* not clearing dh_conn_handle_count */ stats->dh_sweep_ref = 0; @@ -1421,7 +1417,6 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->rec_page_delete = 0; /* not clearing rec_split_stashed_bytes */ /* not clearing rec_split_stashed_objects */ - /* not clearing session_cursor_open */ /* not clearing session_open */ stats->session_query_ts = 0; /* not clearing session_table_alter_fail */ @@ -1745,6 +1740,7 @@ __wt_stat_connection_aggregate( to->fsync_io += WT_STAT_READ(from, fsync_io); to->read_io += WT_STAT_READ(from, read_io); to->write_io += WT_STAT_READ(from, write_io); + to->cursor_cached_count += WT_STAT_READ(from, cursor_cached_count); to->cursor_cache += WT_STAT_READ(from, cursor_cache); to->cursor_create += WT_STAT_READ(from, cursor_create); to->cursor_insert += WT_STAT_READ(from, cursor_insert); @@ -1763,8 +1759,8 @@ __wt_stat_connection_aggregate( WT_STAT_READ(from, cursor_sweep_examined); to->cursor_sweep += WT_STAT_READ(from, cursor_sweep); to->cursor_update += WT_STAT_READ(from, cursor_update); - to->cursors_cached += WT_STAT_READ(from, cursors_cached); to->cursor_reopen += WT_STAT_READ(from, cursor_reopen); + to->cursor_open_count += WT_STAT_READ(from, cursor_open_count); to->cursor_truncate += WT_STAT_READ(from, cursor_truncate); to->dh_conn_handle_count += WT_STAT_READ(from, dh_conn_handle_count); to->dh_sweep_ref += WT_STAT_READ(from, dh_sweep_ref); @@ -1936,7 +1932,6 @@ __wt_stat_connection_aggregate( WT_STAT_READ(from, rec_split_stashed_bytes); to->rec_split_stashed_objects += WT_STAT_READ(from, rec_split_stashed_objects); - to->session_cursor_open += WT_STAT_READ(from, session_cursor_open); to->session_open += WT_STAT_READ(from, session_open); to->session_query_ts += WT_STAT_READ(from, session_query_ts); to->session_table_alter_fail += diff --git a/src/third_party/wiredtiger/test/checkpoint/smoke.sh b/src/third_party/wiredtiger/test/checkpoint/smoke.sh index 39b1f428c2c..2f1d4345ad7 100755 --- a/src/third_party/wiredtiger/test/checkpoint/smoke.sh +++ b/src/third_party/wiredtiger/test/checkpoint/smoke.sh @@ -6,9 +6,6 @@ set -e echo "checkpoint: 3 mixed tables" $TEST_WRAPPER ./t -T 3 -t m -# We are done unless long tests are enabled. -test "$TESTUTIL_ENABLE_LONG_TESTS" = "1" || exit 0 - echo "checkpoint: 6 column-store tables" $TEST_WRAPPER ./t -T 6 -t c diff --git a/src/third_party/wiredtiger/test/csuite/README b/src/third_party/wiredtiger/test/csuite/README new file mode 100644 index 00000000000..c1c42372ec3 --- /dev/null +++ b/src/third_party/wiredtiger/test/csuite/README @@ -0,0 +1,12 @@ +The 'test/csuite' directory includes a collection of sanity tests written in C. +They are expected to be executed as part of 'make check' testing. + +Each sub directory of 'test/csuite' include source code of one csuite test, and +should have one corresponding task in the Evergreen (CI system) configuration file. +See 'test/evergreen.yml' for details. + +When a new csuite test is introduced, a corresponding new Evergreen task should be +crafted and put into the Evergreen configuration file. We have a utility program +'test/evergreen/evg_cfg.py' to help with identifying and auto-generating the +Evergreen configuration for new or missing csuite tests. The program checking +has been bound into developer workflow through 'dist/s_evergreen'. diff --git a/src/third_party/wiredtiger/test/csuite/rwlock/main.c b/src/third_party/wiredtiger/test/csuite/rwlock/main.c index f69628dca40..45ff418c7cc 100644 --- a/src/third_party/wiredtiger/test/csuite/rwlock/main.c +++ b/src/third_party/wiredtiger/test/csuite/rwlock/main.c @@ -55,10 +55,6 @@ main(int argc, char *argv[]) pthread_t dump_id, id[MAX_THREADS]; int i; - /* Ignore unless requested */ - if (!testutil_is_flag_set("TESTUTIL_ENABLE_LONG_TESTS")) - return (EXIT_SUCCESS); - opts = &_opts; memset(opts, 0, sizeof(*opts)); opts->nthreads = 100; @@ -110,7 +106,8 @@ thread_rwlock(void *arg) opts->conn->open_session(opts->conn, NULL, NULL, &wt_session)); session = (WT_SESSION_IMPL *)wt_session; - printf("Running rwlock thread\n"); + if (opts->verbose) + printf("Running rwlock thread\n"); for (i = 1; i <= opts->nops; ++i) { writelock = (i % READS_PER_WRITE == 0); @@ -152,7 +149,7 @@ thread_rwlock(void *arg) __wt_readunlock(session, &rwlock); #endif - if (i % 10000 == 0) { + if (opts->verbose && i % 10000 == 0) { printf("%s", session->id == 20 ? ".\n" : "."); fflush(stdout); } @@ -164,20 +161,24 @@ thread_rwlock(void *arg) } void * -thread_dump(void *arg) { - WT_UNUSED(arg); +thread_dump(void *arg) +{ + TEST_OPTS *opts; + + opts = arg; while (running) { sleep(1); - printf("\n" - "rwlock { current %" PRIu8 ", next %" PRIu8 - ", reader %" PRIu8 ", readers_active %" PRIu32 - ", readers_queued %" PRIu8 " }\n", - rwlock.u.s.current, - rwlock.u.s.next, - rwlock.u.s.reader, - rwlock.u.s.readers_active, - rwlock.u.s.readers_queued); + if (opts->verbose) + printf("\n" + "rwlock { current %" PRIu8 ", next %" PRIu8 + ", reader %" PRIu8 ", readers_active %" PRIu32 + ", readers_queued %" PRIu8 " }\n", + rwlock.u.s.current, + rwlock.u.s.next, + rwlock.u.s.reader, + rwlock.u.s.readers_active, + rwlock.u.s.readers_queued); } return (NULL); diff --git a/src/third_party/wiredtiger/test/csuite/time_shift_test.sh b/src/third_party/wiredtiger/test/csuite/time_shift_test.sh index ae06fd03f36..9c14ae86ae7 100755 --- a/src/third_party/wiredtiger/test/csuite/time_shift_test.sh +++ b/src/third_party/wiredtiger/test/csuite/time_shift_test.sh @@ -11,13 +11,6 @@ set -e # assumption that other factors of the environment will influence the execution # time by less than 20%. - -# need to enable long tests to run test_rwlock -export TESTUTIL_ENABLE_LONG_TESTS=1 - -# We will run only when long tests are enabled. -test "$TESTUTIL_ENABLE_LONG_TESTS" = "1" || exit 0 - EXIT_SUCCESS=0 EXIT_FAILURE=1 diff --git a/src/third_party/wiredtiger/test/csuite/wt2246_col_append/main.c b/src/third_party/wiredtiger/test/csuite/wt2246_col_append/main.c index fca3d8fcd00..2757f991c2a 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2246_col_append/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2246_col_append/main.c @@ -79,9 +79,6 @@ page_init(uint64_t n) } } -/* - * TODO: Platform specific? - */ static void onsig(int signo) { @@ -101,10 +98,6 @@ main(int argc, char *argv[]) uint64_t i, id; char buf[100]; - /* Ignore unless requested */ - if (!testutil_is_flag_set("TESTUTIL_ENABLE_LONG_TESTS")) - return (EXIT_SUCCESS); - opts = &_opts; memset(opts, 0, sizeof(*opts)); opts->table_type = TABLE_ROW; diff --git a/src/third_party/wiredtiger/test/csuite/wt2323_join_visibility/main.c b/src/third_party/wiredtiger/test/csuite/wt2323_join_visibility/main.c index 0de304e61bd..0b99df76cf3 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2323_join_visibility/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2323_join_visibility/main.c @@ -92,10 +92,6 @@ main(int argc, char *argv[]) TEST_OPTS *opts, _opts; const char *tablename; - /* Ignore unless requested */ - if (!testutil_is_flag_set("TESTUTIL_ENABLE_LONG_TESTS")) - return (EXIT_SUCCESS); - opts = &_opts; sharedopts = &_sharedopts; memset(opts, 0, sizeof(*opts)); @@ -318,7 +314,8 @@ thread_insert(void *arg) return (NULL); } -static void *thread_join(void *arg) +static void * +thread_join(void *arg) { SHARED_OPTS *sharedopts; TEST_OPTS *opts; diff --git a/src/third_party/wiredtiger/test/csuite/wt2535_insert_race/main.c b/src/third_party/wiredtiger/test/csuite/wt2535_insert_race/main.c index 966258163f6..19620cec894 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2535_insert_race/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2535_insert_race/main.c @@ -38,6 +38,8 @@ void *thread_insert_race(void *); +static uint64_t ready_counter; + int main(int argc, char *argv[]) { @@ -49,14 +51,10 @@ main(int argc, char *argv[]) uint64_t current_value; int i; - /* Ignore unless requested */ - if (!testutil_is_flag_set("TESTUTIL_ENABLE_LONG_TESTS")) - return (EXIT_SUCCESS); - opts = &_opts; memset(opts, 0, sizeof(*opts)); - opts->nthreads = 10; - opts->nrecords = 1000; + opts->nthreads = 20; + opts->nrecords = 100000; opts->table_type = TABLE_ROW; testutil_check(testutil_parse_opts(argc, argv, opts)); testutil_make_work_dir(opts->home); @@ -119,16 +117,25 @@ thread_insert_race(void *arg) WT_CURSOR *cursor; WT_DECL_RET; WT_SESSION *session; - uint64_t i, value; + uint64_t i, value, ready_counter_local; opts = (TEST_OPTS *)arg; conn = opts->conn; + printf("Running insert thread\n"); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); testutil_check(session->open_cursor( session, opts->uri, NULL, NULL, &cursor)); - printf("Running insert thread\n"); + /* Wait until all the threads are ready to go. */ + (void)__wt_atomic_add64(&ready_counter, 1); + for (;; __wt_yield()) { + WT_ORDERED_READ(ready_counter_local, ready_counter); + if (ready_counter_local >= opts->nthreads) + break; + } + for (i = 0; i < opts->nrecords; ++i) { testutil_check( session->begin_transaction(session, "isolation=snapshot")); diff --git a/src/third_party/wiredtiger/test/csuite/wt2834_join_bloom_fix/main.c b/src/third_party/wiredtiger/test/csuite/wt2834_join_bloom_fix/main.c index 58df56b50b1..a8d44bf3dab 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2834_join_bloom_fix/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2834_join_bloom_fix/main.c @@ -59,10 +59,6 @@ main(int argc, char *argv[]) char posturi[256]; const char *tablename; - /* Ignore unless requested */ - if (!testutil_is_flag_set("TESTUTIL_ENABLE_LONG_TESTS")) - return (EXIT_SUCCESS); - opts = &_opts; memset(opts, 0, sizeof(*opts)); testutil_check(testutil_parse_opts(argc, argv, opts)); diff --git a/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c b/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c index 93fb94836c0..80911ddfd2d 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c @@ -82,10 +82,6 @@ main(int argc, char *argv[]) int i, nfail; const char *tablename; - /* Ignore unless requested */ - if (!testutil_is_flag_set("TESTUTIL_ENABLE_LONG_TESTS")) - return (EXIT_SUCCESS); - opts = &_opts; sharedopts = &_sharedopts; memset(opts, 0, sizeof(*opts)); diff --git a/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c b/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c index 10c0571c9f6..2020b4dcecd 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c @@ -379,7 +379,7 @@ run_check_subtest_range(TEST_OPTS *opts, const char *debugger, bool close_test) low = 0; high = MAX_OP_RANGE; mid = (low + high) / 2; - while (mid != low) { + while (low < mid - 5 || high > mid + 5) { run_check_subtest(opts, debugger, mid, close_test, &nresults); if (nresults > cutoff) @@ -401,8 +401,9 @@ run_check_subtest_range(TEST_OPTS *opts, const char *debugger, bool close_test) got_failure = false; got_success = false; - for (nops = mid - 10; nops < mid + 10; nops++) { - for (i = 0; i < TESTS_PER_OP_VALUE; i++) { + for (i = 0; + i < TESTS_PER_OP_VALUE && (!got_failure || !got_success); i++) + for (nops = mid - 10; nops < mid + 10; nops++) { run_check_subtest(opts, debugger, nops, close_test, &nresults); if (nresults > cutoff) @@ -410,7 +411,7 @@ run_check_subtest_range(TEST_OPTS *opts, const char *debugger, bool close_test) else got_success = true; } - } + /* * Check that it really ran with a crossover point. */ @@ -647,10 +648,6 @@ main(int argc, char *argv[]) uint64_t nresults; const char *debugger; - /* Ignore unless requested */ - if (!testutil_is_flag_set("TESTUTIL_ENABLE_LONG_TESTS")) - return (EXIT_SUCCESS); - opts = &_opts; memset(opts, 0, sizeof(*opts)); debugger = NULL; diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml index 8c957e487ad..be111c6432c 100644 --- a/src/third_party/wiredtiger/test/evergreen.yml +++ b/src/third_party/wiredtiger/test/evergreen.yml @@ -97,7 +97,7 @@ tasks: set -o errexit set -o verbose - ${test_env_vars|} TESTUTIL_ENABLE_LONG_TESTS=1 ${make_command|make} VERBOSE=1 check ${smp_command|} 2>&1 + ${test_env_vars|} ${make_command|make} VERBOSE=1 check ${smp_command|} 2>&1 # Start of normal make check test tasks @@ -621,7 +621,7 @@ tasks: set -o errexit set -o verbose - ${test_env_vars|} TESTUTIL_ENABLE_LONG_TESTS=1 $(pwd)/test_rwlock 2>&1 + ${test_env_vars|} $(pwd)/test_rwlock 2>&1 - name: csuite-wt2246-col-append-test depends_on: @@ -635,7 +635,7 @@ tasks: set -o errexit set -o verbose - ${test_env_vars|} TESTUTIL_ENABLE_LONG_TESTS=1 $(pwd)/test_wt2246_col_append 2>&1 + ${test_env_vars|} $(pwd)/test_wt2246_col_append 2>&1 - name: csuite-wt2323-join-visibility-test depends_on: @@ -649,7 +649,7 @@ tasks: set -o errexit set -o verbose - ${test_env_vars|} TESTUTIL_ENABLE_LONG_TESTS=1 $(pwd)/test_wt2323_join_visibility 2>&1 + ${test_env_vars|} $(pwd)/test_wt2323_join_visibility 2>&1 - name: csuite-wt2535-insert-race-test depends_on: @@ -663,7 +663,7 @@ tasks: set -o errexit set -o verbose - ${test_env_vars|} TESTUTIL_ENABLE_LONG_TESTS=1 $(pwd)/test_wt2535_insert_race 2>&1 + ${test_env_vars|} $(pwd)/test_wt2535_insert_race 2>&1 - name: csuite-wt2834-join-bloom-fix-test depends_on: @@ -677,7 +677,7 @@ tasks: set -o errexit set -o verbose - ${test_env_vars|} TESTUTIL_ENABLE_LONG_TESTS=1 $(pwd)/test_wt2834_join_bloom_fix 2>&1 + ${test_env_vars|} $(pwd)/test_wt2834_join_bloom_fix 2>&1 - name: csuite-wt2853-perf-test depends_on: @@ -691,7 +691,7 @@ tasks: set -o errexit set -o verbose - ${test_env_vars|} TESTUTIL_ENABLE_LONG_TESTS=1 $(pwd)/test_wt2853_perf 2>&1 + ${test_env_vars|} $(pwd)/test_wt2853_perf 2>&1 - name: csuite-wt2909-checkpoint-integrity-test depends_on: @@ -705,7 +705,7 @@ tasks: set -o errexit set -o verbose - ${test_env_vars|} TESTUTIL_ENABLE_LONG_TESTS=1 $(pwd)/test_wt2909_checkpoint_integrity 2>&1 + ${test_env_vars|} $(pwd)/test_wt2909_checkpoint_integrity 2>&1 - name: csuite-wt3338-partial-update-test depends_on: @@ -719,7 +719,7 @@ tasks: set -o errexit set -o verbose - ${test_env_vars|} TESTUTIL_ENABLE_LONG_TESTS=1 $(pwd)/test_wt3338_partial_update 2>&1 + ${test_env_vars|} $(pwd)/test_wt3338_partial_update 2>&1 - name: csuite-wt4333-handle-locks-test depends_on: @@ -733,7 +733,7 @@ tasks: set -o errexit set -o verbose - ${test_env_vars|} TESTUTIL_ENABLE_LONG_TESTS=1 $(pwd)/test_wt4333_handle_locks 2>&1 + ${test_env_vars|} $(pwd)/test_wt4333_handle_locks 2>&1 # End of csuite test tasks diff --git a/src/third_party/wiredtiger/test/evergreen/csuite_test_evg_task.template b/src/third_party/wiredtiger/test/evergreen/csuite_test_evg_task.template new file mode 100644 index 00000000000..42d08c34fa0 --- /dev/null +++ b/src/third_party/wiredtiger/test/evergreen/csuite_test_evg_task.template @@ -0,0 +1,14 @@ + - name: {{task_name}} + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - command: shell.exec + params: + working_dir: "wiredtiger/test/csuite" + script: | + set -o errexit + set -o verbose + + ${test_env_vars|} $(pwd)/{{test_dir}}/test_{{test_dir}} 2>&1 + diff --git a/src/third_party/wiredtiger/test/evergreen/evg_cfg.py b/src/third_party/wiredtiger/test/evergreen/evg_cfg.py new file mode 100755 index 00000000000..0096219619b --- /dev/null +++ b/src/third_party/wiredtiger/test/evergreen/evg_cfg.py @@ -0,0 +1,331 @@ +#! /usr/bin/env python3 + +""" +This program provides an CLI interface to check and generate Evergreen configuration. +""" + +import os +import sys +import re +import subprocess + +try: + import docopt +except Exception as e: + modules = "docopt" + print("ERROR [%s]: %s" % (sys.argv[0], e)) + print("Use pip to install the required library:") + print(" pip install %s" % modules) + sys.exit(0) + +TEST_TYPES = ('make_check', 'csuite') +EVG_CFG_FILE = "test/evergreen.yml" +CSUITE_TEST_DIR = "test/csuite" +MAKE_CHECK_TEST_TMPLT = "test/evergreen/make_check_test_evg_task.template" +CSUITE_TEST_TMPLT = "test/evergreen/csuite_test_evg_task.template" +MAKE_CHECK_TEST_SEARCH_STR = " # End of normal make check test tasks" +CSUITE_TEST_SEARCH_STR = " # End of csuite test tasks" + +# This list of sub directories will be skipped from checking. +# They are not expected to trigger any 'make check' testing. +make_check_subdir_skips = [ + "api/leveldb", # no need to test + "test/csuite", # csuite has its own set of Evergreen tasks, skip the checking here +] + +prog=sys.argv[0] +PROGNAME = os.path.basename(prog) +DESCRIPTION = 'Evergreen configuration helper 0.1' +USAGE = """ +Evergreen configuration helper. + +Usage: + {progname} check [-t <test_type>] [-v] + {progname} generate [-t <test_type>] [-v] + {progname} (-h | --help) + +Options: + -h --help Show this screen. + -t TEST_TYPE The type of test to be checked/generated. + -v Enable verbose logging. + check Check if any missing tests that should be added into Evergreen configuration. + generate Generate Evergreen configuration for missing tests. +""".format(progname=PROGNAME) + +verbose = False + +def debug(msg): + """ A wrapper to print function with checking of verbose flag """ + + if verbose is True: + print(msg) + +def run(cmd): + """ Run a shell command and return the output """ + + if isinstance(cmd, str): + cmd = cmd.split() + + try: + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT).strip().decode() + except Exception as e: + sys.exit("ERROR [%s]: %s" % (prog, e)) + + return output + +def find_tests_missing_evg_cfg(test_type, dirs, evg_cfg_file): + """ + Check the list of 'make check' directories to find out those + that are missing from the Evergreen configuration file. + + The existing Evergreen configuration is expected to have + included one task for each applicable 'make check' directory. + Newly added 'make check' directories that involve real tests + should be identified by this function. + """ + + if not dirs: + sys.exit("\nNo %s directory is found ..." % test_type) + + assert os.path.isfile(evg_cfg_file), "'%s' does not exist" % evg_cfg_file + + with open(evg_cfg_file, 'r') as f: + evg_cfg = f.readlines() + + debug('\n') + missing_tests = {} + for d in dirs: + # Figure out the Evergreen task name from the directory name + + if test_type == 'make_check': + # The Evergreen task name for each 'make check' test is worked out from directory name + # E.g. for 'make check' directory 'test/cursor_order', the corresponding Evergreen + # task name will be 'cursor-order-test'. + + dir_wo_test_prefix = d[len("test/"):] if d.startswith("test/") else d + evg_task_name = dir_wo_test_prefix.replace('/', '-').replace('_', '-') + '-test' + debug("Evergreen task name for make check directory '%s' is: %s" % (d, evg_task_name)) + + elif test_type == 'csuite': + # The Evergreen task name for each 'csuite' test is worked out from sub directory name + # E.g. for 'test/csuite' sub directory 'wt3184_dup_index_collator', the corresponding + # Evergreen task name will be 'csuite-wt3184-dup-index-collator-test'. + + evg_task_name = 'csuite-' + d.replace('_', '-') + '-test' + debug("Evergreen task name for csuite sub directory '%s' is: %s" % (d, evg_task_name)) + + else: + sys.exit("Unsupported test_type '%s'" % test_type) + + # Check if the Evergreen task name exists in current Evergreen configuration + if evg_task_name in str(evg_cfg): + # Match found + continue + else: + # Missing task/test found + missing_tests.update({evg_task_name: d}) + print("Task '%s' (for directory '%s') is missing in %s!" % + (evg_task_name, d, evg_cfg_file)) + + return missing_tests + +def get_make_check_dirs(): + """ + Figure out the 'make check' directories that are applicable for testing + Directories with Makefile.am containing 'TESTS =' are the ones require test. + Skip a few known directories that do not require test or covered separately. + """ + + # Make sure we are under the repo top level directory + os.chdir(run('git rev-parse --show-toplevel')) + + # Search keyword in Makefile.am to identify directories that involve test configuration. + # Need to use subprocess 'shell=True' to get the expected shell command output. + cmd = "find . -name Makefile.am -exec grep -H -e '^TESTS =' {} \; | cut -d: -f1 | cut -c3-" + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True) + mkfiles_with_tests = p.stdout.readlines() + + # Need some string manipulation work here against the subprocess output. + # Cast elements to string, and strip the ending from the string to get directory names. + ending = '/Makefile.am\n' + dirs_with_tests = [d.decode('utf-8')[:-len(ending)] for d in mkfiles_with_tests] + debug("dirs_with_tests: %s" % dirs_with_tests) + + # Remove directories in the skip list + make_check_dirs = [d for d in dirs_with_tests if d not in make_check_subdir_skips] + debug("\nThe list of 'make check' dirs that should be included in Evergreen configuration:\n %s" + % make_check_dirs) + + return make_check_dirs + +def get_csuite_dirs(): + """ + Figure out the 'make check' directories that are applicable for testing + Loop through the list of sub directories under test/csuite/ and skip those WT_TEST.* directories + """ + + assert os.path.isdir(CSUITE_TEST_DIR), "'%s' does not exist" % CSUITE_TEST_DIR + + # Retrieve all sub directories under 'test/csuite' directory + subdirs = [x[1] for x in os.walk(CSUITE_TEST_DIR)][0] + + # Remove directories with name starting with 'WT_TEST' or '.' + regex = re.compile(r'^(WT_TEST|\.)') + csuite_dirs = [d for d in subdirs if not regex.search(d)] + debug("\nThe list of 'csuite' dirs that should be included in Evergreen configuration:\n %s" + % csuite_dirs) + + return csuite_dirs + +def check_missing_tests(test_type): + """ + Check to see if any tests are missing from the Evergreen configuration. + Loop through the list of directories in 'Make.subdirs' file and skip a few known + directories that do not require any test. + """ + + # Retrieve the directories that are applicable for testing based on test type + if test_type == 'make_check': + test_dirs = get_make_check_dirs() + elif test_type == 'csuite': + test_dirs = get_csuite_dirs() + else: + sys.exit("Unsupported test type '%s'" % test_type) + + return find_tests_missing_evg_cfg(test_type, test_dirs, EVG_CFG_FILE) + +def get_evg_task_template(test_type): + """ Retrieve the Evergreen task template based on test type """ + + if test_type == 'make_check': + template_file = MAKE_CHECK_TEST_TMPLT + elif test_type == 'csuite': + template_file = CSUITE_TEST_TMPLT + else: + sys.exit("Unsupported test type '%s'" % test_type) + + assert os.path.isfile(template_file), "'%s' does not exist" % template_file + + with open(template_file, 'r') as f: + template = f.read() + + return template + +def get_search_string(test_type): + """ Retrieve the search string based on test_type """ + + if test_type == 'make_check': + search_str = MAKE_CHECK_TEST_SEARCH_STR + elif test_type == 'csuite': + search_str = CSUITE_TEST_SEARCH_STR + else: + sys.exit("Unsupported test type '%s'" % test_type) + + return search_str + +def generate_evg_cfg_for_missing_tests(test_type, missing_tests): + """ + Generate the Evergreen configuration for the missing tests based on test type. + Will apply the newly generated changes to the Evergreen configuration file directly. + """ + + if not missing_tests: + sys.exit("No missing test is found, exiting ...") + debug("missing_tests: %s" % missing_tests) + + template = get_evg_task_template(test_type) + + evg_cfg_to_add = '' + for task, dir in missing_tests.items(): + # Replace variables in the template with valid values for each missing test + evg_cfg_to_add += template.replace('{{task_name}}', task).replace('{{test_dir}}', dir) + + print("\nBelow Evergreen configuration snippet will be added into existing %s file: \n\n%s" + % (EVG_CFG_FILE, evg_cfg_to_add)) + + assert os.path.isfile(EVG_CFG_FILE), "'%s' does not exist" % EVG_CFG_FILE + + search_str = get_search_string(test_type) + debug("search_str: '%s'" % search_str) + + with open(EVG_CFG_FILE, 'r') as f: + evg_cfg = f.read() + # Insert the new Evergreen configuration for missing tests. + # Use the search string to help locating the position for insert. + new_evg_cfg = evg_cfg.replace(search_str, evg_cfg_to_add + search_str) + + # Write the changes back to the file + with open(EVG_CFG_FILE, 'w') as f: + f.write(new_evg_cfg) + +def evg_cfg(action, test_type): + """ + The main entry function that calls different functions based on action type and test type + """ + + # Make sure the program is run under a checkout of wiredtiger repository + # We could get different string outputs when running 'git config remote.origin.url': + # - 'git@github.com:wiredtiger/wiredtiger.git' (if run locally) + # - 'ssh://git@github.com/wiredtiger/wiredtiger.git' (if run through SSH) + output = run('git config remote.origin.url') + if not 'github.com' in output or not 'wiredtiger.git' in output: + sys.exit("ERROR [%s]: need to run this script inside a wiredtiger repo" % prog) + + # Change directory to repo top level + os.chdir(run('git rev-parse --show-toplevel')) + + missing_tests = {} + if action == 'check': + if test_type in TEST_TYPES: + missing_tests = check_missing_tests(test_type) + elif test_type == 'all': + # Check each of the test type + for t in TEST_TYPES: + # Aggregate the missing tests for each test type together here + missing_tests.update(check_missing_tests(t)) + else: + sys.exit("Unsupported test type '%s'" % test_type) + + # If any missing test is identified, prompt users to run the 'generate' action + # which will auto-generate the Evergreen configuration for those missing tests. + if missing_tests: + prompt = ("\n*** Some tests are missing in Evergreen configuration ***\nPlease\n" + + "\t1) Run '{prog} generate' to generate and apply the Evergreen changes.\n" + + "\t2) Run 'git diff' to see the detail of the Evergreen changes.\n" + + "\t3) Trigger Evergreen patch build to verify the changes before merging.\n" + ).format(prog=prog) + print(prompt) + sys.exit(1) + + elif action == 'generate': + if test_type in TEST_TYPES: + missing_tests = check_missing_tests(test_type) + generate_evg_cfg_for_missing_tests(test_type, missing_tests) + elif test_type == 'all': + # Check each of the test type + for t in TEST_TYPES: + missing_tests = check_missing_tests(t) + generate_evg_cfg_for_missing_tests(t, missing_tests) + else: + sys.exit("Unsupported action type '%s'" % action) + +if __name__ == '__main__': + + args = docopt.docopt(USAGE, version=DESCRIPTION) + + verbose = args['-v'] + debug('\nargs:%s' % args) + + action = None + if args['check']: + action = 'check' + elif args['generate']: + action = 'generate' + assert action in ('check', 'generate') + + test_type = args.get('-t', None) + # If test type is not provided, assuming 'all' types need to be checked + if test_type is None: + test_type = 'all' + + evg_cfg(action, test_type) diff --git a/src/third_party/wiredtiger/test/evergreen/make_check_test_evg_task.template b/src/third_party/wiredtiger/test/evergreen/make_check_test_evg_task.template new file mode 100644 index 00000000000..762bcfff29d --- /dev/null +++ b/src/third_party/wiredtiger/test/evergreen/make_check_test_evg_task.template @@ -0,0 +1,15 @@ + - name: {{task_name}} + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - func: "compile wiredtiger" + - command: shell.exec + params: + working_dir: "wiredtiger" + script: | + set -o errexit + set -o verbose + + ${test_env_vars|} ${make_command|make} VERBOSE=1 check -C {{test_dir}} ${smp_command|} 2>&1 + diff --git a/src/third_party/wiredtiger/test/suite/test_backup10.py b/src/third_party/wiredtiger/test/suite/test_backup10.py index afac740999c..27838834ace 100644 --- a/src/third_party/wiredtiger/test/suite/test_backup10.py +++ b/src/third_party/wiredtiger/test/suite/test_backup10.py @@ -44,10 +44,15 @@ class test_backup10(wttest.WiredTigerTestCase, suite_subprocess): pfx = 'test_backup' + scenarios = make_scenarios([ + ('archiving', dict(archive='true')), + ('not-archiving', dict(archive='false')), + ]) + # Create a large cache, otherwise this test runs quite slowly. def conn_config(self): - return 'cache_size=1G,log=(archive=false,enabled,file_max=%s)' % \ - self.logmax + return 'cache_size=1G,log=(archive=%s,' % self.archive + \ + 'enabled,file_max=%s)' % self.logmax # Run background inserts while running checkpoints repeatedly. def test_backup10(self): diff --git a/src/third_party/wiredtiger/test/suite/test_cursor16.py b/src/third_party/wiredtiger/test/suite/test_cursor16.py index 8eeb46b8de1..1a3b4db2470 100755 --- a/src/third_party/wiredtiger/test/suite/test_cursor16.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor16.py @@ -44,7 +44,7 @@ class test_cursor16(wttest.WiredTigerTestCase): # Returns the number of cursors cached def cached_stats(self): stat_cursor = self.session.open_cursor('statistics:', None, None) - cache = stat_cursor[stat.conn.cursors_cached][2] + cache = stat_cursor[stat.conn.cursor_cached_count][2] stat_cursor.close() return cache diff --git a/src/third_party/wiredtiger/test/suite/test_stat08.py b/src/third_party/wiredtiger/test/suite/test_stat08.py new file mode 100644 index 00000000000..96f0b29b04e --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_stat08.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2018 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest + +# test_stat08.py +# Session statistics for bytes read into the cache. +class test_stat08(wttest.WiredTigerTestCase): + + nentries = 350000 + conn_config = 'cache_size=50MB,statistics=(all)' + entry_value = "abcde" * 40 + BYTES_READ = 4000 + READ_TIME = 4003 + session_stats = { BYTES_READ : "session: bytes read into cache", \ + READ_TIME : "session: page read from disk to cache time (usecs)"} + + def check_stats(self, cur, k): + exp_desc = self.session_stats[k] + cur.set_key(k) + cur.search() + [desc, pvalue, value] = cur.get_values() + self.pr(' stat: \'%s\', \'%s\', \'%s\'' % (desc, pvalue, str(value))) + self.assertEqual(desc, exp_desc ) + if k is self.BYTES_READ or k is self.READ_TIME: + self.assertTrue(value > 0) + + def test_session_stats(self): + self.session = self.conn.open_session() + self.session.create("table:test_stat08", + "key_format=i,value_format=S") + cursor = self.session.open_cursor('table:test_stat08', None, None) + # Write the entries. + for i in range(0, self.nentries): + cursor[i] = self.entry_value + cursor.reset() + + # Read the entries. + i = 0 + for key, value in cursor: + i = i + 1 + cursor.reset() + + # Now check the session statistics for bytes read into the cache. + stat_cur = self.session.open_cursor('statistics:session', None, None) + for k in self.session_stats: + self.check_stats(stat_cur, k) + + # Session stats cursor reset should set all the stats values to zero. + stat_cur.reset() + while stat_cur.next() == 0: + [desc, pvalue, value] = stat_cur.get_values() + self.assertTrue(value == 0) + +if __name__ == '__main__': + wttest.run() |