diff options
Diffstat (limited to 'src/third_party/wiredtiger/dist')
| -rw-r--r-- | src/third_party/wiredtiger/dist/api_data.py | 25 | ||||
| -rw-r--r-- | src/third_party/wiredtiger/dist/filelist | 5 | ||||
| -rw-r--r-- | src/third_party/wiredtiger/dist/flags.py | 10 | ||||
| -rw-r--r-- | src/third_party/wiredtiger/dist/s_define.list | 6 | ||||
| -rwxr-xr-x | src/third_party/wiredtiger/dist/s_docs | 14 | ||||
| -rwxr-xr-x | src/third_party/wiredtiger/dist/s_export | 2 | ||||
| -rwxr-xr-x | src/third_party/wiredtiger/dist/s_stat | 19 | ||||
| -rw-r--r-- | src/third_party/wiredtiger/dist/s_string.ok | 26 | ||||
| -rwxr-xr-x | src/third_party/wiredtiger/dist/s_style | 2 | ||||
| -rw-r--r-- | src/third_party/wiredtiger/dist/s_void | 9 | ||||
| -rw-r--r-- | src/third_party/wiredtiger/dist/stat.py | 7 | ||||
| -rw-r--r-- | src/third_party/wiredtiger/dist/stat_data.py | 289 | ||||
| -rw-r--r-- | src/third_party/wiredtiger/dist/wtperf_config.py | 25 |
13 files changed, 303 insertions, 136 deletions
diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index 9781e58a807..7affc58a217 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -136,8 +136,8 @@ file_config = format_meta + [ configure a compressor for file blocks. Permitted values are \c "none" or custom compression engine name created with WT_CONNECTION::add_compressor. If WiredTiger has builtin support for - \c "snappy", \c "lz4" or \c "zlib" compression, these names are also - available. See @ref compression for more information'''), + \c "lz4", \c "snappy", \c "zlib" or \c "zstd" compression, these names + are also available. See @ref compression for more information'''), Config('cache_resident', 'false', r''' do not ever evict the object's pages from cache. Not compatible with LSM tables; see @ref tuning_cache_resident for more information''', @@ -183,6 +183,12 @@ file_config = format_meta + [ configure Huffman encoding for values. Permitted values are \c "none", \c "english", \c "utf8<file>" or \c "utf16<file>". See @ref huffman for more information'''), + Config('ignore_in_memory_cache_size', 'false', r''' + allow update and insert operations to proceed even if the cache is + already at capacity. Only valid in conjunction with in-memory + databases. Should be used with caution - this configuration allows + WiredTiger to consume memory over the configured cache limit''', + type='boolean'), Config('internal_key_truncate', 'true', r''' configure internal key truncation, discarding unnecessary trailing bytes on internal keys (ignored for custom @@ -410,13 +416,13 @@ connection_runtime_config = [ Config('eviction_dirty_target', '5', r''' perform eviction in worker threads when the cache contains at least this much dirty content, expressed as a percentage of the total cache - size. Ignored if \c in_memory is \c true''', + size.''', min=1, max=99), Config('eviction_dirty_trigger', '20', r''' trigger application threads to perform eviction when the cache contains at least this much dirty content, expressed as a percentage of the total cache size. This setting only alters behavior if it is lower than - eviction_trigger. Ignored if \c in_memory is \c true''', + eviction_trigger''', min=1, max=99), Config('eviction_target', '80', r''' perform eviction in worker threads when the cache contains at least @@ -496,7 +502,8 @@ connection_runtime_config = [ is used to gather statistics, as well as each time statistics are logged using the \c statistics_log configuration. See @ref statistics for more information''', - type='list', choices=['all', 'fast', 'none', 'clear']), + type='list', + choices=['all', 'cache_walk', 'fast', 'none', 'clear', 'tree_walk']), Config('verbose', '', r''' enable messages for various events. Only available if WiredTiger is configured with --enable-verbose. Options are given as a @@ -563,8 +570,9 @@ wiredtiger_open_log_configuration = [ configure a compressor for log records. Permitted values are \c "none" or custom compression engine name created with WT_CONNECTION::add_compressor. If WiredTiger has builtin support - for \c "snappy", \c "lz4" or \c "zlib" compression, these names - are also available. See @ref compression for more information'''), + for \c "lz4", \c "snappy", \c "zlib" or \c "zstd" compression, + these names are also available. See @ref compression for more + information'''), Config('file_max', '100MB', r''' the maximum size of log files''', min='100KB', max='2GB'), @@ -970,7 +978,8 @@ methods = { gathering them, where appropriate (for example, a cache size statistic is not cleared, while the count of cursor insert operations will be cleared). See @ref statistics for more information''', - type='list', choices=['all', 'fast', 'clear', 'size']), + type='list', + choices=['all', 'cache_walk', 'fast', 'clear', 'size', 'tree_walk']), Config('target', '', r''' if non-empty, backup the list of objects; valid only for a backup data source''', diff --git a/src/third_party/wiredtiger/dist/filelist b/src/third_party/wiredtiger/dist/filelist index 19fa1122a27..fe9a17b7799 100644 --- a/src/third_party/wiredtiger/dist/filelist +++ b/src/third_party/wiredtiger/dist/filelist @@ -48,12 +48,12 @@ src/btree/row_modify.c src/btree/row_srch.c src/cache/cache_las.c src/checksum/arm64/crc32-arm64.c ARM64_HOST -src/checksum/power8/crc32.S POWERPC_HOST +src/checksum/power8/crc32.sx POWERPC_HOST src/checksum/power8/crc32_wrapper.c POWERPC_HOST src/checksum/software/checksum.c src/checksum/x86/crc32-x86.c X86_HOST src/checksum/zseries/crc32-s390x.c ZSERIES_HOST -src/checksum/zseries/crc32le-vx.S ZSERIES_HOST +src/checksum/zseries/crc32le-vx.sx ZSERIES_HOST src/config/config.c src/config/config_api.c src/config/config_check.c @@ -90,6 +90,7 @@ src/cursor/cur_table.c src/evict/evict_file.c src/evict/evict_lru.c src/evict/evict_page.c +src/evict/evict_stat.c src/log/log.c src/log/log_auto.c src/log/log_slot.c diff --git a/src/third_party/wiredtiger/dist/flags.py b/src/third_party/wiredtiger/dist/flags.py index 93b6e0cbbf4..e200f95fba6 100644 --- a/src/third_party/wiredtiger/dist/flags.py +++ b/src/third_party/wiredtiger/dist/flags.py @@ -133,6 +133,16 @@ flags = { 'SESSION_QUIET_CORRUPT_FILE', 'SESSION_SERVER_ASYNC', ], + 'stat' : [ + 'STAT_CLEAR', + 'STAT_JSON', + 'STAT_ON_CLOSE', + 'STAT_TYPE_ALL', + 'STAT_TYPE_CACHE_WALK', + 'STAT_TYPE_FAST', + 'STAT_TYPE_SIZE', + 'STAT_TYPE_TREE_WALK', + ], } flag_cnt = {} # Dictionary [flag] : [reference count] diff --git a/src/third_party/wiredtiger/dist/s_define.list b/src/third_party/wiredtiger/dist/s_define.list index 6a1a32004ea..7b11d665de5 100644 --- a/src/third_party/wiredtiger/dist/s_define.list +++ b/src/third_party/wiredtiger/dist/s_define.list @@ -39,14 +39,18 @@ WT_READ_BARRIER WT_REF_SIZE WT_SESSION_LOCKED_CHECKPOINT WT_SESSION_LOCKED_TURTLE -WT_STATS_FIELD_TO_SLOT +WT_STATS_FIELD_TO_OFFSET WT_STATS_SLOT_ID WT_STAT_CONN_DECRV WT_STAT_DATA_DECRV WT_STAT_DECR WT_STAT_DECRV WT_STAT_DECRV_ATOMIC +WT_STAT_DECRV_ATOMIC_BASE +WT_STAT_DECRV_BASE WT_STAT_INCRV_ATOMIC +WT_STAT_INCRV_ATOMIC_BASE +WT_STAT_INCRV_BASE WT_STAT_WRITE WT_TIMEDIFF_US WT_TRET_ERROR_OK diff --git a/src/third_party/wiredtiger/dist/s_docs b/src/third_party/wiredtiger/dist/s_docs index e2b1d2aed11..b4f449fa093 100755 --- a/src/third_party/wiredtiger/dist/s_docs +++ b/src/third_party/wiredtiger/dist/s_docs @@ -1,7 +1,7 @@ #! /bin/sh t=__wt.$$ -trap 'rm -f $t /tmp/__doxy' 0 1 2 3 13 15 +trap 'rm -f $t' 0 1 2 3 13 15 # Skip this when building release packages: docs are built separately test -n "$WT_RELEASE_BUILD" && exit 0 @@ -30,18 +30,22 @@ wtperf_config() { # The Linux ed command writes line numbers to stderr, redirect both # stdout and stderr to keep things quiet. - cc -o /tmp/__doxy ../bench/wtperf/doxy.c && + # + # The OS X cpp program injects line number output in the middle of lines + # and doesn't stringify #XXX entries; use the -E option to the compiler + # instead. + cat ../bench/wtperf/wtperf_opt.i | + ${CC:-cc} -E -DOPT_DEFINE_DOXYGEN - | python wtperf_config.py > $t (echo '/START_AUTO_GENERATED_WTPERF_CONFIGURATION/+3,/STOP_AUTO_GENERATED_WTPERF_CONFIGURATION/-1d' echo 'i' echo '' echo '.' - echo ".r !/tmp/__doxy" + echo ".r $t" echo 'a' echo '' echo '.' echo 'w' - echo 'q') | ed ../src/docs/wtperf.dox 1>/dev/null 2>/dev/null && - rm -f /tmp/__doxy + echo 'q') | ed ../src/docs/wtperf.dox 1>/dev/null 2>/dev/null } structurechk() diff --git a/src/third_party/wiredtiger/dist/s_export b/src/third_party/wiredtiger/dist/s_export index dc69238b270..b8e42c970f9 100755 --- a/src/third_party/wiredtiger/dist/s_export +++ b/src/third_party/wiredtiger/dist/s_export @@ -26,7 +26,7 @@ check() sort | uniq -u | egrep -v \ - 'zlib_extension_init|lz4_extension_init|snappy_extension_init' > $t + 'lz4_extension_init|snappy_extension_init|zlib_extension_init|zstd_extension_init' > $t test -s $t && { echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" diff --git a/src/third_party/wiredtiger/dist/s_stat b/src/third_party/wiredtiger/dist/s_stat index 935c7e1fb43..5d5937e1833 100755 --- a/src/third_party/wiredtiger/dist/s_stat +++ b/src/third_party/wiredtiger/dist/s_stat @@ -20,6 +20,25 @@ search=`sed \ -e d ../src/include/stat.h | sort` +# There are some fields that are used, but we can't detect it. +cat << UNUSED_STAT_FIELDS +lock_checkpoint_count +lock_checkpoint_wait_application +lock_checkpoint_wait_internal +lock_handle_list_count +lock_handle_list_wait_application +lock_handle_list_wait_internal +lock_metadata_count +lock_metadata_wait_application +lock_metadata_wait_internal +lock_schema_count +lock_schema_wait_application +lock_schema_wait_internal +lock_table_count +lock_table_wait_application +lock_table_wait_internal +UNUSED_STAT_FIELDS + echo "$search" fgrep -who "$search" $l) | sort | uniq -u > $t diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index 1887cbd936f..7cf96aec399 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -18,6 +18,7 @@ ASYNC Addr Ailamaki Alakuijala +Alexandrescu's Alloc Async Athanassoulis @@ -59,6 +60,7 @@ COVERITY CPUs CRC CSV +CStream CURSORs CURSTD CallsCustDate @@ -68,6 +70,7 @@ Checksum Checksums CityHash CloseHandle +Collet Comparator Config Coverity @@ -124,6 +127,7 @@ FORALL FOREACH FS FULLFSYNC +Facebook FindClose FindFirstFile Fixup @@ -165,6 +169,7 @@ INSN INTL ISA ITEMs +ITER InitializeCriticalSectionAndSpinCount Inline Intra @@ -372,6 +377,7 @@ WILLNEED WIREDTIGER WRLSN WRNOLOCK +WTPERF WaitForSingleObject WakeAllConditionVariable Wconditional @@ -395,6 +401,12 @@ WriteFile Wuninitialized Wunused XP +Yann +ZSTD +Zlib +Zlib's +Zstd +Zstd's abcdef abcdefghijklmnopqrstuvwxyz addl @@ -416,6 +428,7 @@ argc args argv asm +assertfmt async asyncopp asyncops @@ -511,6 +524,7 @@ collatorp comparator comparep compat +compressStream concat cond conf @@ -530,6 +544,7 @@ cp cpuid crc create's +createCStream crypto cryptobad csv @@ -622,6 +637,7 @@ emp encodings encryptor encryptors +endStream endian english enqueue @@ -749,6 +765,7 @@ infeasible inflateInit infmt init +initCStream initializers initn initsize @@ -784,6 +801,7 @@ isupper isxdigit iter iteratively +iters jnr jrx json @@ -849,6 +867,7 @@ majorp malloc marshall marshalled +maxCLevel maxcpu maxdbs mbll @@ -989,12 +1008,14 @@ qdown qrrSS qsort quartile +queueable qup rN rS rb rbrace rbracket +rcursor rdonly rduppo readlock @@ -1201,6 +1222,7 @@ waitpid walk's warmup wb +wcursor wiredTiger wiredtiger workFactor @@ -1222,6 +1244,10 @@ xxxx xxxxx xxxxxx zalloc +zf zfree zlib +zlib's +zstd +zstd's zu diff --git a/src/third_party/wiredtiger/dist/s_style b/src/third_party/wiredtiger/dist/s_style index 3860a23b991..8e755224ee2 100755 --- a/src/third_party/wiredtiger/dist/s_style +++ b/src/third_party/wiredtiger/dist/s_style @@ -108,7 +108,7 @@ else ! expr "$f" : 'test/.*' > /dev/null && ! expr "$f" : '.*/utilities/.*' > /dev/null; then if ! expr "$f" : '.*/os_alloc.c' > /dev/null && - egrep '[[:space:]]free[(]|[[:space:]]strdup[(]|[[:space:]]strndup[(]|[[:space:]]malloc[(]|[[:space:]]calloc[(]|[[:space:]]realloc[(]' $f > $t; then + egrep '[[:space:]]free[(]|[[:space:]]strdup[(]|[[:space:]]strndup[(]|[[:space:]]malloc[(]|[[:space:]]calloc[(]|[[:space:]]realloc[(]|[[:space:]]sprintf[(]' $f > $t; then test -s $t && { echo "$f: call to illegal function" cat $t diff --git a/src/third_party/wiredtiger/dist/s_void b/src/third_party/wiredtiger/dist/s_void index f7bfbcc7e8e..e5e9f97c0b7 100644 --- a/src/third_party/wiredtiger/dist/s_void +++ b/src/third_party/wiredtiger/dist/s_void @@ -96,10 +96,13 @@ func_ok() -e '/int wiredtiger_extension_init$/d' \ -e '/int wiredtiger_extension_terminate$/d' \ -e '/int wiredtiger_pack_close$/d' \ - -e '/int wt_snappy_pre_size$/d' \ - -e '/int wt_snappy_terminate$/d' \ + -e '/int snappy_pre_size$/d' \ + -e '/int snappy_terminate$/d' \ -e '/int zlib_error$/d' \ - -e '/int zlib_terminate$/d' + -e '/int zlib_terminate$/d' \ + -e '/int zstd_error$/d' \ + -e '/int zstd_pre_size$/d' \ + -e '/int zstd_terminate$/d' } # Complain about functions which return an "int" but which don't return except diff --git a/src/third_party/wiredtiger/dist/stat.py b/src/third_party/wiredtiger/dist/stat.py index c3c85bbe9b4..e42585c1b8c 100644 --- a/src/third_party/wiredtiger/dist/stat.py +++ b/src/third_party/wiredtiger/dist/stat.py @@ -42,8 +42,11 @@ compare_srcfile(tmp_file, '../src/include/stat.h') def print_defines_one(capname, base, stats): for v, l in enumerate(stats, base): desc = l.desc - if 'all_only' in l.flags: - desc += ', only reported if statistics=all is set' + if 'cache_walk' in l.flags: + desc += \ + ', only reported if cache_walk or all statistics are enabled' + if 'tree_walk' in l.flags: + desc += ', only reported if tree_walk or all statistics are enabled' if len(textwrap.wrap(desc, 70)) > 1: f.write('/*!\n') f.write(' * %s\n' % '\n * '.join(textwrap.wrap(desc, 70))) diff --git a/src/third_party/wiredtiger/dist/stat_data.py b/src/third_party/wiredtiger/dist/stat_data.py index 5087afa44dc..bcf5201bd90 100644 --- a/src/third_party/wiredtiger/dist/stat_data.py +++ b/src/third_party/wiredtiger/dist/stat_data.py @@ -9,7 +9,8 @@ # # Data-source statistics are normally aggregated across the set of underlying # objects. Additional optional configuration flags are available: -# all_only Only gets reported when statistics=all set +# cache_walk Only reported when statistics=cache_walk is set +# tree_walk Only reported when statistics=tree_walk is set # max_aggregate Take the maximum value when aggregating statistics # no_clear Value not cleared when statistics cleared # no_scale Don't scale value per second in the logging tool script @@ -46,6 +47,11 @@ class CacheStat(Stat): prefix = 'cache' def __init__(self, name, desc, flags=''): Stat.__init__(self, name, CacheStat.prefix, desc, flags) +class CacheWalkStat(Stat): + prefix = 'cache_walk' + def __init__(self, name, desc, flags=''): + flags += ',cache_walk' + Stat.__init__(self, name, CacheWalkStat.prefix, desc, flags) class CompressStat(Stat): prefix = 'compression' def __init__(self, name, desc, flags=''): @@ -66,6 +72,10 @@ class JoinStat(Stat): prefix = '' # prefix is inserted dynamically def __init__(self, name, desc, flags=''): Stat.__init__(self, name, JoinStat.prefix, desc, flags) +class LockStat(Stat): + prefix = 'lock' + def __init__(self, name, desc, flags=''): + Stat.__init__(self, name, LockStat.prefix, desc, flags) class LogStat(Stat): prefix = 'log' def __init__(self, name, desc, flags=''): @@ -105,11 +115,16 @@ groups['cursor'] = [CursorStat.prefix, SessionStat.prefix] groups['evict'] = [ BlockStat.prefix, CacheStat.prefix, + CacheWalkStat.prefix, ConnStat.prefix, ThreadStat.prefix ] groups['lsm'] = [LSMStat.prefix, TxnStat.prefix] -groups['memory'] = [CacheStat.prefix, ConnStat.prefix, RecStat.prefix] +groups['memory'] = [ + CacheStat.prefix, + CacheWalkStat.prefix, + ConnStat.prefix, + RecStat.prefix] groups['system'] = [ ConnStat.prefix, DhandleStat.prefix, @@ -226,13 +241,32 @@ connection_stats = [ CacheStat('cache_pages_inuse', 'pages currently held in the cache', 'no_clear,no_scale'), CacheStat('cache_pages_requested', 'pages requested from the cache'), CacheStat('cache_read', 'pages read into cache'), + CacheStat('cache_read_app_count', 'application threads page read from disk to cache count'), + CacheStat('cache_read_app_time', 'application threads page read from disk to cache time (usecs)'), CacheStat('cache_read_lookaside', 'pages read into cache requiring lookaside entries'), CacheStat('cache_read_overflow', 'overflow pages read into cache'), CacheStat('cache_write', 'pages written from cache'), + CacheStat('cache_write_app_count', 'application threads page write from cache to disk count'), + CacheStat('cache_write_app_time', 'application threads page write from cache to disk time (usecs)'), CacheStat('cache_write_lookaside', 'page written requiring lookaside records'), CacheStat('cache_write_restore', 'pages written requiring in-memory restoration'), ########################################## + # Cursor operations + ########################################## + CursorStat('cursor_create', 'cursor create calls'), + CursorStat('cursor_insert', 'cursor insert calls'), + CursorStat('cursor_next', 'cursor next calls'), + CursorStat('cursor_prev', 'cursor prev calls'), + CursorStat('cursor_remove', 'cursor remove calls'), + CursorStat('cursor_reset', 'cursor reset calls'), + CursorStat('cursor_restart', 'cursor restarted searches'), + CursorStat('cursor_search', 'cursor search calls'), + CursorStat('cursor_search_near', 'cursor search near calls'), + CursorStat('cursor_truncate', 'truncate calls'), + CursorStat('cursor_update', 'cursor update calls'), + + ########################################## # Dhandle statistics ########################################## DhandleStat('dh_conn_handle_count', 'connection data handles currently active', 'no_clear,no_scale'), @@ -245,6 +279,25 @@ connection_stats = [ DhandleStat('dh_sweeps', 'connection sweeps'), ########################################## + # Locking statistics + ########################################## + LockStat('lock_checkpoint_count', 'checkpoint lock acquisitions'), + LockStat('lock_checkpoint_wait_application', 'checkpoint lock application thread wait time (usecs)'), + LockStat('lock_checkpoint_wait_internal', 'checkpoint lock internal thread wait time (usecs)'), + LockStat('lock_handle_list_count', 'handle-list lock acquisitions'), + LockStat('lock_handle_list_wait_application', 'handle-list lock application thread wait time (usecs)'), + LockStat('lock_handle_list_wait_internal', 'handle-list lock internal thread wait time (usecs)'), + LockStat('lock_metadata_count', 'metadata lock acquisitions'), + LockStat('lock_metadata_wait_application', 'metadata lock application thread wait time (usecs)'), + LockStat('lock_metadata_wait_internal', 'metadata lock internal thread wait time (usecs)'), + LockStat('lock_schema_count', 'schema lock acquisitions'), + LockStat('lock_schema_wait_application', 'schema lock application thread wait time (usecs)'), + LockStat('lock_schema_wait_internal', 'schema lock internal thread wait time (usecs)'), + LockStat('lock_table_count', 'table lock acquisitions'), + LockStat('lock_table_wait_application', 'table lock application thread time waiting for the table lock (usecs)'), + LockStat('lock_table_wait_internal', 'table lock internal thread time waiting for the table lock (usecs)'), + + ########################################## # Logging statistics ########################################## LogStat('log_buffer_size', 'total log buffer size', 'no_clear,no_scale,size'), @@ -286,41 +339,6 @@ connection_stats = [ LogStat('log_zero_fills', 'log files manually zero-filled'), ########################################## - # Reconciliation statistics - ########################################## - RecStat('rec_page_delete', 'pages deleted'), - RecStat('rec_page_delete_fast', 'fast-path pages deleted'), - RecStat('rec_pages', 'page reconciliation calls'), - RecStat('rec_pages_eviction', 'page reconciliation calls for eviction'), - RecStat('rec_split_stashed_bytes', 'split bytes currently awaiting free', 'no_clear,no_scale,size'), - RecStat('rec_split_stashed_objects', 'split objects currently awaiting free', 'no_clear,no_scale'), - - ########################################## - # Transaction statistics - ########################################## - TxnStat('txn_begin', 'transaction begins'), - TxnStat('txn_checkpoint', 'transaction checkpoints'), - TxnStat('txn_checkpoint_fsync_post', 'transaction fsync calls for checkpoint after allocating the transaction ID'), - TxnStat('txn_checkpoint_fsync_post_duration', 'transaction fsync duration for checkpoint after allocating the transaction ID (usecs)', 'no_clear,no_scale'), - TxnStat('txn_checkpoint_generation', 'transaction checkpoint generation', 'no_clear,no_scale'), - TxnStat('txn_checkpoint_running', 'transaction checkpoint currently running', 'no_clear,no_scale'), - TxnStat('txn_checkpoint_scrub_target', 'transaction checkpoint scrub dirty target', 'no_clear,no_scale'), - TxnStat('txn_checkpoint_scrub_time', 'transaction checkpoint scrub time (msecs)', 'no_clear,no_scale'), - TxnStat('txn_checkpoint_time_max', 'transaction checkpoint max time (msecs)', 'no_clear,no_scale'), - TxnStat('txn_checkpoint_time_min', 'transaction checkpoint min time (msecs)', 'no_clear,no_scale'), - TxnStat('txn_checkpoint_time_recent', 'transaction checkpoint most recent time (msecs)', 'no_clear,no_scale'), - TxnStat('txn_checkpoint_time_total', 'transaction checkpoint total time (msecs)', 'no_clear,no_scale'), - TxnStat('txn_commit', 'transactions committed'), - TxnStat('txn_fail_cache', 'transaction failures due to cache overflow'), - TxnStat('txn_pinned_checkpoint_range', 'transaction range of IDs currently pinned by a checkpoint', 'no_clear,no_scale'), - TxnStat('txn_pinned_range', 'transaction range of IDs currently pinned', 'no_clear,no_scale'), - TxnStat('txn_pinned_snapshot_range', 'transaction range of IDs currently pinned by named snapshots', 'no_clear,no_scale'), - TxnStat('txn_rollback', 'transactions rolled back'), - TxnStat('txn_snapshots_created', 'number of named snapshots created'), - TxnStat('txn_snapshots_dropped', 'number of named snapshots dropped'), - TxnStat('txn_sync', 'transaction sync calls'), - - ########################################## # LSM statistics ########################################## LSMStat('lsm_checkpoint_throttle', 'sleep for LSM checkpoint throttle'), @@ -335,6 +353,16 @@ connection_stats = [ LSMStat('lsm_work_units_done', 'tree maintenance operations executed'), ########################################## + # Reconciliation statistics + ########################################## + RecStat('rec_page_delete', 'pages deleted'), + RecStat('rec_page_delete_fast', 'fast-path pages deleted'), + RecStat('rec_pages', 'page reconciliation calls'), + RecStat('rec_pages_eviction', 'page reconciliation calls for eviction'), + RecStat('rec_split_stashed_bytes', 'split bytes currently awaiting free', 'no_clear,no_scale,size'), + RecStat('rec_split_stashed_objects', 'split objects currently awaiting free', 'no_clear,no_scale'), + + ########################################## # Session operations ########################################## SessionStat('session_cursor_open', 'open cursor count', 'no_clear,no_scale'), @@ -357,21 +385,6 @@ connection_stats = [ SessionStat('session_table_verify_success', 'table verify successful calls', 'no_clear,no_scale'), ########################################## - # Total cursor operations - ########################################## - CursorStat('cursor_create', 'cursor create calls'), - CursorStat('cursor_insert', 'cursor insert calls'), - CursorStat('cursor_next', 'cursor next calls'), - CursorStat('cursor_prev', 'cursor prev calls'), - CursorStat('cursor_remove', 'cursor remove calls'), - CursorStat('cursor_reset', 'cursor reset calls'), - CursorStat('cursor_restart', 'cursor restarted searches'), - CursorStat('cursor_search', 'cursor search calls'), - CursorStat('cursor_search_near', 'cursor search near calls'), - CursorStat('cursor_truncate', 'truncate calls'), - CursorStat('cursor_update', 'cursor update calls'), - - ########################################## # Thread Count statistics ########################################## ThreadStat('thread_fsync_active', 'active filesystem fsync calls','no_clear,no_scale'), @@ -379,8 +392,36 @@ connection_stats = [ ThreadStat('thread_write_active', 'active filesystem write calls','no_clear,no_scale'), ########################################## + # Transaction statistics + ########################################## + TxnStat('txn_begin', 'transaction begins'), + TxnStat('txn_checkpoint', 'transaction checkpoints'), + TxnStat('txn_checkpoint_fsync_post', 'transaction fsync calls for checkpoint after allocating the transaction ID'), + TxnStat('txn_checkpoint_fsync_post_duration', 'transaction fsync duration for checkpoint after allocating the transaction ID (usecs)', 'no_clear,no_scale'), + TxnStat('txn_checkpoint_generation', 'transaction checkpoint generation', 'no_clear,no_scale'), + TxnStat('txn_checkpoint_running', 'transaction checkpoint currently running', 'no_clear,no_scale'), + TxnStat('txn_checkpoint_scrub_target', 'transaction checkpoint scrub dirty target', 'no_clear,no_scale'), + TxnStat('txn_checkpoint_scrub_time', 'transaction checkpoint scrub time (msecs)', 'no_clear,no_scale'), + TxnStat('txn_checkpoint_skipped', 'transaction checkpoints skipped because database was clean'), + TxnStat('txn_checkpoint_time_max', 'transaction checkpoint max time (msecs)', 'no_clear,no_scale'), + TxnStat('txn_checkpoint_time_min', 'transaction checkpoint min time (msecs)', 'no_clear,no_scale'), + TxnStat('txn_checkpoint_time_recent', 'transaction checkpoint most recent time (msecs)', 'no_clear,no_scale'), + TxnStat('txn_checkpoint_time_total', 'transaction checkpoint total time (msecs)', 'no_clear,no_scale'), + TxnStat('txn_commit', 'transactions committed'), + TxnStat('txn_fail_cache', 'transaction failures due to cache overflow'), + TxnStat('txn_pinned_checkpoint_range', 'transaction range of IDs currently pinned by a checkpoint', 'no_clear,no_scale'), + TxnStat('txn_pinned_range', 'transaction range of IDs currently pinned', 'no_clear,no_scale'), + TxnStat('txn_pinned_snapshot_range', 'transaction range of IDs currently pinned by named snapshots', 'no_clear,no_scale'), + TxnStat('txn_rollback', 'transactions rolled back'), + TxnStat('txn_snapshots_created', 'number of named snapshots created'), + TxnStat('txn_snapshots_dropped', 'number of named snapshots dropped'), + TxnStat('txn_sync', 'transaction sync calls'), + + ########################################## # Yield statistics ########################################## + YieldStat('application_cache_time', 'application thread time waiting for cache (usecs)'), + YieldStat('application_evict_time', 'application thread time evicting (usecs)'), YieldStat('page_busy_blocked', 'page acquire busy blocked'), YieldStat('page_forcible_evict_blocked', 'page acquire eviction blocked'), YieldStat('page_locked_blocked', 'page acquire locked blocked'), @@ -395,41 +436,30 @@ connection_stats = sorted(connection_stats, key=attrgetter('desc')) ########################################## dsrc_stats = [ ########################################## - # Session operations - ########################################## - SessionStat('session_compact', 'object compaction'), - SessionStat('session_cursor_open', 'open cursor count', 'no_clear,no_scale'), - - ########################################## - # Cursor operations + # Block manager statistics ########################################## - CursorStat('cursor_create', 'create calls'), - CursorStat('cursor_insert', 'insert calls'), - CursorStat('cursor_insert_bulk', 'bulk-loaded cursor-insert calls'), - CursorStat('cursor_insert_bytes', 'cursor-insert key and value bytes inserted', 'size'), - CursorStat('cursor_next', 'next calls'), - CursorStat('cursor_prev', 'prev calls'), - CursorStat('cursor_remove', 'remove calls'), - CursorStat('cursor_remove_bytes', 'cursor-remove key bytes removed', 'size'), - CursorStat('cursor_reset', 'reset calls'), - CursorStat('cursor_restart', 'restarted searches'), - CursorStat('cursor_search', 'search calls'), - CursorStat('cursor_search_near', 'search near calls'), - CursorStat('cursor_truncate', 'truncate calls'), - CursorStat('cursor_update', 'update calls'), - CursorStat('cursor_update_bytes', 'cursor-update value bytes updated', 'size'), + BlockStat('allocation_size', 'file allocation unit size', 'max_aggregate,no_scale,size'), + BlockStat('block_alloc', 'blocks allocated'), + BlockStat('block_checkpoint_size', 'checkpoint size', 'no_scale,size'), + BlockStat('block_extension', 'allocations requiring file extension'), + BlockStat('block_free', 'blocks freed'), + BlockStat('block_magic', 'file magic number', 'max_aggregate,no_scale'), + BlockStat('block_major', 'file major version number', 'max_aggregate,no_scale'), + BlockStat('block_minor', 'minor version number', 'max_aggregate,no_scale'), + BlockStat('block_reuse_bytes', 'file bytes available for reuse', 'no_scale,size'), + BlockStat('block_size', 'file size in bytes', 'no_scale,size'), ########################################## # Btree statistics ########################################## BtreeStat('btree_checkpoint_generation', 'btree checkpoint generation', 'no_clear,no_scale'), - BtreeStat('btree_column_deleted', 'column-store variable-size deleted values', 'no_scale,all_only'), - BtreeStat('btree_column_fix', 'column-store fixed-size leaf pages', 'no_scale,all_only'), - BtreeStat('btree_column_internal', 'column-store internal pages', 'no_scale,all_only'), - BtreeStat('btree_column_rle', 'column-store variable-size RLE encoded values', 'no_scale,all_only'), - BtreeStat('btree_column_variable', 'column-store variable-size leaf pages', 'no_scale,all_only'), + BtreeStat('btree_column_deleted', 'column-store variable-size deleted values', 'no_scale,tree_walk'), + BtreeStat('btree_column_fix', 'column-store fixed-size leaf pages', 'no_scale,tree_walk'), + BtreeStat('btree_column_internal', 'column-store internal pages', 'no_scale,tree_walk'), + BtreeStat('btree_column_rle', 'column-store variable-size RLE encoded values', 'no_scale,tree_walk'), + BtreeStat('btree_column_variable', 'column-store variable-size leaf pages', 'no_scale,tree_walk'), BtreeStat('btree_compact_rewrite', 'pages rewritten by compaction'), - BtreeStat('btree_entries', 'number of key/value pairs', 'no_scale,all_only'), + BtreeStat('btree_entries', 'number of key/value pairs', 'no_scale,tree_walk'), BtreeStat('btree_fixed_len', 'fixed-record size', 'max_aggregate,no_scale,size'), BtreeStat('btree_maximum_depth', 'maximum tree depth', 'max_aggregate,no_scale'), BtreeStat('btree_maxintlkey', 'maximum internal page key size', 'max_aggregate,no_scale,size'), @@ -437,39 +467,9 @@ dsrc_stats = [ BtreeStat('btree_maxleafkey', 'maximum leaf page key size', 'max_aggregate,no_scale,size'), BtreeStat('btree_maxleafpage', 'maximum leaf page size', 'max_aggregate,no_scale,size'), BtreeStat('btree_maxleafvalue', 'maximum leaf page value size', 'max_aggregate,no_scale,size'), - BtreeStat('btree_overflow', 'overflow pages', 'no_scale,all_only'), - BtreeStat('btree_row_internal', 'row-store internal pages', 'no_scale,all_only'), - BtreeStat('btree_row_leaf', 'row-store leaf pages', 'no_scale,all_only'), - - ########################################## - # LSM statistics - ########################################## - LSMStat('bloom_count', 'bloom filters in the LSM tree', 'no_scale'), - LSMStat('bloom_false_positive', 'bloom filter false positives'), - LSMStat('bloom_hit', 'bloom filter hits'), - LSMStat('bloom_miss', 'bloom filter misses'), - LSMStat('bloom_page_evict', 'bloom filter pages evicted from cache'), - LSMStat('bloom_page_read', 'bloom filter pages read into cache'), - LSMStat('bloom_size', 'total size of bloom filters', 'no_scale,size'), - LSMStat('lsm_checkpoint_throttle', 'sleep for LSM checkpoint throttle'), - LSMStat('lsm_chunk_count', 'chunks in the LSM tree', 'no_scale'), - LSMStat('lsm_generation_max', 'highest merge generation in the LSM tree', 'max_aggregate,no_scale'), - LSMStat('lsm_lookup_no_bloom', 'queries that could have benefited from a Bloom filter that did not exist'), - LSMStat('lsm_merge_throttle', 'sleep for LSM merge throttle'), - - ########################################## - # Block manager statistics - ########################################## - BlockStat('allocation_size', 'file allocation unit size', 'max_aggregate,no_scale,size'), - BlockStat('block_alloc', 'blocks allocated'), - BlockStat('block_checkpoint_size', 'checkpoint size', 'no_scale,size'), - BlockStat('block_extension', 'allocations requiring file extension'), - BlockStat('block_free', 'blocks freed'), - BlockStat('block_magic', 'file magic number', 'max_aggregate,no_scale'), - BlockStat('block_major', 'file major version number', 'max_aggregate,no_scale'), - BlockStat('block_minor', 'minor version number', 'max_aggregate,no_scale'), - BlockStat('block_reuse_bytes', 'file bytes available for reuse', 'no_scale,size'), - BlockStat('block_size', 'file size in bytes', 'no_scale,size'), + BtreeStat('btree_overflow', 'overflow pages', 'no_scale,tree_walk'), + BtreeStat('btree_row_internal', 'row-store internal pages', 'no_scale,tree_walk'), + BtreeStat('btree_row_leaf', 'row-store leaf pages', 'no_scale,tree_walk'), ########################################## # Cache and eviction statistics @@ -498,6 +498,28 @@ dsrc_stats = [ CacheStat('cache_write_restore', 'pages written requiring in-memory restoration'), ########################################## + # Cache content statistics + ########################################## + CacheWalkStat('cache_state_avg_written_size', 'Average on-disk page image size seen', 'no_clear,no_scale'), + CacheWalkStat('cache_state_gen_avg_gap', 'Average difference between current eviction generation when the page was last considered', 'no_clear,no_scale'), + CacheWalkStat('cache_state_gen_current', 'Current eviction generation', 'no_clear,no_scale'), + CacheWalkStat('cache_state_gen_max_gap', 'Maximum difference between current eviction generation when the page was last considered', 'no_clear,no_scale'), + CacheWalkStat('cache_state_max_pagesize', 'Maximum page size seen', 'no_clear,no_scale'), + CacheWalkStat('cache_state_memory', 'Pages created in memory and never written', 'no_clear,no_scale'), + CacheWalkStat('cache_state_min_written_size', 'Minimum on-disk page image size seen', 'no_clear,no_scale'), + CacheWalkStat('cache_state_not_queueable', 'Pages that could not be queued for eviction', 'no_clear,no_scale'), + CacheWalkStat('cache_state_pages', 'Total number of pages currently in cache', 'no_clear,no_scale'), + CacheWalkStat('cache_state_pages_clean', 'Clean pages currently in cache', 'no_clear,no_scale'), + CacheWalkStat('cache_state_pages_dirty', 'Dirty pages currently in cache', 'no_clear,no_scale'), + CacheWalkStat('cache_state_pages_internal', 'Internal pages currently in cache', 'no_clear,no_scale'), + CacheWalkStat('cache_state_pages_leaf', 'Leaf pages currently in cache', 'no_clear,no_scale'), + CacheWalkStat('cache_state_queued', 'Pages currently queued for eviction', 'no_clear,no_scale'), + CacheWalkStat('cache_state_refs_skipped', 'Refs skipped during cache traversal', 'no_clear,no_scale'), + CacheWalkStat('cache_state_root_entries', 'Entries in the root page', 'no_clear,no_scale'), + CacheWalkStat('cache_state_root_size', 'Size of the root page', 'no_clear,no_scale'), + CacheWalkStat('cache_state_smaller_alloc_size', 'On-disk page image sizes smaller than a single allocation unit', 'no_clear,no_scale'), + + ########################################## # Compression statistics ########################################## CompressStat('compress_raw_fail', 'raw compression call failed, no additional data available'), @@ -509,6 +531,41 @@ dsrc_stats = [ CompressStat('compress_write_too_small', 'page written was too small to compress'), ########################################## + # Cursor operations + ########################################## + CursorStat('cursor_create', 'create calls'), + CursorStat('cursor_insert', 'insert calls'), + CursorStat('cursor_insert_bulk', 'bulk-loaded cursor-insert calls'), + CursorStat('cursor_insert_bytes', 'cursor-insert key and value bytes inserted', 'size'), + CursorStat('cursor_next', 'next calls'), + CursorStat('cursor_prev', 'prev calls'), + CursorStat('cursor_remove', 'remove calls'), + CursorStat('cursor_remove_bytes', 'cursor-remove key bytes removed', 'size'), + CursorStat('cursor_reset', 'reset calls'), + CursorStat('cursor_restart', 'restarted searches'), + CursorStat('cursor_search', 'search calls'), + CursorStat('cursor_search_near', 'search near calls'), + CursorStat('cursor_truncate', 'truncate calls'), + CursorStat('cursor_update', 'update calls'), + CursorStat('cursor_update_bytes', 'cursor-update value bytes updated', 'size'), + + ########################################## + # LSM statistics + ########################################## + LSMStat('bloom_count', 'bloom filters in the LSM tree', 'no_scale'), + LSMStat('bloom_false_positive', 'bloom filter false positives'), + LSMStat('bloom_hit', 'bloom filter hits'), + LSMStat('bloom_miss', 'bloom filter misses'), + LSMStat('bloom_page_evict', 'bloom filter pages evicted from cache'), + LSMStat('bloom_page_read', 'bloom filter pages read into cache'), + LSMStat('bloom_size', 'total size of bloom filters', 'no_scale,size'), + LSMStat('lsm_checkpoint_throttle', 'sleep for LSM checkpoint throttle'), + LSMStat('lsm_chunk_count', 'chunks in the LSM tree', 'no_scale'), + LSMStat('lsm_generation_max', 'highest merge generation in the LSM tree', 'max_aggregate,no_scale'), + LSMStat('lsm_lookup_no_bloom', 'queries that could have benefited from a Bloom filter that did not exist'), + LSMStat('lsm_merge_throttle', 'sleep for LSM merge throttle'), + + ########################################## # Reconciliation statistics ########################################## RecStat('rec_dictionary', 'dictionary matches'), @@ -527,6 +584,12 @@ dsrc_stats = [ RecStat('rec_suffix_compression', 'internal page key bytes discarded using suffix compression', 'size'), ########################################## + # Session operations + ########################################## + SessionStat('session_compact', 'object compaction'), + SessionStat('session_cursor_open', 'open cursor count', 'no_clear,no_scale'), + + ########################################## # Transaction statistics ########################################## TxnStat('txn_update_conflict', 'update conflicts'), diff --git a/src/third_party/wiredtiger/dist/wtperf_config.py b/src/third_party/wiredtiger/dist/wtperf_config.py new file mode 100644 index 00000000000..72256ed5527 --- /dev/null +++ b/src/third_party/wiredtiger/dist/wtperf_config.py @@ -0,0 +1,25 @@ +# Output a doxgen version of the wtperf configuration options. +import string, sys + +for line in sys.stdin: + if not line.startswith('OPTION '): + continue + + line = line.replace('OPTION ', '') + v = line.split('",') + v[0] = v[0].replace('"', '').strip() + v[1] = v[1].replace('"', '').strip() + v[2] = v[2].replace('"', '').strip() + v[3] = v[3].replace('"', '').strip() + + if v[3] == 'boolean': + if v[2] == '0': + d = 'false' + else: + d = 'true' + elif v[3] == 'string': + d = '"' + v[2] + '"' + else: + d = v[2] + print '@par ' + v[0] + ' (' + v[3] + ', default=' + d + ')' + print v[1] |