diff options
| author | Luke Chen <luke.chen@mongodb.com> | 2018-12-21 13:42:20 +1100 |
|---|---|---|
| committer | Luke Chen <luke.chen@mongodb.com> | 2018-12-21 13:42:20 +1100 |
| commit | 034069139b45ed23cffcf3313ed2029f09a6ee36 (patch) | |
| tree | 844f4362c787130fa05a2bc91cc77285f75a82ee | |
| parent | 84d5a90f500bdf851bcdaa43a21435411c78975b (diff) | |
| download | mongo-034069139b45ed23cffcf3313ed2029f09a6ee36.tar.gz | |
Import wiredtiger: bedf230af338faa9b9ba8741b9a2e5f36353e2f0 from branch mongodb-4.0
ref: b51da4ed90..bedf230af3
for: 4.0.6
WT-4192 Remove WiredTiger raw compression support
WT-4280 Add debugging to know which session has a hazard pointer
WT-4319 Improvements to csuite tests
WT-4331 Further extend max wait time for test_bug019.py
WT-4383 Update session statistics to reflect the operation statistics.
WT-4384 Add documentation about session statistics cursor
WT-4393 Document cursor behaviour for read committed isolation
WT-4410 Split 'unit-test' task to reduce Evergreen Ubuntu build variant runtime
WT-4417 Make os_cache_max and os_cache_dirty_max reconfigurable
WT-4421 Add a way to calculate modify operations
WT-4430 Fix race between prepare and page instantiate with fast truncate
WT-4433 Enable dynamic compression for snappy
WT-4434 Modify zstd compression level from 3 to 6
WT-4438 Use more accurate statistics for cursor cache totals
WT-4442 Add the ability to duplicate a backup cursor
WT-4454 Auto-generate Evergreen configuration when adding new 'make check' or csuite tests
WT-4455 test_wt4156_metadata_salvage with HAVE_ATTACH fails on zSeries
WT-4457 Add a maximum of dirty system buffers for the logging subsystem
WT-4463 Reduce runtime for csuite handle locks testing
WT-4464 In debug output row-store internal page keys may not format correctly
WT-4469 Coverity #105148: redundant test
WT-4470 Minimize testing that requires TESTUTIL_ENABLE_LONG_TESTS
WT-4479 Remove schema lock wait assertion from the test
WT-4480 Fix duplicate backup cursor and archiving check
WT-4481 Evergreen script cleanups
WT-4482 Lint
129 files changed, 2703 insertions, 2180 deletions
diff --git a/src/third_party/wiredtiger/api/leveldb/Makefile.am b/src/third_party/wiredtiger/api/leveldb/Makefile.am index 2cfd9d945a5..34f2f97ec36 100644 --- a/src/third_party/wiredtiger/api/leveldb/Makefile.am +++ b/src/third_party/wiredtiger/api/leveldb/Makefile.am @@ -78,4 +78,4 @@ leveldb_test_LDADD = libwiredtiger_leveldb.la TESTS = $(noinst_PROGRAMS) clean-local: - rm -rf WTLDB_HOME + rm -rf WTLDB_HOME core.* *.core diff --git a/src/third_party/wiredtiger/bench/workgen/runner/compress_ratio.py b/src/third_party/wiredtiger/bench/workgen/runner/compress_ratio.py index 2c5552bfa5d..44320d75e29 100644 --- a/src/third_party/wiredtiger/bench/workgen/runner/compress_ratio.py +++ b/src/third_party/wiredtiger/bench/workgen/runner/compress_ratio.py @@ -82,11 +82,12 @@ conn_config="create,cache_size=2GB,session_max=1000,eviction=(threads_min=4,thre table_config="allocation_size=4k,memory_page_max=10MB,prefix_compression=false,split_pct=90,leaf_page_max=32k,internal_page_max=16k,type=file" compression_opts = { "none" : "block_compressor=none", - "zlib_noraw" : "block_compressor=zlib-noraw", - "zlib_noraw_onepage" : "block_compressor=zlib-noraw,memory_page_image_max=32k", - "zlib_noraw_tenpage" : "block_compressor=zlib-noraw,memory_page_image_max=320k", - "zlib_raw" : "block_compressor=zlib", + "lz4" : "block_compressor=lz4" "snappy" : "block_compressor=snappy" + "zlib" : "block_compressor=zlib", + "zlib_onepage" : "block_compressor=zlib,memory_page_image_max=32k", + "zlib_tenpage" : "block_compressor=zlib,memory_page_image_max=320k", + "zstd" : "block_compressor=zstd" } #conn_config += extensions_config(['compressors/snappy']) conn = wiredtiger_open("WT_TEST", conn_config) diff --git a/src/third_party/wiredtiger/bench/wtperf/Makefile.am b/src/third_party/wiredtiger/bench/wtperf/Makefile.am index 57792e3887f..5690d03224e 100644 --- a/src/third_party/wiredtiger/bench/wtperf/Makefile.am +++ b/src/third_party/wiredtiger/bench/wtperf/Makefile.am @@ -18,4 +18,4 @@ AM_TESTS_ENVIRONMENT = rm -rf WT_TEST ; mkdir WT_TEST ; TESTS_ENVIRONMENT = $(AM_TESTS_ENVIRONMENT) clean-local: - rm -rf WT_TEST *.core + rm -rf WT_TEST core.* *.core diff --git a/src/third_party/wiredtiger/build_win/wiredtiger.def b/src/third_party/wiredtiger/build_win/wiredtiger.def index 79fa84a11e0..71c52bd81af 100644 --- a/src/third_party/wiredtiger/build_win/wiredtiger.def +++ b/src/third_party/wiredtiger/build_win/wiredtiger.def @@ -1,5 +1,6 @@ LIBRARY WIREDTIGER EXPORTS + wiredtiger_calc_modify wiredtiger_config_parser_open wiredtiger_config_validate wiredtiger_crc32c_func diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index 08a322a66e7..13d47d72d07 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -153,10 +153,6 @@ file_runtime_config = common_runtime_config + [ option leads to an advisory call to an appropriate operating system API where available''', choices=['none', 'random', 'sequential']), - Config('cache_resident', 'false', r''' - do not ever evict the object's pages from cache. Not compatible with - LSM tables; see @ref tuning_cache_resident for more information''', - type='boolean'), Config('assert', '', r''' enable enhanced checking. ''', type='category', subconfig= [ @@ -173,6 +169,10 @@ file_runtime_config = common_runtime_config + [ if mixed read use is allowed.''', choices=['always','never','none']) ], undoc=True), + Config('cache_resident', 'false', r''' + do not ever evict the object's pages from cache. Not compatible with + LSM tables; see @ref tuning_cache_resident for more information''', + type='boolean'), Config('log', '', r''' the transaction log configuration for this object. Only valid if log is enabled in ::wiredtiger_open''', @@ -181,6 +181,17 @@ file_runtime_config = common_runtime_config + [ if false, this object has checkpoint-level durability''', type='boolean'), ]), + Config('os_cache_max', '0', r''' + maximum system buffer cache usage, in bytes. If non-zero, evict + object blocks from the system buffer cache after that many bytes + from this object are read or written into the buffer cache''', + min=0), + Config('os_cache_dirty_max', '0', r''' + maximum dirty system buffer cache usage, in bytes. If non-zero, + schedule writes for dirty blocks belonging to this object in the + system buffer cache after that many bytes from this object are + written into the buffer cache''', + min=0), ] # Per-file configuration @@ -318,17 +329,6 @@ file_config = format_meta + file_runtime_config + [ for pages to be temporarily larger than this value. This setting is ignored for LSM trees, see \c chunk_size''', min='512B', max='10TB'), - Config('os_cache_max', '0', r''' - maximum system buffer cache usage, in bytes. If non-zero, evict - object blocks from the system buffer cache after that many bytes - from this object are read or written into the buffer cache''', - min=0), - Config('os_cache_dirty_max', '0', r''' - maximum dirty system buffer cache usage, in bytes. If non-zero, - schedule writes for dirty blocks belonging to this object in the - system buffer cache after that many bytes from this object are - written into the buffer cache''', - min=0), Config('prefix_compression', 'false', r''' configure prefix compression on row-store leaf pages''', type='boolean'), @@ -676,6 +676,13 @@ log_configuration_common = [ Config('archive', 'true', r''' automatically archive unneeded log files''', type='boolean'), + Config('os_cache_dirty_pct', '0', r''' + maximum dirty system buffer cache usage, as a percentage of the + log's \c file_max. If non-zero, schedule writes for dirty blocks + belonging to the log in the system buffer cache after that percentage + of the log has been written into the buffer cache without an + intervening file sync.''', + min='0', max='100'), Config('prealloc', 'true', r''' pre-allocate log files''', type='boolean'), diff --git a/src/third_party/wiredtiger/dist/filelist b/src/third_party/wiredtiger/dist/filelist index 79590313b89..1bbeeb3c7a3 100644 --- a/src/third_party/wiredtiger/dist/filelist +++ b/src/third_party/wiredtiger/dist/filelist @@ -62,6 +62,7 @@ src/config/config_collapse.c src/config/config_def.c src/config/config_ext.c src/config/config_upgrade.c +src/conn/api_calc_modify.c src/conn/api_strerror.c src/conn/api_version.c src/conn/conn_api.c diff --git a/src/third_party/wiredtiger/dist/function.py b/src/third_party/wiredtiger/dist/function.py index 446f16dc47f..7c632b4e227 100644 --- a/src/third_party/wiredtiger/dist/function.py +++ b/src/third_party/wiredtiger/dist/function.py @@ -49,11 +49,14 @@ types = [ 'TEST_', 'WT_', 'wt_', + 'DWORD', 'double', 'float', 'intmax_t', 'intptr_t', + 'clock_t', 'pid_t', + 'pthread_t', 'size_t', 'ssize_t', 'time_t', diff --git a/src/third_party/wiredtiger/dist/s_all b/src/third_party/wiredtiger/dist/s_all index a0ea9918206..51ea98ab154 100755 --- a/src/third_party/wiredtiger/dist/s_all +++ b/src/third_party/wiredtiger/dist/s_all @@ -86,6 +86,7 @@ run "sh ./s_style" COMMANDS=" 2>&1 ./s_define > ${t_pfx}s_define 2>&1 ./s_docs > ${t_pfx}s_docs +2>&1 ./s_evergreen > ${t_pfx}s_evergreen 2>&1 ./s_export > ${t_pfx}s_export 2>&1 ./s_funcs > ${t_pfx}s_funcs 2>&1 ./s_function > ${t_pfx}s_function diff --git a/src/third_party/wiredtiger/dist/s_clang-tidy b/src/third_party/wiredtiger/dist/s_clang-tidy index 89d3fa4112a..8fd140ca883 100644 --- a/src/third_party/wiredtiger/dist/s_clang-tidy +++ b/src/third_party/wiredtiger/dist/s_clang-tidy @@ -48,16 +48,22 @@ args="-checks=*" args="$args,-android-cloexec-fopen" args="$args,-clang-analyzer-core.NullDereference" args="$args,-clang-analyzer-optin.performance.Padding" +args="$args,-cppcoreguidelines-avoid-magic-numbers" args="$args,-google-readability-braces-around-statements" args="$args,-hicpp-braces-around-statements" +args="$args,-hicpp-multiway-paths-covered" args="$args,-hicpp-no-assembler" args="$args,-hicpp-signed-bitwise" +args="$args,-hicpp-uppercase-literal-suffix" +args="$args,-hicpp-uppercase-literal-suffix-header-guard" args="$args,-llvm-header-guard" args="$args,-llvm-include-order" args="$args,-readability-braces-around-statements" args="$args,-readability-inconsistent-declaration-parameter-name" +args="$args,-readability-magic-numbers" args="$args,-readability-named-parameter" args="$args,-readability-non-const-parameter" +args="$args,-readability-uppercase-literal-suffix" # clang-tidy gets unhappy if we toss the whole tree at it, so run # through a file at a time. diff --git a/src/third_party/wiredtiger/dist/s_copyright.list b/src/third_party/wiredtiger/dist/s_copyright.list index e5014ef5921..934488e3d88 100644 --- a/src/third_party/wiredtiger/dist/s_copyright.list +++ b/src/third_party/wiredtiger/dist/s_copyright.list @@ -41,6 +41,7 @@ skip src/include/flags.h skip src/include/queue.h skip src/log/log_auto.c skip src/support/stat.c +skip test/evergreen/evg_cfg.py skip test/packing/intpack-test.c skip test/packing/intpack-test2.c skip test/packing/packing-test.c diff --git a/src/third_party/wiredtiger/dist/s_evergreen b/src/third_party/wiredtiger/dist/s_evergreen new file mode 100755 index 00000000000..9815d2e64a0 --- /dev/null +++ b/src/third_party/wiredtiger/dist/s_evergreen @@ -0,0 +1,24 @@ +#! /bin/sh + +t=__wt.$$ +trap 'rm -f $t' 0 1 2 3 13 15 + +toplevel_dir=$(git rev-parse --show-toplevel) +program=${toplevel_dir}/test/evergreen/evg_cfg.py + +# Run checking program to identify missing tests in Evergreen configuration +${program} check >$t 2>&1 +e=$? + +test -s $t && { + echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" + echo "$0: $program check" + cat $t + echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" + + # Don't exit non-zero unless the python script did, the script + # requires python modules that are commonly not installed, and + # in that case it exits 0. Post the complaint, but don't fail. + exit $e +} +exit 0 diff --git a/src/third_party/wiredtiger/dist/s_export.list b/src/third_party/wiredtiger/dist/s_export.list index e49fa113d96..e85bf62517d 100644 --- a/src/third_party/wiredtiger/dist/s_export.list +++ b/src/third_party/wiredtiger/dist/s_export.list @@ -1,4 +1,5 @@ # List of OK external symbols. +wiredtiger_calc_modify wiredtiger_config_parser_open wiredtiger_config_validate wiredtiger_crc32c_func diff --git a/src/third_party/wiredtiger/dist/s_funcs.list b/src/third_party/wiredtiger/dist/s_funcs.list index 95c568a19ff..bcedc19a459 100644 --- a/src/third_party/wiredtiger/dist/s_funcs.list +++ b/src/third_party/wiredtiger/dist/s_funcs.list @@ -33,6 +33,7 @@ __wt_stat_join_aggregate __wt_stat_join_clear_all __wt_stream_set_no_buffer __wt_try_readlock +wiredtiger_calc_modify wiredtiger_config_parser_open wiredtiger_config_validate wiredtiger_pack_int diff --git a/src/third_party/wiredtiger/dist/s_stat b/src/third_party/wiredtiger/dist/s_stat index 32cf16fd3d4..d0ea3ab9cce 100755 --- a/src/third_party/wiredtiger/dist/s_stat +++ b/src/third_party/wiredtiger/dist/s_stat @@ -30,6 +30,7 @@ lock_commit_timestamp_wait_application lock_commit_timestamp_wait_internal lock_commit_timestamp_write_count lock_dhandle_read_count +lock_dhandle_wait lock_dhandle_wait_application lock_dhandle_wait_internal lock_dhandle_write_count @@ -41,6 +42,7 @@ lock_read_timestamp_wait_application lock_read_timestamp_wait_internal lock_read_timestamp_write_count lock_schema_count +lock_schema_wait lock_schema_wait_application lock_schema_wait_internal lock_table_read_count diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index 247ad261085..c5546aed751 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -528,6 +528,7 @@ bzDecompressInit bzalloc bzfree bzip +calc call's calloc cas @@ -700,6 +701,7 @@ endian english enqueue enqueued +enum env eof epi @@ -952,6 +954,7 @@ marshalled maxCLevel maxcpu maxdbs +maxdiff mbll mbss mem @@ -999,9 +1002,11 @@ needkey needvalue negint nentries +nentriesp newbar newfile newuri +newv nextprev nfilename nhex @@ -1030,6 +1035,7 @@ numSymbols numbare offpage ok +oldv ondisk onint online @@ -1217,6 +1223,7 @@ subgets subinit sublicense subone +suboptimal subtest subtree sunique diff --git a/src/third_party/wiredtiger/dist/stat_data.py b/src/third_party/wiredtiger/dist/stat_data.py index e4e56194040..50e7be0039f 100644 --- a/src/third_party/wiredtiger/dist/stat_data.py +++ b/src/third_party/wiredtiger/dist/stat_data.py @@ -296,7 +296,8 @@ connection_stats = [ ########################################## # Cursor operations ########################################## - CursorStat('cursors_cached', 'cursors currently cached', 'no_clear,no_scale'), + CursorStat('cursor_open_count', 'open cursor count', 'no_clear,no_scale'), + CursorStat('cursor_cached_count', 'cached cursor count', 'no_clear,no_scale'), CursorStat('cursor_cache', 'cursor close calls that result in cache'), CursorStat('cursor_create', 'cursor create calls'), CursorStat('cursor_insert', 'cursor insert calls'), @@ -469,7 +470,6 @@ connection_stats = [ ########################################## # Session operations ########################################## - SessionOpStat('session_cursor_open', 'open cursor count', 'no_clear,no_scale'), SessionOpStat('session_open', 'open session count', 'no_clear,no_scale'), SessionOpStat('session_query_ts', 'session query timestamp calls'), SessionOpStat('session_table_alter_fail', 'table alter failed calls', 'no_clear,no_scale'), @@ -690,9 +690,6 @@ dsrc_stats = [ ########################################## # Compression statistics ########################################## - CompressStat('compress_raw_fail', 'raw compression call failed, no additional data available'), - CompressStat('compress_raw_fail_temporary', 'raw compression call failed, additional data available'), - CompressStat('compress_raw_ok', 'raw compression call succeeded'), CompressStat('compress_read', 'compressed pages read'), CompressStat('compress_write', 'compressed pages written'), CompressStat('compress_write_fail', 'page written failed to compress'), @@ -701,6 +698,7 @@ dsrc_stats = [ ########################################## # Cursor operations ########################################## + CursorStat('cursor_open_count', 'open cursor count', 'no_clear,no_scale'), CursorStat('cursor_cache', 'close calls that result in cache'), CursorStat('cursor_create', 'create calls'), CursorStat('cursor_insert', 'insert calls'), @@ -759,8 +757,6 @@ dsrc_stats = [ # Session operations ########################################## SessionOpStat('session_compact', 'object compaction'), - SessionOpStat('session_cursors_cached', 'cached cursor count', 'no_clear,no_scale'), - SessionOpStat('session_cursor_open', 'open cursor count', 'no_clear,no_scale'), ########################################## # Transaction statistics diff --git a/src/third_party/wiredtiger/examples/c/Makefile.am b/src/third_party/wiredtiger/examples/c/Makefile.am index 41a9bcdc6bb..e441d45af46 100644 --- a/src/third_party/wiredtiger/examples/c/Makefile.am +++ b/src/third_party/wiredtiger/examples/c/Makefile.am @@ -39,4 +39,4 @@ AM_TESTS_ENVIRONMENT = WIREDTIGER_HOME=`mktemp -d WT_HOME.XXXX` ; export WIREDTI LOG_COMPILER = $(TEST_WRAPPER) clean-local: - rm -rf WT_HOME* *.core backup_full.* backup_incr.* + rm -rf WT_HOME* core.* *.core backup_full.* backup_incr.* diff --git a/src/third_party/wiredtiger/examples/c/ex_all.c b/src/third_party/wiredtiger/examples/c/ex_all.c index fe6bf86804b..d9be2299833 100644 --- a/src/third_party/wiredtiger/examples/c/ex_all.c +++ b/src/third_party/wiredtiger/examples/c/ex_all.c @@ -578,6 +578,11 @@ cursor_statistics(WT_SESSION *session) "statistics:table:mytable", NULL, "statistics=(all,clear)", &cursor)); /*! [Statistics cursor clear configuration] */ + + /*! [Statistics cursor session] */ + error_check(session->open_cursor( + session, "statistics:session", NULL, NULL, &cursor)); + /*! [Statistics cursor session] */ } static void @@ -1352,6 +1357,26 @@ main(int argc, char *argv[]) } { + /*! [Calculate a modify operation] */ + WT_MODIFY mod[3]; + int nmod = 3; + WT_ITEM prev, newv; + prev.data = "the quick brown fox jumped over the lazy dog. " \ + "THE QUICK BROWN FOX JUMPED OVER THE LAZY DOG. " \ + "the quick brown fox jumped over the lazy dog. " \ + "THE QUICK BROWN FOX JUMPED OVER THE LAZY DOG. "; + prev.size = strlen(prev.data); + newv.data = "A quick brown fox jumped over the lazy dog. " \ + "THE QUICK BROWN FOX JUMPED OVER THE LAZY DOG. " \ + "then a quick brown fox jumped over the lazy dog. " \ + "THE QUICK BROWN FOX JUMPED OVER THE LAZY DOG. " \ + "then what?"; + newv.size = strlen(newv.data); + error_check(wiredtiger_calc_modify(NULL, &prev, &newv, 20, mod, &nmod)); + /*! [Calculate a modify operation] */ + } + + { const char *buffer = "some string"; size_t len = strlen(buffer); /*! [Checksum a buffer] */ diff --git a/src/third_party/wiredtiger/examples/c/ex_stat.c b/src/third_party/wiredtiger/examples/c/ex_stat.c index bc13197038a..fb9e9d07c98 100644 --- a/src/third_party/wiredtiger/examples/c/ex_stat.c +++ b/src/third_party/wiredtiger/examples/c/ex_stat.c @@ -37,6 +37,7 @@ void print_derived_stats(WT_SESSION *); void print_file_stats(WT_SESSION *); void print_join_cursor_stats(WT_SESSION *); void print_overflow_pages(WT_SESSION *); +void print_session_stats(WT_SESSION *); static const char *home; @@ -114,6 +115,20 @@ print_join_cursor_stats(WT_SESSION *session) } void +print_session_stats(WT_SESSION *session) +{ + WT_CURSOR *stat_cursor; + + /*! [statistics session function] */ + error_check(session->open_cursor(session, + "statistics:session", NULL, NULL, &stat_cursor)); + + print_cursor(stat_cursor); + error_check(stat_cursor->close(stat_cursor)); + /*! [statistics session function] */ +} + +void print_overflow_pages(WT_SESSION *session) { /*! [statistics retrieve by key] */ @@ -217,6 +232,8 @@ main(int argc, char *argv[]) print_join_cursor_stats(session); + print_session_stats(session); + print_overflow_pages(session); print_derived_stats(session); diff --git a/src/third_party/wiredtiger/examples/java/com/wiredtiger/examples/ex_all.java b/src/third_party/wiredtiger/examples/java/com/wiredtiger/examples/ex_all.java index 6649e14bd05..6485aeff566 100644 --- a/src/third_party/wiredtiger/examples/java/com/wiredtiger/examples/ex_all.java +++ b/src/third_party/wiredtiger/examples/java/com/wiredtiger/examples/ex_all.java @@ -492,6 +492,11 @@ cursor_statistics(Session session) null, "statistics=(all,clear)"); /*! [Statistics cursor clear configuration] */ + /*! [Statistics cursor session] */ + cursor = session.open_cursor( + "statistics:session", null, null); + /*! [Statistics cursor session] */ + return (true); } diff --git a/src/third_party/wiredtiger/examples/java/com/wiredtiger/examples/ex_stat.java b/src/third_party/wiredtiger/examples/java/com/wiredtiger/examples/ex_stat.java index 9e7cc723bab..fb5082516cc 100644 --- a/src/third_party/wiredtiger/examples/java/com/wiredtiger/examples/ex_stat.java +++ b/src/third_party/wiredtiger/examples/java/com/wiredtiger/examples/ex_stat.java @@ -119,6 +119,22 @@ public class ex_stat { } int + print_session_stats(Session session) + throws WiredTigerException + { + Cursor cursor; + int ret; + + /*! [statistics session function] */ + cursor = session.open_cursor("statistics:session", null, null); + ret = print_cursor(cursor); + ret = cursor.close(); + /*! [statistics session function] */ + + return (ret); + } + + int print_overflow_pages(Session session) throws WiredTigerException { @@ -264,6 +280,8 @@ public class ex_stat { ret = print_join_cursor_stats(session); + ret = print_session_stats(session); + ret = print_overflow_pages(session); ret = print_derived_stats(session); diff --git a/src/third_party/wiredtiger/ext/compressors/lz4/lz4_compress.c b/src/third_party/wiredtiger/ext/compressors/lz4/lz4_compress.c index dc90500dcdb..a024e328ea7 100644 --- a/src/third_party/wiredtiger/ext/compressors/lz4/lz4_compress.c +++ b/src/third_party/wiredtiger/ext/compressors/lz4/lz4_compress.c @@ -62,6 +62,10 @@ typedef struct { * decompressed, not the number of bytes decompressed; store that value in the * destination buffer as well. * + * (Since raw compression has been removed from WiredTiger, the lz4 compression + * code no longer calls LZ4_compress_destSize. Some support remains to support + * existing compressed objects.) + * * Use fixed-size, 4B values (WiredTiger never writes buffers larger than 4GB). * * The unused field is available for a mode flag if one is needed in the future, @@ -210,6 +214,9 @@ lz4_decompress(WT_COMPRESSOR *compressor, WT_SESSION *session, * other words, our caller doesn't know how many bytes will result from * decompression, likely hasn't provided us a large enough buffer, and * we have to allocate a scratch buffer. + * + * Even though raw compression has been removed from WiredTiger, this + * code remains for backward compatibility with existing objects. */ if (dst_len < prefix.uncompressed_len) { if ((dst_tmp = wt_api->scr_alloc( @@ -238,102 +245,6 @@ lz4_decompress(WT_COMPRESSOR *compressor, WT_SESSION *session, } /* - * lz4_find_slot -- - * Find the slot containing the target offset (binary search). - */ -static inline uint32_t -lz4_find_slot(int target_arg, uint32_t *offsets, uint32_t slots) -{ - uint32_t base, indx, limit, target; - - indx = 1; /* -Wuninitialized */ - - target = (uint32_t)target_arg; /* Type conversion */ - - /* Fast check if we consumed it all, it's a likely result. */ - if (target >= offsets[slots]) - return (slots); - - /* - * Figure out which slot we got to: binary search. Note the test of - * offset (slot + 1), that's (end-byte + 1) for slot. - */ - for (base = 0, limit = slots; limit != 0; limit >>= 1) { - indx = base + (limit >> 1); - if (target > offsets[indx + 1]) { - base = indx + 1; - --limit; - } - } - - return (indx); -} - -/* - * lz4_compress_raw -- - * Pack records into a specified on-disk page size. - */ -static int -lz4_compress_raw(WT_COMPRESSOR *compressor, WT_SESSION *session, - size_t page_max, int split_pct, size_t extra, - uint8_t *src, uint32_t *offsets, uint32_t slots, - uint8_t *dst, size_t dst_len, int final, - size_t *result_lenp, uint32_t *result_slotsp) -{ - LZ4_PREFIX prefix; - uint32_t slot; - int lz4_len, sourceSize, targetDestSize; - - (void)compressor; /* Unused parameters */ - (void)session; - (void)split_pct; - (void)final; - - /* - * Set the source and target sizes. The target size is complicated: we - * don't want to exceed the smaller of the maximum page size or the - * destination buffer length, and in both cases we have to take into - * account the space for our overhead and the extra bytes required by - * our caller. - */ - sourceSize = (int)offsets[slots]; - targetDestSize = (int)(page_max < dst_len ? page_max : dst_len); - targetDestSize -= (int)(sizeof(LZ4_PREFIX) + extra); - - /* Compress, starting after the prefix bytes. */ - lz4_len = LZ4_compress_destSize((const char *)src, - (char *)dst + sizeof(LZ4_PREFIX), &sourceSize, targetDestSize); - - /* - * If compression succeeded and the compressed length is smaller than - * the original size, return success. - */ - if (lz4_len != 0) { - /* Find the first slot we didn't compress. */ - slot = lz4_find_slot(sourceSize, offsets, slots); - - if ((size_t)lz4_len + sizeof(LZ4_PREFIX) < offsets[slot]) { - prefix.compressed_len = (uint32_t)lz4_len; - prefix.uncompressed_len = (uint32_t)sourceSize; - prefix.useful_len = offsets[slot]; - prefix.unused = 0; -#ifdef WORDS_BIGENDIAN - lz4_prefix_swap(&prefix); -#endif - memcpy(dst, &prefix, sizeof(LZ4_PREFIX)); - - *result_slotsp = slot; - *result_lenp = (size_t)lz4_len + sizeof(LZ4_PREFIX); - return (0); - } - } - - *result_slotsp = 0; - *result_lenp = 1; - return (0); -} - -/* * lz4_pre_size -- * WiredTiger LZ4 destination buffer sizing for compression. */ @@ -372,20 +283,15 @@ lz4_terminate(WT_COMPRESSOR *compressor, WT_SESSION *session) * Add a LZ4 compressor. */ static int -lz_add_compressor(WT_CONNECTION *connection, bool raw, const char *name) +lz_add_compressor(WT_CONNECTION *connection, const char *name) { LZ4_COMPRESSOR *lz4_compressor; int ret; - /* - * There are two almost identical LZ4 compressors: one using raw - * compression to target a specific block size, and one without. - */ if ((lz4_compressor = calloc(1, sizeof(LZ4_COMPRESSOR))) == NULL) return (errno); lz4_compressor->compressor.compress = lz4_compress; - lz4_compressor->compressor.compress_raw = raw ? lz4_compress_raw : NULL; lz4_compressor->compressor.decompress = lz4_decompress; lz4_compressor->compressor.pre_size = lz4_pre_size; lz4_compressor->compressor.terminate = lz4_terminate; @@ -416,9 +322,11 @@ lz4_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) (void)config; /* Unused parameters */ - if ((ret = lz_add_compressor(connection, true, "lz4")) != 0) + if ((ret = lz_add_compressor(connection, "lz4")) != 0) return (ret); - if ((ret = lz_add_compressor(connection, false, "lz4-noraw")) != 0) + + /* Raw compression API backward compatibility. */ + if ((ret = lz_add_compressor(connection, "lz4-noraw")) != 0) return (ret); return (0); } diff --git a/src/third_party/wiredtiger/ext/compressors/nop/nop_compress.c b/src/third_party/wiredtiger/ext/compressors/nop/nop_compress.c index 586f6c8831b..3a5baaedc79 100644 --- a/src/third_party/wiredtiger/ext/compressors/nop/nop_compress.c +++ b/src/third_party/wiredtiger/ext/compressors/nop/nop_compress.c @@ -170,7 +170,6 @@ wiredtiger_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) * Heap memory (not static), because it can support multiple databases. */ nop_compressor->compressor.compress = nop_compress; - nop_compressor->compressor.compress_raw = NULL; nop_compressor->compressor.decompress = nop_decompress; nop_compressor->compressor.pre_size = nop_pre_size; nop_compressor->compressor.terminate = nop_terminate; diff --git a/src/third_party/wiredtiger/ext/compressors/snappy/snappy_compress.c b/src/third_party/wiredtiger/ext/compressors/snappy/snappy_compress.c index 03a17d28a1b..f369a0c2965 100644 --- a/src/third_party/wiredtiger/ext/compressors/snappy/snappy_compress.c +++ b/src/third_party/wiredtiger/ext/compressors/snappy/snappy_compress.c @@ -260,7 +260,6 @@ snappy_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) return (errno); snappy_compressor->compressor.compress = snappy_compression; - snappy_compressor->compressor.compress_raw = NULL; snappy_compressor->compressor.decompress = snappy_decompression; snappy_compressor->compressor.pre_size = snappy_pre_size; snappy_compressor->compressor.terminate = snappy_terminate; diff --git a/src/third_party/wiredtiger/ext/compressors/zlib/zlib_compress.c b/src/third_party/wiredtiger/ext/compressors/zlib/zlib_compress.c index d5c0d0fb318..dd9cfdeb88b 100644 --- a/src/third_party/wiredtiger/ext/compressors/zlib/zlib_compress.c +++ b/src/third_party/wiredtiger/ext/compressors/zlib/zlib_compress.c @@ -198,239 +198,6 @@ zlib_decompress(WT_COMPRESSOR *compressor, WT_SESSION *session, } /* - * zlib_find_slot -- - * Find the slot containing the target offset (binary search). - */ -static inline uint32_t -zlib_find_slot(uint64_t target, uint32_t *offsets, uint32_t slots) -{ - uint32_t base, indx, limit; - - indx = 1; - - /* Figure out which slot we got to: binary search */ - if (target >= offsets[slots]) - indx = slots; - else if (target > offsets[1]) - for (base = 2, limit = slots - base; limit != 0; limit >>= 1) { - indx = base + (limit >> 1); - if (target < offsets[indx]) - continue; - base = indx + 1; - --limit; - } - - return (indx); -} - -/* - * zlib_compress_raw -- - * Pack records into a specified on-disk page size. - */ -static int -zlib_compress_raw(WT_COMPRESSOR *compressor, WT_SESSION *session, - size_t page_max, int split_pct, size_t extra, - uint8_t *src, uint32_t *offsets, uint32_t slots, - uint8_t *dst, size_t dst_len, int final, - size_t *result_lenp, uint32_t *result_slotsp) -{ - ZLIB_COMPRESSOR *zlib_compressor; - ZLIB_OPAQUE opaque; - z_stream *best_zs, *last_zs, _last_zs, *zs, _zs; - uint32_t curr_slot, last_slot, zlib_reserved; - bool increase_reserve; - int ret, tret; - - (void)split_pct; /* Unused parameters */ - (void)final; - - zlib_compressor = (ZLIB_COMPRESSOR *)compressor; - - /* - * Experimentally derived, reserve this many bytes for zlib to finish - * up a buffer. If this isn't sufficient, we don't fail but we will be - * inefficient. - */ -#define WT_ZLIB_RESERVED 24 -#define WT_ZLIB_RESERVED_MAX 48 - zlib_reserved = WT_ZLIB_RESERVED; - - if (0) { -retry: /* If we reached our maximum reserve, quit. */ - if (zlib_reserved == WT_ZLIB_RESERVED_MAX) - return (0); - zlib_reserved = WT_ZLIB_RESERVED_MAX; - } - - best_zs = last_zs = NULL; - last_slot = 0; - increase_reserve = false; - ret = 0; - - zs = &_zs; - memset(zs, 0, sizeof(*zs)); - zs->zalloc = zalloc; - zs->zfree = zfree; - opaque.compressor = compressor; - opaque.session = session; - zs->opaque = &opaque; - - if ((ret = deflateInit(zs, zlib_compressor->zlib_level)) != Z_OK) - return (zlib_error(compressor, session, "deflateInit", ret)); - - zs->next_in = src; - zs->next_out = dst; - - /* - * Set the target size. The target size is complicated: we don't want - * to exceed the smaller of the maximum page size or the destination - * buffer length, and in both cases we have to take into account the - * space required by zlib to finish up the buffer and the extra bytes - * required by our caller. - */ - zs->avail_out = (uint32_t)(page_max < dst_len ? page_max : dst_len); - zs->avail_out -= (uint32_t)(zlib_reserved + extra); - - /* - * Strategy: take the available output size and compress that much - * input. Continue until there is no input small enough or the - * compression fails to fit. - */ - for (;;) { - /* Find the next slot we will try to compress up to. */ - curr_slot = zlib_find_slot( - zs->total_in + zs->avail_out, offsets, slots); - if (curr_slot > last_slot) { - zs->avail_in = offsets[curr_slot] - offsets[last_slot]; - while (zs->avail_in > 0 && zs->avail_out > 0) - if ((ret = deflate(zs, Z_SYNC_FLUSH)) != Z_OK) { - ret = zlib_error(compressor, - session, "deflate", ret); - goto err; - } - } - - /* - * We didn't do a deflate, or it didn't work: use the last saved - * position (if any). - */ - if (curr_slot <= last_slot || zs->avail_in > 0) { - best_zs = last_zs; - break; - } - - /* - * If there's more compression to do, save a snapshot and keep - * going, otherwise, use the current compression. - */ - last_slot = curr_slot; - if (zs->avail_out > 0) { - /* Discard any previously saved snapshot. */ - if (last_zs != NULL) { - ret = deflateEnd(last_zs); - last_zs = NULL; - if (ret != Z_OK && ret != Z_DATA_ERROR) { - ret = zlib_error(compressor, - session, "deflateEnd", ret); - goto err; - } - } - last_zs = &_last_zs; - if ((ret = deflateCopy(last_zs, zs)) != Z_OK) { - last_zs = NULL; - ret = zlib_error( - compressor, session, "deflateCopy", ret); - goto err; - } - continue; - } - - best_zs = zs; - break; - } - - if (last_slot > 0 && best_zs != NULL) { - /* Add the reserved bytes and try to finish the compression. */ - best_zs->avail_out += zlib_reserved; - ret = deflate(best_zs, Z_FINISH); - - /* - * If the end marker didn't fit with the default value, try - * again with a maximum value; if that doesn't work, report we - * got no work done, WiredTiger will compress the (possibly - * large) page image using ordinary compression instead. - */ - if (ret == Z_OK || ret == Z_BUF_ERROR) { - last_slot = 0; - increase_reserve = true; - } else if (ret != Z_STREAM_END) { - ret = zlib_error( - compressor, session, "deflate end block", ret); - goto err; - } - ret = 0; - } - -err: if ((tret = deflateEnd(zs)) != Z_OK && tret != Z_DATA_ERROR) - ret = zlib_error(compressor, session, "deflateEnd", tret); - if (last_zs != NULL && - (tret = deflateEnd(last_zs)) != Z_OK && tret != Z_DATA_ERROR) - ret = zlib_error(compressor, session, "deflateEnd", tret); - - if (ret == 0 && last_slot > 0) { - *result_slotsp = last_slot; - *result_lenp = (size_t)best_zs->total_out; - } else { - /* We didn't manage to compress anything. */ - *result_slotsp = 0; - *result_lenp = 1; - - if (increase_reserve) - goto retry; - } - -#if 0 - /* Decompress the result and confirm it matches the original source. */ - if (ret == 0 && last_slot > 0) { - WT_EXTENSION_API *wt_api; - void *decomp; - size_t result_len; - - wt_api = ((ZLIB_COMPRESSOR *)compressor)->wt_api; - - if ((decomp = zalloc( - &opaque, 1, (uint32_t)best_zs->total_in + 100)) == NULL) { - (void)wt_api->err_printf(wt_api, session, - "zlib_compress_raw: zalloc failure"); - return (ENOMEM); - } - if ((ret = zlib_decompress( - compressor, session, dst, (size_t)best_zs->total_out, - decomp, (size_t)best_zs->total_in + 100, &result_len)) == 0) - if (memcmp(src, decomp, result_len) != 0) { - (void)wt_api->err_printf(wt_api, session, - "zlib_compress_raw: " - "deflate compare with original source"); - return (WT_ERROR); - } - zfree(&opaque, decomp); - } -#endif - -#if 0 - if (ret == 0 && last_slot > 0) - fprintf(stderr, - "zlib_compress_raw (%s): page_max %" PRIuMAX ", slots %" - PRIu32 ", take %" PRIu32 ": %" PRIu32 " -> %" PRIuMAX "\n", - final ? "final" : "not final", (uintmax_t)page_max, - slots, last_slot, offsets[last_slot], - (uintmax_t)*result_lenp); -#endif - - return (ret); -} - -/* * zlib_terminate -- * WiredTiger zlib compression termination. */ @@ -448,22 +215,15 @@ zlib_terminate(WT_COMPRESSOR *compressor, WT_SESSION *session) * Add a zlib compressor. */ static int -zlib_add_compressor( - WT_CONNECTION *connection, bool raw, const char *name, int zlib_level) +zlib_add_compressor(WT_CONNECTION *connection, const char *name, int zlib_level) { ZLIB_COMPRESSOR *zlib_compressor; int ret; - /* - * There are two almost identical zlib compressors: one using raw - * compression to target a specific block size, and one without. - */ if ((zlib_compressor = calloc(1, sizeof(ZLIB_COMPRESSOR))) == NULL) return (errno); zlib_compressor->compressor.compress = zlib_compress; - zlib_compressor->compressor.compress_raw = raw ? - zlib_compress_raw : NULL; zlib_compressor->compressor.decompress = zlib_decompress; zlib_compressor->compressor.pre_size = NULL; zlib_compressor->compressor.terminate = zlib_terminate; @@ -542,11 +302,12 @@ zlib_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) if ((ret = zlib_init_config(connection, config, &zlib_level)) != 0) return (ret); - if ((ret = zlib_add_compressor( - connection, true, "zlib", zlib_level)) != 0) + if ((ret = zlib_add_compressor(connection, "zlib", zlib_level)) != 0) return (ret); + + /* Raw compression API backward compatibility. */ if ((ret = zlib_add_compressor( - connection, false, "zlib-noraw", zlib_level)) != 0) + connection, "zlib-noraw", zlib_level)) != 0) return (ret); return (0); } diff --git a/src/third_party/wiredtiger/ext/compressors/zstd/zstd_compress.c b/src/third_party/wiredtiger/ext/compressors/zstd/zstd_compress.c index 40a872f92e2..210bc4b30df 100644 --- a/src/third_party/wiredtiger/ext/compressors/zstd/zstd_compress.c +++ b/src/third_party/wiredtiger/ext/compressors/zstd/zstd_compress.c @@ -296,7 +296,7 @@ zstd_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) * ratio). In other words, position zstd as a zlib replacement, having * similar compression at much higher compression/decompression speeds. */ - compression_level = 3; + compression_level = 6; if ((ret = zstd_init_config(connection, config, &compression_level)) != 0) return (ret); @@ -305,7 +305,6 @@ zstd_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) return (errno); zstd_compressor->compressor.compress = zstd_compress; - zstd_compressor->compressor.compress_raw = NULL; zstd_compressor->compressor.decompress = zstd_decompress; zstd_compressor->compressor.pre_size = zstd_pre_size; zstd_compressor->compressor.terminate = zstd_terminate; diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 516c9a2d684..40a0b23fa86 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -1,5 +1,5 @@ { - "commit": "b51da4ed903efd9daf4b9c04385f01bbb8d37d7d", + "commit": "bedf230af338faa9b9ba8741b9a2e5f36353e2f0", "github": "wiredtiger/wiredtiger.git", "vendor": "wiredtiger", "branch": "mongodb-4.0" diff --git a/src/third_party/wiredtiger/lang/java/wiredtiger.i b/src/third_party/wiredtiger/lang/java/wiredtiger.i index dfac0e9ec8d..5ef1658a322 100644 --- a/src/third_party/wiredtiger/lang/java/wiredtiger.i +++ b/src/third_party/wiredtiger/lang/java/wiredtiger.i @@ -357,6 +357,7 @@ WT_CLASS(struct __wt_async_op, WT_ASYNC_OP, op) %ignore __wt_modify::position; %ignore __wt_modify::size; %ignore __wt_cursor::modify; +%ignore wiredtiger_calc_modify; %ignore __wt_cursor::compare(WT_CURSOR *, WT_CURSOR *, int *); %rename (compare_wrap) __wt_cursor::compare; diff --git a/src/third_party/wiredtiger/lang/python/Makefile.am b/src/third_party/wiredtiger/lang/python/Makefile.am index b32d0321194..d7539c35bdd 100644 --- a/src/third_party/wiredtiger/lang/python/Makefile.am +++ b/src/third_party/wiredtiger/lang/python/Makefile.am @@ -24,6 +24,6 @@ install-exec-local: # clean up both. Don't rely on "setup.py clean" -- everything that should # be removed is created under the build directory. clean-local: - rm -rf build _wiredtiger.so wiredtiger_wrap.o WT_TEST + rm -rf build _wiredtiger.so wiredtiger_wrap.o WT_TEST core.* *.core TESTS = run-ex_access diff --git a/src/third_party/wiredtiger/lang/python/wiredtiger.i b/src/third_party/wiredtiger/lang/python/wiredtiger.i index 9ba6bd15c78..47ed727f3a8 100644 --- a/src/third_party/wiredtiger/lang/python/wiredtiger.i +++ b/src/third_party/wiredtiger/lang/python/wiredtiger.i @@ -151,6 +151,41 @@ from packing import pack, unpack } } +%typemap(in,numinputs=1) (WT_MODIFY *entries, int *nentriesp) (WT_MODIFY *mod, int nentries) { + nentries = (int) PyLong_AsLong($input); + if (__wt_calloc_def(NULL, (size_t)nentries, &mod) != 0) + SWIG_exception_fail(SWIG_MemoryError, "WT calloc failed"); + $1 = mod; + $2 = &nentries; +} + +%typemap(argout) (WT_MODIFY *entries, int *nentriesp) { + int i; + $result = PyList_New(*$2); + for (i = 0; i < *$2; i++) { + PyObject *o = SWIG_NewPointerObj(Py_None, SWIGTYPE_p___wt_modify, 0); + PyObject_SetAttrString(o, "data", PyString_FromStringAndSize( + $1[i].data.data, $1[i].data.size)); + PyObject_SetAttrString(o, "offset", + PyInt_FromLong($1[i].offset)); + PyObject_SetAttrString(o, "size", + PyInt_FromLong($1[i].size)); + PyList_SetItem($result, i, o); + } +} + +%typemap(in) const WT_ITEM * (WT_ITEM val, long sz) { + if (PyString_AsStringAndSize($input, &val.data, &sz) < 0) + SWIG_exception_fail(SWIG_AttributeError, + "bad string value for WT_ITEM"); + val.size = (size_t)sz; + $1 = &val; +} + +%typemap(freearg) (WT_MODIFY *entries, int *nentriesp) { + __wt_free(NULL, $1); +} + %typemap(in) WT_MODIFY * (int len, WT_MODIFY *modarray, int i) { len = PyList_Size($input); /* @@ -209,7 +244,7 @@ from packing import pack, unpack } %typemap(freearg) WT_MODIFY * { - /* The WT_MODIFY arg is in position 2. Is there a better way? */ + /* The WT_MODIFY arg is in position 2. Is there a better way? */ WT_MODIFY *modarray = modarray2; size_t i, len; @@ -980,6 +1015,9 @@ typedef int int_void; self.data = data self.offset = offset self.size = size + + def __repr__(self): + return 'Modify(\'%s\', %d, %d)' % (self.data, self.offset, self.size) %} }; @@ -1008,6 +1046,7 @@ int diagnostic_build() { #endif } %} + int diagnostic_build(); /* Remove / rename parts of the C API that we don't want in Python. */ diff --git a/src/third_party/wiredtiger/src/btree/bt_handle.c b/src/third_party/wiredtiger/src/btree/bt_handle.c index bf7ad49ccc3..e413dd7ca72 100644 --- a/src/third_party/wiredtiger/src/btree/bt_handle.c +++ b/src/third_party/wiredtiger/src/btree/bt_handle.c @@ -459,8 +459,8 @@ __btree_conf(WT_SESSION_IMPL *session, WT_CKPT *ckpt) * Don't do compression adjustment for fixed-size column store, the * leaf page sizes don't change. (We could adjust internal pages but not * internal pages, but that seems an unlikely use case.) - * XXX - * Don't do compression adjustment of snappy-compressed blocks. + * XXX + * Don't do compression adjustment of snappy-compressed blocks. */ btree->intlpage_compadjust = false; btree->maxintlpage_precomp = btree->maxintlpage; diff --git a/src/third_party/wiredtiger/src/btree/bt_io.c b/src/third_party/wiredtiger/src/btree/bt_io.c index 7e7909eed9f..3a5dd48e392 100644 --- a/src/third_party/wiredtiger/src/btree/bt_io.c +++ b/src/third_party/wiredtiger/src/btree/bt_io.c @@ -145,6 +145,7 @@ __wt_bt_read(WT_SESSION_IMPL *session, WT_STAT_DATA_INCR(session, compress_read); WT_STAT_CONN_INCRV(session, cache_bytes_read, dsk->mem_size); WT_STAT_DATA_INCRV(session, cache_bytes_read, dsk->mem_size); + WT_STAT_SESSION_INCRV(session, bytes_read, dsk->mem_size); (void)__wt_atomic_add64( &S2C(session)->cache->bytes_read, dsk->mem_size); @@ -185,7 +186,7 @@ __wt_bt_write(WT_SESSION_IMPL *session, WT_ITEM *buf, WT_KEYED_ENCRYPTOR *kencryptor; WT_PAGE_HEADER *dsk; size_t dst_len, len, result_len, size, src_len; - uint64_t time_start, time_stop; + uint64_t time_diff, time_start, time_stop; uint8_t *dst, *src; int compression_failed; /* Extension API, so not a bool. */ bool data_checksum, encrypted, timer; @@ -388,15 +389,17 @@ __wt_bt_write(WT_SESSION_IMPL *session, WT_ITEM *buf, /* Update some statistics now that the write is done */ if (timer) { time_stop = __wt_clock(session); + time_diff = WT_CLOCKDIFF_US(time_stop, time_start); WT_STAT_CONN_INCR(session, cache_write_app_count); - WT_STAT_CONN_INCRV(session, cache_write_app_time, - WT_CLOCKDIFF_US(time_stop, time_start)); + WT_STAT_CONN_INCRV(session, cache_write_app_time, time_diff); + WT_STAT_SESSION_INCRV(session, write_time, time_diff); } WT_STAT_CONN_INCR(session, cache_write); WT_STAT_DATA_INCR(session, cache_write); WT_STAT_CONN_INCRV(session, cache_bytes_write, dsk->mem_size); WT_STAT_DATA_INCRV(session, cache_bytes_write, dsk->mem_size); + WT_STAT_SESSION_INCRV(session, bytes_write, dsk->mem_size); (void)__wt_atomic_add64( &S2C(session)->cache->bytes_written, dsk->mem_size); diff --git a/src/third_party/wiredtiger/src/btree/bt_read.c b/src/third_party/wiredtiger/src/btree/bt_read.c index f6f7212eb85..dd560b39c0b 100644 --- a/src/third_party/wiredtiger/src/btree/bt_read.c +++ b/src/third_party/wiredtiger/src/btree/bt_read.c @@ -415,7 +415,7 @@ __page_read(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags) WT_ITEM tmp; WT_PAGE *notused; size_t addr_size; - uint64_t time_start, time_stop; + uint64_t time_diff, time_start, time_stop; uint32_t page_flags, final_state, new_state, previous_state; const uint8_t *addr; bool timer; @@ -482,9 +482,10 @@ __page_read(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags) WT_ERR(__wt_bt_read(session, &tmp, addr, addr_size)); if (timer) { time_stop = __wt_clock(session); + time_diff = WT_CLOCKDIFF_US(time_stop, time_start); WT_STAT_CONN_INCR(session, cache_read_app_count); - WT_STAT_CONN_INCRV(session, cache_read_app_time, - WT_CLOCKDIFF_US(time_stop, time_start)); + WT_STAT_CONN_INCRV(session, cache_read_app_time, time_diff); + WT_STAT_SESSION_INCRV(session, read_time, time_diff); } /* diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index 1bda9a62e25..1095fe18560 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -95,6 +95,9 @@ static const WT_CONFIG_CHECK static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure_log_subconfigs[] = { { "archive", "boolean", NULL, NULL, NULL, 0 }, + { "os_cache_dirty_pct", "int", + NULL, "min=0,max=100", + NULL, 0 }, { "prealloc", "boolean", NULL, NULL, NULL, 0 }, { "zero_fill", "boolean", NULL, NULL, NULL, 0 }, { NULL, NULL, NULL, NULL, NULL, 0 } @@ -169,7 +172,7 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = { confchk_wiredtiger_open_file_manager_subconfigs, 3 }, { "log", "category", NULL, NULL, - confchk_WT_CONNECTION_reconfigure_log_subconfigs, 3 }, + confchk_WT_CONNECTION_reconfigure_log_subconfigs, 4 }, { "lsm_manager", "category", NULL, NULL, confchk_wiredtiger_open_lsm_manager_subconfigs, 2 }, @@ -250,6 +253,8 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_alter[] = { { "log", "category", NULL, NULL, confchk_WT_SESSION_create_log_subconfigs, 1 }, + { "os_cache_dirty_max", "int", NULL, "min=0", NULL, 0 }, + { "os_cache_max", "int", NULL, "min=0", NULL, 0 }, { NULL, NULL, NULL, NULL, NULL, 0 } }; @@ -785,6 +790,9 @@ static const WT_CONFIG_CHECK { "compressor", "string", NULL, NULL, NULL, 0 }, { "enabled", "boolean", NULL, NULL, NULL, 0 }, { "file_max", "int", NULL, "min=100KB,max=2GB", NULL, 0 }, + { "os_cache_dirty_pct", "int", + NULL, "min=0,max=100", + NULL, 0 }, { "path", "string", NULL, NULL, NULL, 0 }, { "prealloc", "boolean", NULL, NULL, NULL, 0 }, { "recover", "string", @@ -868,7 +876,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { { "in_memory", "boolean", NULL, NULL, NULL, 0 }, { "log", "category", NULL, NULL, - confchk_wiredtiger_open_log_subconfigs, 8 }, + confchk_wiredtiger_open_log_subconfigs, 9 }, { "lsm_manager", "category", NULL, NULL, confchk_wiredtiger_open_lsm_manager_subconfigs, 2 }, @@ -974,7 +982,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { { "in_memory", "boolean", NULL, NULL, NULL, 0 }, { "log", "category", NULL, NULL, - confchk_wiredtiger_open_log_subconfigs, 8 }, + confchk_wiredtiger_open_log_subconfigs, 9 }, { "lsm_manager", "category", NULL, NULL, confchk_wiredtiger_open_lsm_manager_subconfigs, 2 }, @@ -1077,7 +1085,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { { "hazard_max", "int", NULL, "min=15", NULL, 0 }, { "log", "category", NULL, NULL, - confchk_wiredtiger_open_log_subconfigs, 8 }, + confchk_wiredtiger_open_log_subconfigs, 9 }, { "lsm_manager", "category", NULL, NULL, confchk_wiredtiger_open_lsm_manager_subconfigs, 2 }, @@ -1178,7 +1186,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { { "hazard_max", "int", NULL, "min=15", NULL, 0 }, { "log", "category", NULL, NULL, - confchk_wiredtiger_open_log_subconfigs, 8 }, + confchk_wiredtiger_open_log_subconfigs, 9 }, { "lsm_manager", "category", NULL, NULL, confchk_wiredtiger_open_lsm_manager_subconfigs, 2 }, @@ -1284,9 +1292,10 @@ static const WT_CONFIG_ENTRY config_entries[] = { "eviction_checkpoint_target=1,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" ",file_manager=(close_handle_minimum=250,close_idle_time=30," - "close_scan_interval=10),log=(archive=true,prealloc=true," - "zero_fill=false),lsm_manager=(merge=true,worker_thread_max=4)," - "lsm_merge=true,operation_tracking=(enabled=false,path=\".\")," + "close_scan_interval=10),log=(archive=true,os_cache_dirty_pct=0," + "prealloc=true,zero_fill=false),lsm_manager=(merge=true," + "worker_thread_max=4),lsm_merge=true," + "operation_tracking=(enabled=false,path=\".\")," "shared_cache=(chunk=10MB,name=,quota=0,reserve=0,size=500MB)," "statistics=none,statistics_log=(json=false,on_close=false," "sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," @@ -1317,8 +1326,9 @@ static const WT_CONFIG_ENTRY config_entries[] = { { "WT_SESSION.alter", "access_pattern_hint=none,app_metadata=," "assert=(commit_timestamp=none,read_timestamp=none)," - "cache_resident=false,exclusive_refreshed=true,log=(enabled=true)", - confchk_WT_SESSION_alter, 6 + "cache_resident=false,exclusive_refreshed=true,log=(enabled=true)" + ",os_cache_dirty_max=0,os_cache_max=0", + confchk_WT_SESSION_alter, 8 }, { "WT_SESSION.begin_transaction", "ignore_prepare=false,isolation=,name=,priority=0,read_timestamp=" @@ -1536,10 +1546,10 @@ static const WT_CONFIG_ENTRY config_entries[] = { "extensions=,file_extend=,file_manager=(close_handle_minimum=250," "close_idle_time=30,close_scan_interval=10),hazard_max=1000," "in_memory=false,log=(archive=true,compressor=,enabled=false," - "file_max=100MB,path=\".\",prealloc=true,recover=on," - "zero_fill=false),lsm_manager=(merge=true,worker_thread_max=4)," - "lsm_merge=true,mmap=true,multiprocess=false," - "operation_tracking=(enabled=false,path=\".\"),readonly=false," + "file_max=100MB,os_cache_dirty_pct=0,path=\".\",prealloc=true," + "recover=on,zero_fill=false),lsm_manager=(merge=true," + "worker_thread_max=4),lsm_merge=true,mmap=true,multiprocess=false" + ",operation_tracking=(enabled=false,path=\".\"),readonly=false," "salvage=false,session_max=100,session_scratch_max=2MB," "session_table_cache=true,shared_cache=(chunk=10MB,name=,quota=0," "reserve=0,size=500MB),statistics=none,statistics_log=(json=false" @@ -1563,10 +1573,10 @@ static const WT_CONFIG_ENTRY config_entries[] = { "extensions=,file_extend=,file_manager=(close_handle_minimum=250," "close_idle_time=30,close_scan_interval=10),hazard_max=1000," "in_memory=false,log=(archive=true,compressor=,enabled=false," - "file_max=100MB,path=\".\",prealloc=true,recover=on," - "zero_fill=false),lsm_manager=(merge=true,worker_thread_max=4)," - "lsm_merge=true,mmap=true,multiprocess=false," - "operation_tracking=(enabled=false,path=\".\"),readonly=false," + "file_max=100MB,os_cache_dirty_pct=0,path=\".\",prealloc=true," + "recover=on,zero_fill=false),lsm_manager=(merge=true," + "worker_thread_max=4),lsm_merge=true,mmap=true,multiprocess=false" + ",operation_tracking=(enabled=false,path=\".\"),readonly=false," "salvage=false,session_max=100,session_scratch_max=2MB," "session_table_cache=true,shared_cache=(chunk=10MB,name=,quota=0," "reserve=0,size=500MB),statistics=none,statistics_log=(json=false" @@ -1589,16 +1599,16 @@ static const WT_CONFIG_ENTRY config_entries[] = { ",extensions=,file_extend=,file_manager=(close_handle_minimum=250" ",close_idle_time=30,close_scan_interval=10),hazard_max=1000," "log=(archive=true,compressor=,enabled=false,file_max=100MB," - "path=\".\",prealloc=true,recover=on,zero_fill=false)," - "lsm_manager=(merge=true,worker_thread_max=4),lsm_merge=true," - "mmap=true,multiprocess=false,operation_tracking=(enabled=false," - "path=\".\"),readonly=false,salvage=false,session_max=100," - "session_scratch_max=2MB,session_table_cache=true," - "shared_cache=(chunk=10MB,name=,quota=0,reserve=0,size=500MB)," - "statistics=none,statistics_log=(json=false,on_close=false," - "path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," - "timing_stress_for_test=,transaction_sync=(enabled=false," - "method=fsync),verbose=,version=(major=0,minor=0),write_through=", + "os_cache_dirty_pct=0,path=\".\",prealloc=true,recover=on," + "zero_fill=false),lsm_manager=(merge=true,worker_thread_max=4)," + "lsm_merge=true,mmap=true,multiprocess=false," + "operation_tracking=(enabled=false,path=\".\"),readonly=false," + "salvage=false,session_max=100,session_scratch_max=2MB," + "session_table_cache=true,shared_cache=(chunk=10MB,name=,quota=0," + "reserve=0,size=500MB),statistics=none,statistics_log=(json=false" + ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\"" + ",wait=0),timing_stress_for_test=,transaction_sync=(enabled=false" + ",method=fsync),verbose=,version=(major=0,minor=0),write_through=", confchk_wiredtiger_open_basecfg, 42 }, { "wiredtiger_open_usercfg", @@ -1614,16 +1624,16 @@ static const WT_CONFIG_ENTRY config_entries[] = { ",extensions=,file_extend=,file_manager=(close_handle_minimum=250" ",close_idle_time=30,close_scan_interval=10),hazard_max=1000," "log=(archive=true,compressor=,enabled=false,file_max=100MB," - "path=\".\",prealloc=true,recover=on,zero_fill=false)," - "lsm_manager=(merge=true,worker_thread_max=4),lsm_merge=true," - "mmap=true,multiprocess=false,operation_tracking=(enabled=false," - "path=\".\"),readonly=false,salvage=false,session_max=100," - "session_scratch_max=2MB,session_table_cache=true," - "shared_cache=(chunk=10MB,name=,quota=0,reserve=0,size=500MB)," - "statistics=none,statistics_log=(json=false,on_close=false," - "path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," - "timing_stress_for_test=,transaction_sync=(enabled=false," - "method=fsync),verbose=,write_through=", + "os_cache_dirty_pct=0,path=\".\",prealloc=true,recover=on," + "zero_fill=false),lsm_manager=(merge=true,worker_thread_max=4)," + "lsm_merge=true,mmap=true,multiprocess=false," + "operation_tracking=(enabled=false,path=\".\"),readonly=false," + "salvage=false,session_max=100,session_scratch_max=2MB," + "session_table_cache=true,shared_cache=(chunk=10MB,name=,quota=0," + "reserve=0,size=500MB),statistics=none,statistics_log=(json=false" + ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\"" + ",wait=0),timing_stress_for_test=,transaction_sync=(enabled=false" + ",method=fsync),verbose=,write_through=", confchk_wiredtiger_open_usercfg, 41 }, { NULL, NULL, NULL, 0 } diff --git a/src/third_party/wiredtiger/src/conn/api_calc_modify.c b/src/third_party/wiredtiger/src/conn/api_calc_modify.c new file mode 100644 index 00000000000..a5a7fdff2b7 --- /dev/null +++ b/src/third_party/wiredtiger/src/conn/api_calc_modify.c @@ -0,0 +1,193 @@ +/*- + * Copyright (c) 2014-2018 MongoDB, Inc. + * Copyright (c) 2008-2014 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +#define WT_CM_BLOCKSIZE 8 +#define WT_CM_MINMATCH 64 +#define WT_CM_STARTGAP (WT_CM_BLOCKSIZE / 2) + +typedef struct { + WT_SESSION_IMPL *session; + + const uint8_t *s1, *e1; /* Start / end of pre-image. */ + const uint8_t *s2, *e2; /* Start / end of after-image. */ + + const uint8_t *used1, *used2; /* Used up to here. */ + + size_t maxdiff; + int maxentries; +} WT_CM_STATE; + +typedef struct { + const uint8_t *m1, *m2; + size_t len; +} WT_CM_MATCH; + +/* + * __cm_add_modify -- + * Add a modify operation to the list of entries. + * + * Fails if all entries are used or the maximum bytes of difference is + * exceeded. + */ +static int +__cm_add_modify(WT_CM_STATE *cms, const uint8_t *p2, + const uint8_t *m1, const uint8_t *m2, WT_MODIFY *entries, int *nentriesp) +{ + WT_MODIFY *mod; + size_t len1, len2; + + WT_ASSERT(cms->session, m1 >= cms->used1 && m2 >= cms->used2); + + len1 = (size_t)(m1 - cms->used1); + len2 = (size_t)(m2 - cms->used2); + + if (*nentriesp >= cms->maxentries || len2 > cms->maxdiff) + return (WT_NOTFOUND); + + mod = entries + (*nentriesp)++; + mod->offset = (size_t)(p2 - cms->s2); + mod->size = len1; + mod->data.data = p2; + mod->data.size = len2; + cms->maxdiff -= len2; + + return (0); +} + +/* + * __cm_extend -- + * Given a potential match size, extend to find the complete match. + */ +static void +__cm_extend(WT_CM_STATE *cms, + const uint8_t *m1, const uint8_t *m2, WT_CM_MATCH *match) +{ + const uint8_t *p1, *p2; + + /* Step past the end and before the beginning of the matching block. */ + for (p1 = m1, p2 = m2; + p1 < cms->e1 && p2 < cms->e2 && *p1 == *p2; + p1++, p2++) + ; + + for (; m1 >= cms->used1 && m2 >= cms->used2 && *m1 == *m2; + m1--, m2--) + ; + + match->m1 = m1 + 1; + match->m2 = m2 + 1; + match->len = p1 > m1 ? (size_t)((p1 - m1) - 1) : 0; +} + +/* + * __cm_fingerprint -- + * Calculate an integral "fingerprint" of a block of bytes. + */ +static inline uint64_t +__cm_fingerprint(const uint8_t *p) +{ + uint64_t h; + + WT_STATIC_ASSERT(sizeof(h) <= WT_CM_BLOCKSIZE); + memcpy(&h, p, WT_CM_BLOCKSIZE); + return (h); +} + +/* + * wiredtiger_calc_modify -- + * Calculate a set of WT_MODIFY operations to represent an update. + */ +int +wiredtiger_calc_modify(WT_SESSION *wt_session, + const WT_ITEM *oldv, const WT_ITEM *newv, + size_t maxdiff, WT_MODIFY *entries, int *nentriesp) +{ + WT_CM_MATCH match; + WT_CM_STATE cms; + size_t gap, i; + uint64_t h, hend, hstart; + const uint8_t *p1, *p2; + bool start; + + if (oldv->size < WT_CM_MINMATCH || newv->size < WT_CM_MINMATCH) + return (WT_NOTFOUND); + + cms.session = (WT_SESSION_IMPL *)wt_session; + + cms.s1 = cms.used1 = oldv->data; + cms.e1 = cms.s1 + oldv->size; + cms.s2 = cms.used2 = newv->data; + cms.e2 = cms.s2 + newv->size; + cms.maxdiff = maxdiff; + cms.maxentries = *nentriesp; + *nentriesp = 0; + + /* Ignore matches at the beginning / end. */ + __cm_extend(&cms, cms.s1, cms.s2, &match); + cms.used1 += match.len; + cms.used2 += match.len; + if (cms.used1 < cms.e1 && cms.used2 < cms.e2) { + __cm_extend(&cms, cms.e1 - 1, cms.e2 - 1, &match); + cms.e1 -= match.len; + cms.e2 -= match.len; + } + + if (cms.used1 + WT_CM_BLOCKSIZE >= cms.e1 || + cms.used2 + WT_CM_BLOCKSIZE >= cms.e2) + goto end; + + /* + * Walk through the post-image, maintaining start / end markers + * separated by a gap in the pre-image. If the current point in the + * post-image matches either marker, try to extend the match to find a + * (large) range of matching bytes. If the end of the range is reached + * in the post-image without finding a good match, double the size of + * the gap, update the markers and keep trying. + */ + hstart = hend = 0; + i = gap = 0; + for (p1 = cms.used1, p2 = cms.used2, start = true; + p1 + WT_CM_BLOCKSIZE <= cms.e1 && p2 + WT_CM_BLOCKSIZE <= cms.e2; + p2++, i++) { + if (start || i == gap) { + p1 += gap; + gap = start ? WT_CM_STARTGAP : gap * 2; + if (p1 + gap + WT_CM_BLOCKSIZE >= cms.e1) + break; + if (gap > maxdiff) + return (WT_NOTFOUND); + hstart = start ? __cm_fingerprint(p1) : hend; + hend = __cm_fingerprint(p1 + gap); + start = false; + i = 0; + } + h = __cm_fingerprint(p2); + match.len = 0; + if (h == hstart) + __cm_extend(&cms, p1, p2, &match); + else if (h == hend) + __cm_extend(&cms, p1 + gap, p2, &match); + + if (match.len < WT_CM_MINMATCH) + continue; + + WT_RET(__cm_add_modify(&cms, cms.used2, match.m1, match.m2, + entries, nentriesp)); + cms.used1 = p1 = match.m1 + match.len; + cms.used2 = p2 = match.m2 + match.len; + start = true; + } + +end: if (cms.used1 < cms.e1 || cms.used2 < cms.e2) + WT_RET(__cm_add_modify(&cms, cms.used2, cms.e1, cms.e2, + entries, nentriesp)); + + return (0); +} diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index 085100dc2b3..c01fd84b7d0 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -2613,10 +2613,11 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, } WT_ERR(__wt_config_gets(session, cfg, "buffer_alignment", &cval)); - if (cval.val == -1) - conn->buffer_alignment = - (conn->direct_io == 0) ? 0 : WT_BUFFER_ALIGNMENT_DEFAULT; - else + if (cval.val == -1) { + conn->buffer_alignment = 0; + if (conn->direct_io != 0) + conn->buffer_alignment = WT_BUFFER_ALIGNMENT_DEFAULT; + } else conn->buffer_alignment = (size_t)cval.val; #ifndef HAVE_POSIX_MEMALIGN if (conn->buffer_alignment != 0) diff --git a/src/third_party/wiredtiger/src/conn/conn_handle.c b/src/third_party/wiredtiger/src/conn/conn_handle.c index 42b8510cbd3..e8d5c41b167 100644 --- a/src/third_party/wiredtiger/src/conn/conn_handle.c +++ b/src/third_party/wiredtiger/src/conn/conn_handle.c @@ -57,11 +57,11 @@ __wt_connection_init(WT_CONNECTION_IMPL *conn) WT_RET(__wt_spin_init(session, &conn->fh_lock, "file list")); WT_SPIN_INIT_TRACKED(session, &conn->metadata_lock, metadata); WT_RET(__wt_spin_init(session, &conn->reconfig_lock, "reconfigure")); - WT_SPIN_INIT_TRACKED(session, &conn->schema_lock, schema); + WT_SPIN_INIT_SESSION_TRACKED(session, &conn->schema_lock, schema); WT_RET(__wt_spin_init(session, &conn->turtle_lock, "turtle file")); /* Read-write locks */ - WT_RWLOCK_INIT_TRACKED(session, &conn->dhandle_lock, dhandle); + WT_RWLOCK_INIT_SESSION_TRACKED(session, &conn->dhandle_lock, dhandle); WT_RET(__wt_rwlock_init(session, &conn->hot_backup_lock)); WT_RWLOCK_INIT_TRACKED(session, &conn->table_lock, table); diff --git a/src/third_party/wiredtiger/src/conn/conn_log.c b/src/third_party/wiredtiger/src/conn/conn_log.c index dc6e4a82509..f04407d2e10 100644 --- a/src/third_party/wiredtiger/src/conn/conn_log.c +++ b/src/third_party/wiredtiger/src/conn/conn_log.c @@ -289,6 +289,10 @@ __logmgr_config( WT_STAT_CONN_SET(session, log_max_filesize, conn->log_file_max); } + WT_RET(__wt_config_gets(session, cfg, "log.os_cache_dirty_pct", &cval)); + if (cval.val != 0) + conn->log_dirty_max = (conn->log_file_max * cval.val) / 100; + /* * If pre-allocation is configured, set the initial number to a few. * We'll adapt as load dictates. diff --git a/src/third_party/wiredtiger/src/conn/conn_stat.c b/src/third_party/wiredtiger/src/conn/conn_stat.c index ffbc1caf2bb..0e1ceae2fa0 100644 --- a/src/third_party/wiredtiger/src/conn/conn_stat.c +++ b/src/third_party/wiredtiger/src/conn/conn_stat.c @@ -80,7 +80,7 @@ __wt_conn_stat_init(WT_SESSION_IMPL *session) WT_STAT_SET(session, stats, file_open, conn->open_file_count); WT_STAT_SET(session, - stats, session_cursor_open, conn->open_cursor_count); + stats, cursor_open_count, conn->open_cursor_count); WT_STAT_SET(session, stats, dh_conn_handle_count, conn->dhandle_count); WT_STAT_SET(session, stats, rec_split_stashed_objects, conn->stashed_objects); @@ -613,6 +613,9 @@ __statlog_server(void *arg) session = arg; conn = S2C(session); + WT_CLEAR(path); + WT_CLEAR(tmp); + /* * We need a temporary place to build a path and an entry prefix. * The length of the path plus 128 should be more than enough. @@ -620,10 +623,8 @@ __statlog_server(void *arg) * We also need a place to store the current path, because that's * how we know when to close/re-open the file. */ - WT_CLEAR(path); WT_ERR(__wt_buf_init(session, &path, strlen(conn->stat_path) + 128)); WT_ERR(__wt_buf_setstr(session, &path, "")); - WT_CLEAR(tmp); WT_ERR(__wt_buf_init(session, &tmp, strlen(conn->stat_path) + 128)); WT_ERR(__wt_buf_setstr(session, &tmp, "")); diff --git a/src/third_party/wiredtiger/src/cursor/cur_backup.c b/src/third_party/wiredtiger/src/cursor/cur_backup.c index 32021a6f7e2..91bf3bb4931 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_backup.c +++ b/src/third_party/wiredtiger/src/cursor/cur_backup.c @@ -13,9 +13,10 @@ static int __backup_list_append( WT_SESSION_IMPL *, WT_CURSOR_BACKUP *, const char *); static int __backup_list_uri_append(WT_SESSION_IMPL *, const char *, bool *); static int __backup_start( - WT_SESSION_IMPL *, WT_CURSOR_BACKUP *, const char *[]); + WT_SESSION_IMPL *, WT_CURSOR_BACKUP *, bool, const char *[]); static int __backup_stop(WT_SESSION_IMPL *, WT_CURSOR_BACKUP *); -static int __backup_uri(WT_SESSION_IMPL *, const char *[], bool *, bool *); +static int __backup_uri( + WT_SESSION_IMPL *, const char *[], bool, bool *, bool *); /* * __curbackup_next -- @@ -66,6 +67,23 @@ err: API_END_RET(session, ret); } /* + * __backup_free -- + * Free list resources for a backup cursor. + */ +static void +__backup_free(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb) +{ + int i; + + if (cb->list != NULL) { + for (i = 0; cb->list[i] != NULL; ++i) + __wt_free(session, cb->list[i]); + __wt_free(session, cb->list); + } + +} + +/* * __curbackup_close -- * WT_CURSOR->close method for the backup cursor type. */ @@ -89,7 +107,13 @@ err: * discarded when the cursor is closed), because that cursor will never * not be responsible for cleanup. */ - if (F_ISSET(cb, WT_CURBACKUP_LOCKER)) + if (F_ISSET(cb, WT_CURBACKUP_DUP)) { + __backup_free(session, cb); + /* Make sure the original backup cursor is still open. */ + WT_ASSERT(session, F_ISSET(session, WT_SESSION_BACKUP_CURSOR)); + F_CLR(session, WT_SESSION_BACKUP_DUP); + F_CLR(cb, WT_CURBACKUP_DUP); + } else if (F_ISSET(cb, WT_CURBACKUP_LOCKER)) WT_TRET(__backup_stop(session, cb)); __wt_cursor_close(cursor); @@ -103,8 +127,8 @@ err: * WT_SESSION->open_cursor method for the backup cursor type. */ int -__wt_curbackup_open(WT_SESSION_IMPL *session, - const char *uri, const char *cfg[], WT_CURSOR **cursorp) +__wt_curbackup_open(WT_SESSION_IMPL *session, const char *uri, + WT_CURSOR *other, const char *cfg[], WT_CURSOR **cursorp) { WT_CURSOR_STATIC_INIT(iface, __wt_cursor_get_key, /* get-key */ @@ -148,7 +172,7 @@ __wt_curbackup_open(WT_SESSION_IMPL *session, */ WT_WITH_CHECKPOINT_LOCK(session, WT_WITH_SCHEMA_LOCK(session, - ret = __backup_start(session, cb, cfg))); + ret = __backup_start(session, cb, other != NULL, cfg))); WT_ERR(ret); WT_ERR(__wt_cursor_init(cursor, uri, NULL, cfg, cursorp)); @@ -193,8 +217,8 @@ err: WT_TRET(__wt_fs_directory_list_free(session, &logfiles, logcount)); * Start a backup. */ static int -__backup_start( - WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb, const char *cfg[]) +__backup_start(WT_SESSION_IMPL *session, + WT_CURSOR_BACKUP *cb, bool is_dup, const char *cfg[]) { WT_CONNECTION_IMPL *conn; WT_DECL_RET; @@ -216,48 +240,71 @@ __backup_start( * Single thread hot backups: we're holding the schema lock, so we * know we'll serialize with other attempts to start a hot backup. */ - if (conn->hot_backup) + if (conn->hot_backup && !is_dup) WT_RET_MSG( session, EINVAL, "there is already a backup cursor open"); - /* - * The hot backup copy is done outside of WiredTiger, which means file - * blocks can't be freed and re-allocated until the backup completes. - * The checkpoint code checks the backup flag, and if a backup cursor - * is open checkpoints aren't discarded. We release the lock as soon - * as we've set the flag, we don't want to block checkpoints, we just - * want to make sure no checkpoints are deleted. The checkpoint code - * holds the lock until it's finished the checkpoint, otherwise we - * could start a hot backup that would race with an already-started - * checkpoint. - * - * We are holding the checkpoint and schema locks so schema operations - * will not see the backup file list until it is complete and valid. - */ - __wt_writelock(session, &conn->hot_backup_lock); - conn->hot_backup = true; - conn->hot_backup_list = NULL; - __wt_writeunlock(session, &conn->hot_backup_lock); + if (F_ISSET(session, WT_SESSION_BACKUP_DUP) && is_dup) + WT_RET_MSG(session, EINVAL, + "there is already a duplicate backup cursor open"); - /* We're the lock holder, we own cleanup. */ - F_SET(cb, WT_CURBACKUP_LOCKER); + if (!is_dup) { + /* + * The hot backup copy is done outside of WiredTiger, which + * means file blocks can't be freed and re-allocated until the + * backup completes. The checkpoint code checks the backup flag, + * and if a backup cursor is open checkpoints aren't discarded. + * We release the lock as soon as we've set the flag, we don't + * want to block checkpoints, we just want to make sure no + * checkpoints are deleted. The checkpoint code holds the lock + * until it's finished the checkpoint, otherwise we could start + * a hot backup that would race with an already-started + * checkpoint. + * + * We are holding the checkpoint and schema locks so schema + * operations will not see the backup file list until it is + * complete and valid. + */ + __wt_writelock(session, &conn->hot_backup_lock); + conn->hot_backup = true; + conn->hot_backup_list = NULL; + __wt_writeunlock(session, &conn->hot_backup_lock); + + /* We're the lock holder, we own cleanup. */ + F_SET(cb, WT_CURBACKUP_LOCKER); + + /* + * Create a temporary backup file. This must be opened before + * generating the list of targets in backup_uri. This file will + * later be renamed to the correct name depending on whether or + * not we're doing an incremental backup. We need a temp file + * so that if we fail or crash while filling it, the existence + * of a partial file doesn't confuse restarting in the source + * database. + */ + WT_ERR(__wt_fopen(session, WT_BACKUP_TMP, + WT_FS_OPEN_CREATE, WT_STREAM_WRITE, &cb->bfs)); + } - /* - * Create a temporary backup file. This must be opened before - * generating the list of targets in backup_uri. This file will - * later be renamed to the correct name depending on whether or not - * we're doing an incremental backup. We need a temp file so that if - * we fail or crash while filling it, the existence of a partial file - * doesn't confuse restarting in the source database. - */ - WT_ERR(__wt_fopen(session, WT_BACKUP_TMP, - WT_FS_OPEN_CREATE, WT_STREAM_WRITE, &cb->bfs)); /* * If targets were specified, add them to the list. Otherwise it is a * full backup, add all database objects and log files to the list. */ target_list = false; - WT_ERR(__backup_uri(session, cfg, &target_list, &log_only)); + WT_ERR(__backup_uri(session, + cfg, is_dup, &target_list, &log_only)); + /* + * For a duplicate cursor, all the work is done in backup_uri. The only + * usage accepted is "target=("log:")" so error if not log only. + */ + if (is_dup) { + if (!log_only) + WT_ERR_MSG(session, EINVAL, + "duplicate backup cursor must be for logs only."); + F_SET(cb, WT_CURBACKUP_DUP); + F_SET(session, WT_SESSION_BACKUP_DUP); + goto done; + } if (!target_list) { /* * It's important to first gather the log files to be copied @@ -282,9 +329,15 @@ __backup_start( /* Add the hot backup and standard WiredTiger files to the list. */ if (log_only) { /* - * We also open an incremental backup source file so that we - * can detect a crash with an incremental backup existing in - * the source directory versus an improper destination. + * If this is not a duplicate cursor, using the log target is an + * incremental backup. If this is a duplicate cursor then using + * the log target on an existing backup cursor means this cursor + * returns the current list of log files. That list was set up + * when parsing the URI so we don't have anything to do here. + * + * We also open an incremental backup source file so that we can + * detect a crash with an incremental backup existing in the + * source directory versus an improper destination. */ dest = WT_INCREMENTAL_BACKUP; WT_ERR(__wt_fopen(session, WT_INCREMENTAL_SRC, @@ -306,7 +359,8 @@ __backup_start( } err: /* Close the hot backup file. */ - WT_TRET(__wt_fclose(session, &cb->bfs)); + if (cb->bfs != NULL) + WT_TRET(__wt_fclose(session, &cb->bfs)); if (srcfs != NULL) WT_TRET(__wt_fclose(session, &srcfs)); if (ret == 0) { @@ -315,8 +369,10 @@ err: /* Close the hot backup file. */ __wt_writelock(session, &conn->hot_backup_lock); conn->hot_backup_list = cb->list; __wt_writeunlock(session, &conn->hot_backup_lock); + F_SET(session, WT_SESSION_BACKUP_CURSOR); } +done: return (ret); } @@ -329,27 +385,26 @@ __backup_stop(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb) { WT_CONNECTION_IMPL *conn; WT_DECL_RET; - int i; conn = S2C(session); /* Release all btree names held by the backup. */ + WT_ASSERT(session, !F_ISSET(cb, WT_CURBACKUP_DUP)); + /* If it's not a dup backup cursor, make sure one isn't open. */ + WT_ASSERT(session, !F_ISSET(session, WT_SESSION_BACKUP_DUP)); __wt_writelock(session, &conn->hot_backup_lock); conn->hot_backup_list = NULL; __wt_writeunlock(session, &conn->hot_backup_lock); - if (cb->list != NULL) { - for (i = 0; cb->list[i] != NULL; ++i) - __wt_free(session, cb->list[i]); - __wt_free(session, cb->list); - } + __backup_free(session, cb); /* Remove any backup specific file. */ WT_TRET(__wt_backup_file_remove(session)); - /* Checkpoint deletion can proceed, as can the next hot backup. */ + /* Checkpoint deletion and next hot backup can proceed. */ __wt_writelock(session, &conn->hot_backup_lock); conn->hot_backup = false; __wt_writeunlock(session, &conn->hot_backup_lock); + F_CLR(session, WT_SESSION_BACKUP_CURSOR); return (ret); } @@ -371,8 +426,8 @@ __backup_all(WT_SESSION_IMPL *session) * Backup a list of objects. */ static int -__backup_uri(WT_SESSION_IMPL *session, - const char *cfg[], bool *foundp, bool *log_only) +__backup_uri(WT_SESSION_IMPL *session, const char *cfg[], + bool is_dup, bool *foundp, bool *log_only) { WT_CONFIG targetconf; WT_CONFIG_ITEM cval, k, v; @@ -413,9 +468,10 @@ __backup_uri(WT_SESSION_IMPL *session, if (WT_PREFIX_MATCH(uri, "log:")) { /* * Log archive cannot mix with incremental backup, don't - * let that happen. + * let that happen. If we're a duplicate cursor + * archiving is already temporarily suspended. */ - if (FLD_ISSET( + if (!is_dup && FLD_ISSET( S2C(session)->log_flags, WT_CONN_LOG_ARCHIVE)) WT_ERR_MSG(session, EINVAL, "incremental backup not possible when " diff --git a/src/third_party/wiredtiger/src/cursor/cur_stat.c b/src/third_party/wiredtiger/src/cursor/cur_stat.c index a65043b138a..d49830c19b2 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_stat.c +++ b/src/third_party/wiredtiger/src/cursor/cur_stat.c @@ -270,6 +270,10 @@ __curstat_reset(WT_CURSOR *cursor) cst->notinitialized = cst->notpositioned = true; F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); + /* Reset the session statistics to zero. */ + if (strcmp(cursor->uri, "statistics:session") == 0) + __wt_stat_session_clear_single(&session->stats); + err: API_END_RET(session, ret); } @@ -520,15 +524,6 @@ __curstat_join_init(WT_SESSION_IMPL *session, static void __curstat_session_init(WT_SESSION_IMPL *session, WT_CURSOR_STAT *cst) { - /* This is a stub at the moment, initialize the session stats to 0. */ - session->stats.bytes_read = 0; - session->stats.bytes_write = 0; - session->stats.cache_time = 0; - session->stats.lock_dhandle_wait = 0; - session->stats.lock_schema_wait = 0; - session->stats.read_time = 0; - session->stats.write_time = 0; - /* * Copy stats from the session to the cursor. Optionally clear the * session's statistics. diff --git a/src/third_party/wiredtiger/src/cursor/cur_std.c b/src/third_party/wiredtiger/src/cursor/cur_std.c index 3217b55cfa4..bef7dc615a9 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_std.c +++ b/src/third_party/wiredtiger/src/cursor/cur_std.c @@ -620,9 +620,8 @@ __wt_cursor_cache(WT_CURSOR *cursor, WT_DATA_HANDLE *dhandle) TAILQ_INSERT_HEAD(&session->cursor_cache[bucket], cursor, q); (void)__wt_atomic_sub32(&S2C(session)->open_cursor_count, 1); - WT_STAT_DATA_DECR(session, session_cursor_open); - WT_STAT_DATA_INCR(session, session_cursors_cached); - WT_STAT_CONN_INCR(session, cursors_cached); + WT_STAT_CONN_INCR_ATOMIC(session, cursor_cached_count); + WT_STAT_DATA_DECR(session, cursor_open_count); F_SET(cursor, WT_CURSTD_CACHED); return (ret); } @@ -646,9 +645,8 @@ __wt_cursor_reopen(WT_CURSOR *cursor, WT_DATA_HANDLE *dhandle) WT_DHANDLE_RELEASE(dhandle); } (void)__wt_atomic_add32(&S2C(session)->open_cursor_count, 1); - WT_STAT_DATA_INCR(session, session_cursor_open); - WT_STAT_DATA_DECR(session, session_cursors_cached); - WT_STAT_CONN_DECR(session, cursors_cached); + WT_STAT_CONN_DECR_ATOMIC(session, cursor_cached_count); + WT_STAT_DATA_INCR(session, cursor_open_count); bucket = cursor->uri_hash % WT_HASH_ARRAY_SIZE; TAILQ_REMOVE(&session->cursor_cache[bucket], cursor, q); @@ -870,7 +868,7 @@ __wt_cursor_close(WT_CURSOR *cursor) TAILQ_REMOVE(&session->cursors, cursor, q); (void)__wt_atomic_sub32(&S2C(session)->open_cursor_count, 1); - WT_STAT_DATA_DECR(session, session_cursor_open); + WT_STAT_DATA_DECR(session, cursor_open_count); } __wt_buf_free(session, &cursor->key); __wt_buf_free(session, &cursor->value); @@ -1142,7 +1140,7 @@ __wt_cursor_init(WT_CURSOR *cursor, F_SET(cursor, WT_CURSTD_OPEN); (void)__wt_atomic_add32(&S2C(session)->open_cursor_count, 1); - WT_STAT_DATA_INCR(session, session_cursor_open); + WT_STAT_DATA_INCR(session, cursor_open_count); *cursorp = (cdump != NULL) ? cdump : cursor; return (0); diff --git a/src/third_party/wiredtiger/src/docs/Doxyfile b/src/third_party/wiredtiger/src/docs/Doxyfile index 60b6c4690b0..567a8cb92fb 100644 --- a/src/third_party/wiredtiger/src/docs/Doxyfile +++ b/src/third_party/wiredtiger/src/docs/Doxyfile @@ -210,7 +210,7 @@ ALIASES = \ configempty{2}="@param config configuration string, see @ref config_strings. No values currently permitted." \ configend=" </table>" \ configstart{2}="@param config configuration string, see @ref config_strings. Permitted values:\n <table>@hrow{Name,Effect,Values}" \ -"ebusy_errors=@returns zero on success, EBUSY if the object is not available for exclusive access, and a non-zero error code on failure. See @ref error_handling \"Error handling\" for details." \ + ebusy_errors="@returns zero on success, EBUSY if the object is not available for exclusive access, and a non-zero error code on failure. See @ref error_handling \"Error handling\" for details." \ errors="@returns zero on success and a non-zero error code on failure. See @ref error_handling \"Error handling\" for details." \ exclusive="This method requires exclusive access to the specified data source(s). If any cursors are open with the specified name(s) or a data source is otherwise in use, the call will fail and return \c EBUSY.\n\n" \ ex_ref{1}="@ref \1 \"\1\"" \ diff --git a/src/third_party/wiredtiger/src/docs/compression.dox b/src/third_party/wiredtiger/src/docs/compression.dox index 74bed5c6f68..147c4fe2936 100644 --- a/src/third_party/wiredtiger/src/docs/compression.dox +++ b/src/third_party/wiredtiger/src/docs/compression.dox @@ -121,7 +121,7 @@ an extension. For example, with the WiredTiger library installed in @snippet ex_all.c Configure zstd extension -The default compression level for the zstd compression is 3; compression +The default compression level for the zstd compression is 6; compression can be configured to other levels using the additional configuration argument \c compression_level. diff --git a/src/third_party/wiredtiger/src/docs/cursors.dox b/src/third_party/wiredtiger/src/docs/cursors.dox index b6271951f91..6244f02514a 100644 --- a/src/third_party/wiredtiger/src/docs/cursors.dox +++ b/src/third_party/wiredtiger/src/docs/cursors.dox @@ -63,7 +63,8 @@ At \c read-committed (the default) or \c snapshot isolation levels, committed changes from concurrent transactions become visible when no cursor is positioned. In other words, at these isolation levels, all cursors in a session read from a stable snapshot while any cursor in the -session remains positioned. +session remains positioned. A call to WT_CURSOR::next or WT_CURSOR::prev +on a positioned cursor will not update the snapshot. Cursor positions survive transaction boundaries, unless a transaction is rolled back. When a transaction is rolled-back either implicitly diff --git a/src/third_party/wiredtiger/src/docs/data-sources.dox b/src/third_party/wiredtiger/src/docs/data-sources.dox index 7f1879e0ffe..aaee4adc04a 100644 --- a/src/third_party/wiredtiger/src/docs/data-sources.dox +++ b/src/third_party/wiredtiger/src/docs/data-sources.dox @@ -38,7 +38,7 @@ cursor types that give access to data managed by WiredTiger: key=<code>string</code>\, value=<code>string</code>\,<br> see @ref metadata for details} @row{<tt>statistics:[\<data source URI\>]</tt>, - database, data source or join statistics cursor, + database, data source, join or session statistics cursor, key=<code>int id</code>\,<br> value=<code>(string description\, string value\, uint64_t value)</code>\,<br> @@ -106,9 +106,10 @@ WiredTiger database as well as statistics for individual data sources. The statistics are at two levels: per-database and per-individual data source. Database-wide statistics are retrieved with the \c "statistics:" URI; individual data source statistics are available by specifying -\c "statistics:<data source URI>". Additionally, statistics about a -join cursor can be retrieved by specifying \c "statistics:join" and -supplying the join cursor as an argument in the SESSION::open_cursor call. +\c "statistics:<data source URI>". Statistics about a join cursor can be +retrieved by specifying \c "statistics:join" and supplying the join cursor as an +argument in the SESSION::open_cursor call. Statistics about a session can be +retrieved by specifying \c "statistics:session". The statistic key is an integer from the list of keys in @ref_single statistics_keys "Statistics Keys". Statistics cursors return @@ -118,7 +119,8 @@ unsigned 64-bit integral value, respectively. The cursor's statistics values are loaded when the cursor is opened and remain unchanged for the life of the cursor, unless the cursor is reset -by calling the WT_CURSOR::reset method, which reloads the values. +by calling the WT_CURSOR::reset method, which reloads the values. A reset call +on session statistics cursor would set the statistics values to zero. The following is an example of printing run-time statistics about the WiredTiger engine: @@ -133,7 +135,11 @@ The following is an example of printing statistics about a join cursor: @snippet ex_stat.c statistics join cursor function -These three examples can use a common display routine that iterates through the +The following is an example of printing statistics about a session cursor: + +@snippet ex_stat.c statistics session function + +These four examples can use a common display routine that iterates through the statistics until the cursor returns the end of the list. @snippet ex_stat.c statistics display function diff --git a/src/third_party/wiredtiger/src/docs/spell.ok b/src/third_party/wiredtiger/src/docs/spell.ok index d5347709248..b336b5ab17a 100644 --- a/src/third_party/wiredtiger/src/docs/spell.ok +++ b/src/third_party/wiredtiger/src/docs/spell.ok @@ -152,6 +152,7 @@ colgroups combinatorial command's comparator +compressibility cond config configurign @@ -355,7 +356,6 @@ nolock nolocking nommap nop -noraw nosql nosync notgranted @@ -382,9 +382,9 @@ perf petabyte pget php +pid plantuml png -pid posix pre prepends @@ -465,7 +465,6 @@ substring superset sys sz -tid tRuE tablename tcl @@ -473,6 +472,7 @@ tcmalloc td th thang +tid timestamp timestamps todo diff --git a/src/third_party/wiredtiger/src/docs/statistics.dox b/src/third_party/wiredtiger/src/docs/statistics.dox index 19b7b17257b..3735a009d56 100644 --- a/src/third_party/wiredtiger/src/docs/statistics.dox +++ b/src/third_party/wiredtiger/src/docs/statistics.dox @@ -74,8 +74,8 @@ when the cursor was opened), as well as statistics logging, when it is configured. The following example configures WiredTiger to maintain only relatively -inexpensive statistics, and to clears statistics after they are gathered -or logged: +inexpensive statistics, and to clear statistics after they are gathered or +logged: @snippet ex_all.c Statistics clear configuration @@ -88,6 +88,14 @@ in the join (see WT_SESSION::join); the uri of each reference cursor appears as a prefix in the description field returned as a value by the statistics cursor. +The following example opens a statistics cursor on an open WiredTiger session: + +@snippet ex_all.c Statistics cursor session + +A reset call on this cursor would reset the session statistics values to zero, +with or without the statistics configuration \c clear being specified to the +WT_SESSION::open_cursor method. + @section statistics_log Statistics logging WiredTiger will optionally log database statistics into files when the diff --git a/src/third_party/wiredtiger/src/docs/tune-page-size-and-comp.dox b/src/third_party/wiredtiger/src/docs/tune-page-size-and-comp.dox index 96b0fda2333..9659d389bf0 100644 --- a/src/third_party/wiredtiger/src/docs/tune-page-size-and-comp.dox +++ b/src/third_party/wiredtiger/src/docs/tune-page-size-and-comp.dox @@ -107,7 +107,7 @@ trade-off between frequency of exclusive access to the pages (for reconciliation or splitting pages into smaller pages) versus the duration that the exclusive access is required. - Configuration: -\n Specified as memory_page_max configuration option to WT_SESSION::create(). An +\n Specified as memory_page_max configuration option to WT_SESSION::create. An example of such a configuration string is as follows: <pre> @@ -141,7 +141,7 @@ number of keys that can be stored in an internal page, which is internal_page_max divided by key size. Applications should choose an appropriate internal_page_max size that avoids the B-Tree from getting too deep. - Configuration: -\n Specified as internal_page_max configuration option to WT_SESSION::create(). +\n Specified as internal_page_max configuration option to WT_SESSION::create. An example of such a configuration string is as follows: <pre> @@ -167,7 +167,7 @@ increase leaf_page_max to transfer more data per I/O. - Applications focused on read/write amplification might decrease the page size to better match the underlying storage block size. - Configuration: -\n Specified as leaf_page_max configuration option to WT_SESSION::create(). An +\n Specified as leaf_page_max configuration option to WT_SESSION::create. An example of such a configuration string is as follows: <pre> @@ -193,7 +193,7 @@ on the platform (4KB for most common server platforms) 18,000 bytes requires 5 allocation units and wastes about 2KB of space. If the allocation size is 16KB, the same overflow item would waste more than 10KB. - Configuration: -\n Specified as allocation_size configuration option to WT_SESSION::create(). An +\n Specified as allocation_size configuration option to WT_SESSION::create. An example of such a configuration string is as follows: <pre> @@ -260,7 +260,7 @@ size will be ignored when the larger keys and values are being written, and a larger page will be created as necessary. - Configuration: \n Specified as internal_key_max, leaf_key_max and leaf_value_max configuration -options to WT_SESSION::create(). An example of configuration string for a large +options to WT_SESSION::create. An example of configuration string for a large leaf overflow value: <pre> @@ -287,7 +287,7 @@ pages grow to 1MB over and over. The default value for split_pct is 75%, intended to keep large pages relatively large, while still giving split pages room to grow. - Configuration: -\n Specified as split_pct configuration option to WT_SESSION::create(). An +\n Specified as split_pct configuration option to WT_SESSION::create. An example of such a configuration string is as follows: <pre> @@ -297,11 +297,11 @@ example of such a configuration string is as follows: @section compression_considerations Compression considerations WiredTiger compresses data at several stages to preserve memory and disk space. Applications can configure these different compression algorithms to tailor -their requirements between memory, disk and CPU consumption. Compression +their requirements between memory, disk and CPU consumption. Compression algorithms other than block compression work by modifying how the keys and -values are represented, and hence reduce data size in-memory and on-disk. Block -compression on the other hand compress the data in its binary representation -while saving it on the disk. +values are represented, and will reduce data size both in-memory and on-disk. +Block compression compresses the data in its binary representation while +saving it on the disk, and so only reduces the data size on-disk. Configuring compression may change application throughput. For example, in applications using solid-state drives (where I/O is less expensive), turning @@ -310,102 +310,90 @@ applications where I/O costs are more expensive, turning on compression may increase application performance by reducing the overall number of I/O operations. -WiredTiger uses some internal algorithms to compress the amount of data stored -that are not configurable, but always on. For example, run-length reduces the -size requirement by storing sequential, duplicate values in the store only a -single time (with an associated count). +WiredTiger uses some internal algorithms to compress the amount of data +stored that are not configurable, but always on. For example, run-length +encoding reduces in-memory and on-disk size requirements by storing +sequential, duplicate values in a column-store object only a single time. Different compression options available with WiredTiger: - - Key-prefix - - Reduces the size requirement by storing any identical key prefix only once -per page. The cost is additional CPU and memory when operating on the in-memory -tree. Specifically, reverse sequential cursor movement (but not forward) through -a prefix-compressed page or the random lookup of a key/value pair will allocate -sufficient memory to hold some number of uncompressed keys. So, for example, if -key prefix compression only saves a small number of bytes per key, the -additional memory cost of instantiating the uncompressed key may mean prefix -compression is not worthwhile. Further, in cases where the on-disk cost is the -primary concern, block compression may mean prefix compression is less useful. - - Configuration: -\n Specified as prefix_compression configuration option to -WT_SESSION::create(). Applications may limit the use of prefix compression by -configuring the minimum number of bytes that must be gained before prefix -compression is used with prefix_compression_min configuration option. An example -of such a configuration string is as follows: - +- Key-prefix +\n +Reduces the size requirement by storing any identical key prefix +only once per page. The cost is additional CPU and memory when operating on +the in-memory tree. Specifically, reverse sequential cursor movement (but not +forward) through a prefix-compressed page or the random lookup of a key/value +pair will allocate sufficient memory to hold some number of uncompressed keys. +So, for example, if key prefix compression only saves a small number of bytes +per key, the additional memory cost of instantiating the uncompressed key may +mean prefix compression is not worthwhile. Further, in cases where the on-disk +cost is the primary concern, block compression may mean prefix compression is +less useful. +\n\n +Key-prefix configuration: +\n +Specified using \c the prefix_compression configuration option to +WT_SESSION::create. Applications may limit the use of prefix compression +by configuring the minimum number of bytes that must be gained before +prefix compression is used with the \c prefix_compression_min configuration +option. An example of such a configuration string is as follows: <pre> - "key_format=S,value_format=S,prefix_compression=true,prefix_compression_min=7" +"key_format=S,value_format=S,prefix_compression=true,prefix_compression_min=7" </pre> - - Dictionary - - Reduces the size requirement by storing any identical value only once per -page. - - Configuration: -\n Specified as dictionary configuration configuration option to -WT_SESSION::create(), which specifies the maximum number of unique values +- Dictionary +\n +Reduces the size requirement by storing any identical value only once per page. +\n\n +Dictionary configuration: +\n +Specified using the \c dictionary configuration configuration option to +WT_SESSION::create, which specifies the maximum number of unique values remembered in the B-Tree row-store leaf page value dictionary. An example of such a configuration string is as follows: - <pre> - "key_format=S,value_format=S,dictionary=1000" +"key_format=S,value_format=S,dictionary=1000" </pre> - - Huffman - - Reduces the size requirement by compressing individual key/value items, and -can be separately configured either or both keys and values. The additional CPU -cost of Huffman encoding can be high, and should be considered. (See Huffman -Encoding for details.) - - Configuration: -\n Specified as huffman_key and/or huffman_value configuration option to -WT_SESSION::create(). These options can take values of "english" (to use a -built-in English language frequency table), "utf8<file>" or "utf16<file>" (to -use a custom utf8 or utf16 symbol frequency table file). An example of such a -configuration string is as follows: - +- Huffman +\n +Reduces the size requirement by compressing individual key/value items, and +can be separately configured either or both keys and values. The additional +CPU cost of Huffman encoding can be high, and should be considered. (See @ref +huffman for details.) +\n\n +Huffman configuration: +\n +Specified using the \c huffman_key and \c huffman_value configuration +options to WT_SESSION::create. These options can take values of "english" +(to use a built-in English language frequency table), "utf8<file>" or +"utf16<file>" (to use a custom UTF-8 or UTF-16 symbol frequency table file). +An example of such a configuration string is as follows: <pre> - "key_format=S,value_format=S,huffman_key=english,huffman_value=english" +"key_format=S,value_format=S,huffman_key=english,huffman_value=english" </pre> - - Block Compression - - Reduces the size requirement of on-disk objects by compressing blocks of +- Block Compression +\n +Reduces the size requirement of on-disk objects by compressing blocks of the backing object's file. The additional CPU cost of block compression can be high, and should be considered. When block compression has been configured, configured page sizes will not match the actual size of the page on disk. - - WiredTiger provides two methods of compressing your data when using block -compression: the raw and noraw methods. These methods change how WiredTiger -works to fit data into the blocks that are stored on disk. Applications needing -to write specific sized blocks may want to consider implementing a -WT_COMPRESSOR::compress_raw function. - - Noraw compression: -\n A fixed amount of data is given to the compression system, then turned into -a compressed block of data. The amount of data chosen to compress is the data -needed to fill the uncompressed block. Thus when compressed, the block will be -smaller than the normal data size and the sizes written to disk will often vary -depending on how compressible the data being stored is. Algorithms using noraw -compression include zlib-noraw, lz4-noraw and snappy. -Noraw compression is better suited for workloads with random access patterns -because each block will tend to be smaller and require less work to read and -decompress. - - Raw compression: -\n WiredTiger's raw compression takes advantage of compressors that provide a -streaming compression API. Using the streaming API WiredTiger will try to fit as -much data as possible into one block. This means that blocks created with raw -compression should be of similar size. Using a streaming compression method -should also make for less overhead in compression, as the setup and initial work -for compressing is done fewer times compared to the amount of data stored. -Algorithms using raw compression include zlib, lz4. -Compared to noraw, raw compression provides more compression while using more -CPU. Raw compression may provide a performance advantage in workloads where data -is accessed sequentially. That is because more data is generally packed into -each block on disk. - - Configuration: -\n Specified as the block_compressor configuration option to -WT_SESSION::create(). If WiredTiger has builtin support for "lz4", "snappy", -"zlib" or "zstd" compression, these names are available as the value to the -option. An example of such a configuration string is as follows: - +\n +When block compression is configured, chunks of data are given to the +compression system, then returned as a compressed block of data. The amount of +data chosen to compress is based on previous compression results. When +compressed, the size written to disk will vary depending on the compressibility +of the stored data. +\n\n +Block compression configuration: +\n +Specified using the \c block_compressor configuration option to +WT_SESSION::create. If WiredTiger was built with support for "lz4", "snappy", +"zlib" or "zstd" compression, these names are available as the value to +the configuration option. An example of such a configuration string is as +follows: <pre> - "key_format=S,value_format=S,block_compressor=snappy" +"key_format=S,value_format=S,block_compressor=snappy" </pre> See @ref compression for further information on how to configure and enable diff --git a/src/third_party/wiredtiger/src/docs/upgrading.dox b/src/third_party/wiredtiger/src/docs/upgrading.dox index 3428e2781d9..5895650ca5e 100644 --- a/src/third_party/wiredtiger/src/docs/upgrading.dox +++ b/src/third_party/wiredtiger/src/docs/upgrading.dox @@ -4,6 +4,17 @@ @section version_311 Upgrading to Version 3.1.1 <dl> +<dt>Raw compression</dt> +<dd> +Support for "raw compression" has been removed in the 3.1.1 release. Only +applications configuring their own compressors will require modification, +those applications should remove their initialization of the \c WT_COMPRESSOR +structure's \c WT_COMPRESSOR::compress_raw field. Applications configuring +the "lz4" or "zlib" compressors (the existing WiredTiger compressors that +supported raw compression), will work without change, but applications may +see different compression ratios from previous releases. +</dd> + <dt>WiredTiger timestamps</dt> <dd> In previous releases of WiredTiger, it was possible to disable timestamp @@ -11,6 +22,7 @@ support as well as to configure a timestamp size different from the 64-bit default, using the <code>--with-timestamp-size=X</code> configuration option. That is no longer the case, in the 3.1.1 release, timestamps are always configured, and are always 64-bit unsigned integers. +</dd> <dt>WT_CURSOR::modify transaction requirements</dt> <dd> diff --git a/src/third_party/wiredtiger/src/evict/evict_lru.c b/src/third_party/wiredtiger/src/evict/evict_lru.c index 776c10d03e9..c7cff1bc54f 100644 --- a/src/third_party/wiredtiger/src/evict/evict_lru.c +++ b/src/third_party/wiredtiger/src/evict/evict_lru.c @@ -2444,6 +2444,7 @@ err: if (timer) { time_stop = __wt_clock(session); elapsed = WT_CLOCKDIFF_US(time_stop, time_start); WT_STAT_CONN_INCRV(session, application_cache_time, elapsed); + WT_STAT_SESSION_INCRV(session, cache_time, elapsed); session->cache_wait_us += elapsed; if (cache->cache_max_wait_us != 0 && session->cache_wait_us > cache->cache_max_wait_us) { diff --git a/src/third_party/wiredtiger/src/evict/evict_page.c b/src/third_party/wiredtiger/src/evict/evict_page.c index f5765cc4313..771ff32e328 100644 --- a/src/third_party/wiredtiger/src/evict/evict_page.c +++ b/src/third_party/wiredtiger/src/evict/evict_page.c @@ -38,7 +38,7 @@ __evict_exclusive(WT_SESSION_IMPL *session, WT_REF *ref) * Check for a hazard pointer indicating another thread is using the * page, meaning the page cannot be evicted. */ - if (__wt_hazard_check(session, ref) == NULL) + if (__wt_hazard_check(session, ref, NULL) == NULL) return (0); WT_STAT_DATA_INCR(session, cache_eviction_hazard); diff --git a/src/third_party/wiredtiger/src/include/cell.i b/src/third_party/wiredtiger/src/include/cell.i index 214d13b0206..cea27e1b26f 100644 --- a/src/third_party/wiredtiger/src/include/cell.i +++ b/src/third_party/wiredtiger/src/include/cell.i @@ -818,29 +818,3 @@ __wt_page_cell_data_ref(WT_SESSION_IMPL *session, { return (__cell_data_ref(session, page, page->type, unpack, store)); } - -/* - * __wt_cell_data_copy -- - * Copy the data from an unpacked cell into a buffer. - */ -static inline int -__wt_cell_data_copy(WT_SESSION_IMPL *session, - int page_type, WT_CELL_UNPACK *unpack, WT_ITEM *store) -{ - /* - * We have routines to both copy and reference a cell's information. In - * most cases, all we need is a reference and we prefer that, especially - * when returning key/value items. In a few we need a real copy: call - * the standard reference function and get a reference. In some cases, - * a copy will be made (for example, when reading an overflow item from - * the underlying object. If that happens, we're done, otherwise make - * a copy. - * - * We don't require two versions of this function, no callers need to - * handle WT_CELL_VALUE_OVFL_RM cells. - */ - WT_RET(__wt_dsk_cell_data_ref(session, page_type, unpack, store)); - if (!WT_DATA_IN_ITEM(store)) - WT_RET(__wt_buf_set(session, store, store->data, store->size)); - return (0); -} diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h index bb96b8d18b0..5fb0cee2b91 100644 --- a/src/third_party/wiredtiger/src/include/connection.h +++ b/src/third_party/wiredtiger/src/include/connection.h @@ -341,6 +341,7 @@ struct __wt_connection_impl { WT_LOG *log; /* Logging structure */ WT_COMPRESSOR *log_compressor;/* Logging compressor */ uint32_t log_cursors; /* Log cursor count */ + wt_off_t log_dirty_max; /* Log dirty system cache max size */ wt_off_t log_file_max; /* Log file max size */ const char *log_path; /* Logging path format */ uint32_t log_prealloc; /* Log file pre-allocation */ diff --git a/src/third_party/wiredtiger/src/include/cursor.h b/src/third_party/wiredtiger/src/include/cursor.h index 55a4cdb7a13..bac2a1d7a20 100644 --- a/src/third_party/wiredtiger/src/include/cursor.h +++ b/src/third_party/wiredtiger/src/include/cursor.h @@ -80,7 +80,8 @@ struct __wt_cursor_backup { size_t list_next; /* AUTOMATIC FLAG VALUE GENERATION START */ -#define WT_CURBACKUP_LOCKER 0x1u /* Hot-backup started */ +#define WT_CURBACKUP_DUP 0x1u /* Duplicated backup cursor */ +#define WT_CURBACKUP_LOCKER 0x2u /* Hot-backup started */ /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint8_t flags; }; diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index ca7f0597f88..50eaa16e847 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -298,7 +298,7 @@ extern int __wt_statlog_destroy(WT_SESSION_IMPL *session, bool is_close) WT_GCC_ extern int __wt_sweep_config(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_sweep_create(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_sweep_destroy(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_curbackup_open(WT_SESSION_IMPL *session, const char *uri, const char *cfg[], WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_curbackup_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *other, const char *cfg[], WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_backup_file_remove(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_curbulk_init(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk, bool bitmap, bool skip_sort_check) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_curconfig_open(WT_SESSION_IMPL *session, const char *uri, const char *cfg[], WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -717,7 +717,7 @@ __wt_hazard_set(WT_SESSION_IMPL *session, WT_REF *ref, bool *busyp ); extern int __wt_hazard_clear(WT_SESSION_IMPL *session, WT_REF *ref) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_hazard_close(WT_SESSION_IMPL *session); -extern WT_HAZARD *__wt_hazard_check(WT_SESSION_IMPL *session, WT_REF *ref); +extern WT_HAZARD *__wt_hazard_check(WT_SESSION_IMPL *session, WT_REF *ref, WT_SESSION_IMPL **sessionp); extern u_int __wt_hazard_count(WT_SESSION_IMPL *session, WT_REF *ref); extern bool __wt_hazard_check_assert(WT_SESSION_IMPL *session, void *ref, bool waitfor) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_fill_hex(const uint8_t *src, size_t src_max, uint8_t *dest, size_t dest_max, size_t *lenp); diff --git a/src/third_party/wiredtiger/src/include/log.h b/src/third_party/wiredtiger/src/include/log.h index b75f375dbee..389a33db8be 100644 --- a/src/third_party/wiredtiger/src/include/log.h +++ b/src/third_party/wiredtiger/src/include/log.h @@ -211,10 +211,11 @@ struct __wt_logslot { WT_ITEM slot_buf; /* Buffer for grouped writes */ /* AUTOMATIC FLAG VALUE GENERATION START */ -#define WT_SLOT_CLOSEFH 0x1u /* Close old fh on release */ -#define WT_SLOT_FLUSH 0x2u /* Wait for write */ -#define WT_SLOT_SYNC 0x4u /* Needs sync on release */ -#define WT_SLOT_SYNC_DIR 0x8u /* Directory sync on release */ +#define WT_SLOT_CLOSEFH 0x01u /* Close old fh on release */ +#define WT_SLOT_FLUSH 0x02u /* Wait for write */ +#define WT_SLOT_SYNC 0x04u /* Needs sync on release */ +#define WT_SLOT_SYNC_DIR 0x08u /* Directory sync on release */ +#define WT_SLOT_SYNC_DIRTY 0x10u /* Sync system buffers on release */ /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t flags; WT_CACHE_LINE_PAD_END @@ -222,6 +223,11 @@ struct __wt_logslot { #define WT_SLOT_INIT_FLAGS 0 +#define WT_SLOT_SYNC_FLAGS \ + (WT_SLOT_SYNC | \ + WT_SLOT_SYNC_DIR | \ + WT_SLOT_SYNC_DIRTY) + #define WT_WITH_SLOT_LOCK(session, log, op) do { \ WT_ASSERT(session, !F_ISSET(session, WT_SESSION_LOCKED_SLOT)); \ WT_WITH_LOCK_WAIT(session, \ @@ -267,6 +273,7 @@ struct __wt_log { WT_LSN alloc_lsn; /* Next LSN for allocation */ WT_LSN bg_sync_lsn; /* Latest background sync LSN */ WT_LSN ckpt_lsn; /* Last checkpoint LSN */ + WT_LSN dirty_lsn; /* LSN of last non-synced write */ WT_LSN first_lsn; /* First LSN */ WT_LSN sync_dir_lsn; /* LSN of the last directory sync */ WT_LSN sync_lsn; /* LSN of the last sync */ diff --git a/src/third_party/wiredtiger/src/include/mutex.h b/src/third_party/wiredtiger/src/include/mutex.h index ba32d166f03..703ee12afc1 100644 --- a/src/third_party/wiredtiger/src/include/mutex.h +++ b/src/third_party/wiredtiger/src/include/mutex.h @@ -53,6 +53,7 @@ struct __wt_rwlock { /* Read/write lock */ int16_t stat_write_count_off; /* write acquisitions offset */ int16_t stat_app_usecs_off; /* waiting application threads offset */ int16_t stat_int_usecs_off; /* waiting server threads offset */ + int16_t stat_session_usecs_off; /* waiting session offset */ WT_CONDVAR *cond_readers; /* Blocking readers */ WT_CONDVAR *cond_writers; /* Blocking writers */ @@ -65,16 +66,23 @@ struct __wt_rwlock { /* Read/write lock */ * Implemented as a macro so we can pass in a statistics field and convert * it into a statistics structure array offset. */ -#define WT_RWLOCK_INIT_TRACKED(session, l, name) do { \ - WT_RET(__wt_rwlock_init(session, l)); \ - (l)->stat_read_count_off = (int16_t)WT_STATS_FIELD_TO_OFFSET( \ - S2C(session)->stats, lock_##name##_read_count); \ - (l)->stat_write_count_off = (int16_t)WT_STATS_FIELD_TO_OFFSET( \ - S2C(session)->stats, lock_##name##_write_count); \ - (l)->stat_app_usecs_off = (int16_t)WT_STATS_FIELD_TO_OFFSET( \ - S2C(session)->stats, lock_##name##_wait_application); \ - (l)->stat_int_usecs_off = (int16_t)WT_STATS_FIELD_TO_OFFSET( \ - S2C(session)->stats, lock_##name##_wait_internal); \ +#define WT_RWLOCK_INIT_TRACKED(session, l, name) do { \ + WT_RET(__wt_rwlock_init(session, l)); \ + (l)->stat_read_count_off = (int16_t)WT_STATS_FIELD_TO_OFFSET( \ + S2C(session)->stats, lock_##name##_read_count); \ + (l)->stat_write_count_off = (int16_t)WT_STATS_FIELD_TO_OFFSET( \ + S2C(session)->stats, lock_##name##_write_count); \ + (l)->stat_app_usecs_off = (int16_t)WT_STATS_FIELD_TO_OFFSET( \ + S2C(session)->stats, lock_##name##_wait_application); \ + (l)->stat_int_usecs_off = (int16_t)WT_STATS_FIELD_TO_OFFSET( \ + S2C(session)->stats, lock_##name##_wait_internal); \ +} while (0) + +#define WT_RWLOCK_INIT_SESSION_TRACKED(session, l, name) do { \ + WT_RWLOCK_INIT_TRACKED(session, l, name); \ + (l)->stat_session_usecs_off = \ + (int16_t)WT_SESSION_STATS_FIELD_TO_OFFSET( \ + &(session)->stats, lock_##name##_wait); \ } while (0) /* @@ -112,6 +120,7 @@ struct __wt_spinlock { int16_t stat_count_off; /* acquisitions offset */ int16_t stat_app_usecs_off; /* waiting application threads offset */ int16_t stat_int_usecs_off; /* waiting server threads offset */ + int16_t stat_session_usecs_off; /* waiting session offset */ int8_t initialized; /* Lock initialized, for cleanup */ diff --git a/src/third_party/wiredtiger/src/include/mutex.i b/src/third_party/wiredtiger/src/include/mutex.i index 8a2699f031d..93c745bd5fc 100644 --- a/src/third_party/wiredtiger/src/include/mutex.i +++ b/src/third_party/wiredtiger/src/include/mutex.i @@ -286,6 +286,13 @@ __wt_spin_unlock(WT_SESSION_IMPL *session, WT_SPINLOCK *t) S2C(session)->stats, lock_##name##_wait_internal); \ } while (0) +#define WT_SPIN_INIT_SESSION_TRACKED(session, t, name) do { \ + WT_SPIN_INIT_TRACKED(session, t, name); \ + (t)->stat_session_usecs_off = \ + (int16_t)WT_SESSION_STATS_FIELD_TO_OFFSET( \ + &(session)->stats, lock_##name##_wait); \ +} while (0) + /* * __wt_spin_lock_track -- * Spinlock acquisition, with tracking. @@ -293,21 +300,25 @@ __wt_spin_unlock(WT_SESSION_IMPL *session, WT_SPINLOCK *t) static inline void __wt_spin_lock_track(WT_SESSION_IMPL *session, WT_SPINLOCK *t) { - uint64_t time_start, time_stop; - int64_t **stats; + uint64_t time_diff, time_start, time_stop; + int64_t *session_stats, **stats; if (t->stat_count_off != -1 && WT_STAT_ENABLED(session)) { time_start = __wt_clock(session); __wt_spin_lock(session, t); time_stop = __wt_clock(session); + time_diff = WT_CLOCKDIFF_US(time_stop, time_start); stats = (int64_t **)S2C(session)->stats; + session_stats = (int64_t *)&(session->stats); stats[session->stat_bucket][t->stat_count_off]++; if (F_ISSET(session, WT_SESSION_INTERNAL)) stats[session->stat_bucket][t->stat_int_usecs_off] += - (int64_t)WT_CLOCKDIFF_US(time_stop, time_start); - else + (int64_t)time_diff; + else { stats[session->stat_bucket][t->stat_app_usecs_off] += - (int64_t)WT_CLOCKDIFF_US(time_stop, time_start); + (int64_t)time_diff; + } + session_stats[t->stat_session_usecs_off] += (int64_t)time_diff; } else __wt_spin_lock(session, t); } diff --git a/src/third_party/wiredtiger/src/include/session.h b/src/third_party/wiredtiger/src/include/session.h index 092f2259edd..294ca1503a5 100644 --- a/src/third_party/wiredtiger/src/include/session.h +++ b/src/third_party/wiredtiger/src/include/session.h @@ -163,30 +163,32 @@ struct __wt_session_impl { u_int stat_bucket; /* Statistics bucket offset */ /* AUTOMATIC FLAG VALUE GENERATION START */ -#define WT_SESSION_CACHE_CURSORS 0x000001u -#define WT_SESSION_CAN_WAIT 0x000002u -#define WT_SESSION_IGNORE_CACHE_SIZE 0x000004u -#define WT_SESSION_INTERNAL 0x000008u -#define WT_SESSION_LOCKED_CHECKPOINT 0x000010u -#define WT_SESSION_LOCKED_HANDLE_LIST_READ 0x000020u -#define WT_SESSION_LOCKED_HANDLE_LIST_WRITE 0x000040u -#define WT_SESSION_LOCKED_METADATA 0x000080u -#define WT_SESSION_LOCKED_PASS 0x000100u -#define WT_SESSION_LOCKED_SCHEMA 0x000200u -#define WT_SESSION_LOCKED_SLOT 0x000400u -#define WT_SESSION_LOCKED_TABLE_READ 0x000800u -#define WT_SESSION_LOCKED_TABLE_WRITE 0x001000u -#define WT_SESSION_LOCKED_TURTLE 0x002000u -#define WT_SESSION_LOGGING_INMEM 0x004000u -#define WT_SESSION_LOOKASIDE_CURSOR 0x008000u -#define WT_SESSION_NO_DATA_HANDLES 0x010000u -#define WT_SESSION_NO_LOGGING 0x020000u -#define WT_SESSION_NO_RECONCILE 0x040000u -#define WT_SESSION_NO_SCHEMA_LOCK 0x080000u -#define WT_SESSION_QUIET_CORRUPT_FILE 0x100000u -#define WT_SESSION_READ_WONT_NEED 0x200000u -#define WT_SESSION_SCHEMA_TXN 0x400000u -#define WT_SESSION_SERVER_ASYNC 0x800000u +#define WT_SESSION_BACKUP_CURSOR 0x0000001u +#define WT_SESSION_BACKUP_DUP 0x0000002u +#define WT_SESSION_CACHE_CURSORS 0x0000004u +#define WT_SESSION_CAN_WAIT 0x0000008u +#define WT_SESSION_IGNORE_CACHE_SIZE 0x0000010u +#define WT_SESSION_INTERNAL 0x0000020u +#define WT_SESSION_LOCKED_CHECKPOINT 0x0000040u +#define WT_SESSION_LOCKED_HANDLE_LIST_READ 0x0000080u +#define WT_SESSION_LOCKED_HANDLE_LIST_WRITE 0x0000100u +#define WT_SESSION_LOCKED_METADATA 0x0000200u +#define WT_SESSION_LOCKED_PASS 0x0000400u +#define WT_SESSION_LOCKED_SCHEMA 0x0000800u +#define WT_SESSION_LOCKED_SLOT 0x0001000u +#define WT_SESSION_LOCKED_TABLE_READ 0x0002000u +#define WT_SESSION_LOCKED_TABLE_WRITE 0x0004000u +#define WT_SESSION_LOCKED_TURTLE 0x0008000u +#define WT_SESSION_LOGGING_INMEM 0x0010000u +#define WT_SESSION_LOOKASIDE_CURSOR 0x0020000u +#define WT_SESSION_NO_DATA_HANDLES 0x0040000u +#define WT_SESSION_NO_LOGGING 0x0080000u +#define WT_SESSION_NO_RECONCILE 0x0100000u +#define WT_SESSION_NO_SCHEMA_LOCK 0x0200000u +#define WT_SESSION_QUIET_CORRUPT_FILE 0x0400000u +#define WT_SESSION_READ_WONT_NEED 0x0800000u +#define WT_SESSION_SCHEMA_TXN 0x1000000u +#define WT_SESSION_SERVER_ASYNC 0x2000000u /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t flags; diff --git a/src/third_party/wiredtiger/src/include/stat.h b/src/third_party/wiredtiger/src/include/stat.h index 3714d03d1ce..35bd30419da 100644 --- a/src/third_party/wiredtiger/src/include/stat.h +++ b/src/third_party/wiredtiger/src/include/stat.h @@ -84,6 +84,9 @@ #define WT_STATS_FIELD_TO_OFFSET(stats, fld) \ (int)(&(stats)[0]->fld - (int64_t *)(stats)[0]) +#define WT_SESSION_STATS_FIELD_TO_OFFSET(stats, fld) \ + (int)(&(stats)->fld - (int64_t *)(stats)) + /* AUTOMATIC FLAG VALUE GENERATION START */ #define WT_STAT_CLEAR 0x01u #define WT_STAT_JSON 0x02u @@ -163,7 +166,8 @@ __wt_stats_clear(void *stats_arg, int slot) } while (0) #define WT_STAT_DECRV_ATOMIC_BASE(session, stat, fld, value) do { \ if (WT_STAT_ENABLED(session)) \ - __wt_atomic_subi64(&(stat)->fld, (int64_t)(value)); \ + (void) \ + __wt_atomic_subi64(&(stat)->fld, (int64_t)(value)); \ } while (0) #define WT_STAT_INCRV_BASE(session, stat, fld, value) do { \ if (WT_STAT_ENABLED(session)) \ @@ -171,7 +175,8 @@ __wt_stats_clear(void *stats_arg, int slot) } while (0) #define WT_STAT_INCRV_ATOMIC_BASE(session, stat, fld, value) do { \ if (WT_STAT_ENABLED(session)) \ - __wt_atomic_addi64(&(stat)->fld, (int64_t)(value)); \ + (void) \ + __wt_atomic_addi64(&(stat)->fld, (int64_t)(value)); \ } while (0) #define WT_STAT_DECRV(session, stats, fld, value) do { \ @@ -259,6 +264,12 @@ __wt_stats_clear(void *stats_arg, int slot) } while (0) /* + * Update per session statistics. + */ +#define WT_STAT_SESSION_INCRV(session, fld, value) \ + WT_STAT_INCRV_BASE(session, &(session)->stats, fld, value) + +/* * Construct histogram increment functions to put the passed value into the * right bucket. Bucket ranges, represented by various statistics, depend upon * whether the passed value is in milliseconds or microseconds. Also values @@ -455,6 +466,7 @@ struct __wt_connection_stats { int64_t fsync_io; int64_t read_io; int64_t write_io; + int64_t cursor_cached_count; int64_t cursor_cache; int64_t cursor_create; int64_t cursor_insert; @@ -472,8 +484,8 @@ struct __wt_connection_stats { int64_t cursor_sweep_examined; int64_t cursor_sweep; int64_t cursor_update; - int64_t cursors_cached; int64_t cursor_reopen; + int64_t cursor_open_count; int64_t cursor_truncate; int64_t dh_conn_handle_count; int64_t dh_sweep_ref; @@ -586,7 +598,6 @@ struct __wt_connection_stats { int64_t rec_page_delete; int64_t rec_split_stashed_bytes; int64_t rec_split_stashed_objects; - int64_t session_cursor_open; int64_t session_open; int64_t session_query_ts; int64_t session_table_alter_fail; @@ -791,9 +802,6 @@ struct __wt_dsrc_stats { int64_t compress_write; int64_t compress_write_fail; int64_t compress_write_too_small; - int64_t compress_raw_fail_temporary; - int64_t compress_raw_fail; - int64_t compress_raw_ok; int64_t cursor_insert_bulk; int64_t cursor_cache; int64_t cursor_create; @@ -805,6 +813,7 @@ struct __wt_dsrc_stats { int64_t cursor_insert; int64_t cursor_modify; int64_t cursor_next; + int64_t cursor_open_count; int64_t cursor_prev; int64_t cursor_remove; int64_t cursor_reserve; @@ -827,9 +836,7 @@ struct __wt_dsrc_stats { int64_t rec_pages; int64_t rec_pages_eviction; int64_t rec_page_delete; - int64_t session_cursors_cached; int64_t session_compact; - int64_t session_cursor_open; int64_t txn_update_conflict; }; diff --git a/src/third_party/wiredtiger/src/include/txn.i b/src/third_party/wiredtiger/src/include/txn.i index 7de59e498d4..a297db2cf9e 100644 --- a/src/third_party/wiredtiger/src/include/txn.i +++ b/src/third_party/wiredtiger/src/include/txn.i @@ -343,51 +343,56 @@ __wt_txn_unmodify(WT_SESSION_IMPL *session) } /* - * __wt_txn_op_commit_page_del -- - * Make the transaction ID and timestamp updates necessary to a ref that - * was created by a fast delete truncate operation. + * __wt_txn_op_apply_prepare_state -- + * Apply the correct prepare state and the timestamp to the ref and to any + * updates in the page del update list. */ static inline void -__wt_txn_op_commit_page_del(WT_SESSION_IMPL *session, WT_REF *ref) +__wt_txn_op_apply_prepare_state( + WT_SESSION_IMPL *session, WT_REF *ref, bool commit) { WT_TXN *txn; WT_UPDATE **updp; + wt_timestamp_t ts; uint32_t previous_state; + uint8_t prepare_state; txn = &session->txn; - /* Avoid locking the page if a previous eviction already cleaned up. */ - if (ref->page_del->update_list == NULL) - return; - /* - * Lock the ref to ensure we don't race with eviction freeing the - * page deleted update list. + * Lock the ref to ensure we don't race with eviction freeing the page + * deleted update list or with a page instantiate. */ for (;; __wt_yield()) { previous_state = ref->state; + WT_ASSERT(session, previous_state != WT_REF_READING); if (previous_state != WT_REF_LOCKED && __wt_atomic_casv32( &ref->state, previous_state, WT_REF_LOCKED)) break; } + if (commit) { + ts = txn->commit_timestamp; + prepare_state = WT_PREPARE_RESOLVED; + } else { + ts = txn->prepare_timestamp; + prepare_state = WT_PREPARE_INPROGRESS; + } for (updp = ref->page_del->update_list; updp != NULL && *updp != NULL; ++updp) { - (*updp)->timestamp = txn->commit_timestamp; - if (F_ISSET(txn, WT_TXN_PREPARE)) - /* - * Holding the ref locked means we have exclusive - * access, so don't need to use the prepare locked - * transition state. - */ - (*updp)->prepare_state = WT_PREPARE_RESOLVED; + (*updp)->timestamp = ts; + /* + * Holding the ref locked means we have exclusive access, so if + * we are committing we don't need to use the prepare locked + * transition state. + */ + (*updp)->prepare_state = prepare_state; } + ref->page_del->timestamp = ts; + WT_PUBLISH(ref->page_del->prepare_state, prepare_state); - /* - * Publish to ensure we don't let the page be evicted and the updates - * discarded before being written. - */ + /* Unlock the page by setting it back to it's previous state */ WT_REF_SET_STATE(ref, previous_state); } @@ -416,8 +421,13 @@ __wt_txn_op_set_timestamp(WT_SESSION_IMPL *session, WT_TXN_OP *op) return; if (F_ISSET(txn, WT_TXN_PREPARE)) { + /* + * We have a commit timestamp for a prepare transaction, this is + * only possible as part of a transaction commit call. + */ if (op->type == WT_TXN_OP_REF_DELETE) - __wt_txn_op_commit_page_del(session, op->u.ref); + __wt_txn_op_apply_prepare_state( + session, op->u.ref, true); else { upd = op->u.op_upd; diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index f04ad40693e..33d0b1ed74d 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -361,7 +361,8 @@ struct __wt_cursor { * with WT_CURSOR::prev will move to the last record. * * In the case of a statistics cursor, resetting the cursor refreshes - * the statistics information returned. + * the statistics information returned. Resetting a session statistics + * cursor resets all the session statistics values to zero. * * @snippet ex_all.c Reset the cursor * @@ -1168,6 +1169,15 @@ struct __wt_session { * @config{ enabled, if false\, this object has * checkpoint-level durability., a boolean flag; default \c true.} * @config{ ),,} + * @config{os_cache_dirty_max, maximum dirty system buffer cache usage\, + * in bytes. If non-zero\, schedule writes for dirty blocks belonging + * to this object in the system buffer cache after that many bytes from + * this object are written into the buffer cache., an integer greater + * than or equal to 0; default \c 0.} + * @config{os_cache_max, maximum system buffer cache usage\, in bytes. + * If non-zero\, evict object blocks from the system buffer cache after + * that many bytes from this object are read or written into the buffer + * cache., an integer greater than or equal to 0; default \c 0.} * @configend * @errors */ @@ -2281,6 +2291,12 @@ struct __wt_connection { * configuration options defined below.} * @config{ archive, automatically archive * unneeded log files., a boolean flag; default \c true.} + * @config{ os_cache_dirty_pct, maximum dirty + * system buffer cache usage\, as a percentage of the log's \c file_max. + * If non-zero\, schedule writes for dirty blocks belonging to the log + * in the system buffer cache after that percentage of the log has been + * written into the buffer cache without an intervening file sync., an + * integer between 0 and 100; default \c 0.} * @config{ prealloc, pre-allocate log files., a * boolean flag; default \c true.} * @config{ zero_fill, manually write zeroes into @@ -2938,9 +2954,15 @@ struct __wt_connection { * logging subsystem., a boolean flag; default \c false.} * @config{ file_max, the maximum size of log files., an * integer between 100KB and 2GB; default \c 100MB.} - * @config{ path, the name of a directory into which log - * files are written. The directory must already exist. If the value is not an - * absolute path\, the path is relative to the database home (see @ref + * @config{ os_cache_dirty_pct, maximum dirty system + * buffer cache usage\, as a percentage of the log's \c file_max. If non-zero\, + * schedule writes for dirty blocks belonging to the log in the system buffer + * cache after that percentage of the log has been written into the buffer cache + * without an intervening file sync., an integer between 0 and 100; default \c + * 0.} + * @config{ path, the name of a directory into which + * log files are written. The directory must already exist. If the value is + * not an absolute path\, the path is relative to the database home (see @ref * absolute_path for more information)., a string; default \c ".".} * @config{ prealloc, pre-allocate log files., a boolean * flag; default \c true.} @@ -3612,10 +3634,43 @@ struct __wt_config_parser { uint32_t (*wiredtiger_crc32c_func(void))(const void *, size_t) WT_ATTRIBUTE_LIBRARY_VISIBLE; -/*! @} */ #endif /* !defined(SWIG) */ /*! + * Calculate a set of WT_MODIFY operations to represent an update. + * This call will calculate a set of modifications to an old value that produce + * the new value. If more modifications are required than fit in the array + * passed in by the caller, or if more bytes have changed than the \c maxdiff + * parameter, the call will fail. The matching algorithm is approximate, so it + * may fail and return WT_NOTFOUND if a matching set of WT_MODIFY operations + * is not found. + * + * The \c maxdiff parameter bounds how much work will be done searching for a + * match: to ensure a match is found, it may need to be set larger than actual + * number of bytes that differ between the old and new values. In particular, + * repeated patterns of bytes in the values can lead to suboptimal matching, + * and matching ranges less than 64 bytes long will not be detected. + * + * If the call succeeds, the WT_MODIFY operations will point into \c newv, + * which must remain valid until WT_CURSOR::modify is called. + * + * @snippet ex_all.c Calculate a modify operation + * + * @param session the current WiredTiger session (may be NULL) + * @param oldv old value + * @param newv new value + * @param maxdiff maximum bytes difference + * @param[out] entries array of modifications producing the new value + * @param[in,out] nentriesp size of \c entries array passed in, + * set to the number of entries used + * @errors + */ +int wiredtiger_calc_modify(WT_SESSION *session, + const WT_ITEM *oldv, const WT_ITEM *newv, + size_t maxdiff, WT_MODIFY *entries, int *nentriesp) + WT_ATTRIBUTE_LIBRARY_VISIBLE; + +/*! * Get version information. * * @snippet ex_all.c Get the WiredTiger library version #1 @@ -3629,6 +3684,8 @@ uint32_t (*wiredtiger_crc32c_func(void))(const void *, size_t) const char *wiredtiger_version(int *majorp, int *minorp, int *patchp) WT_ATTRIBUTE_LIBRARY_VISIBLE; +/*! @} */ + /******************************************* * Error returns *******************************************/ @@ -3851,121 +3908,6 @@ struct __wt_compressor { size_t *result_lenp, int *compression_failed); /*! - * Callback to compress a list of byte strings. - * - * WT_COMPRESSOR::compress_raw gives applications fine-grained control - * over disk block size when writing row-store or variable-length - * column-store pages. Where this level of control is not required by - * the underlying storage device, set the WT_COMPRESSOR::compress_raw - * callback to \c NULL and WiredTiger will internally split each page - * into blocks, each block then compressed by WT_COMPRESSOR::compress. - * - * WT_COMPRESSOR::compress_raw takes a source buffer and an array of - * 0-based offsets of byte strings in that buffer. The callback then - * encodes none, some or all of the byte strings and copies the encoded - * representation into a destination buffer. The callback returns the - * number of byte strings encoded and the bytes needed for the encoded - * representation. The encoded representation has header information - * prepended and is written as a block to the underlying file object. - * - * On entry, \c page_max is the configured maximum size for objects of - * this type. (This value is provided for convenience, and will be - * either the \c internal_page_max or \c leaf_page_max value specified - * to WT_SESSION::create when the object was created.) - * - * On entry, \c split_pct is the configured Btree page split size for - * this object. (This value is provided for convenience, and will be - * the \c split_pct value specified to WT_SESSION::create when the - * object was created.) - * - * On entry, \c extra is a count of additional bytes that will be added - * to the encoded representation before it is written. In other words, - * if the target write size is 8KB, the returned encoded representation - * should be less than or equal to (8KB - \c extra). The method does - * not need to skip bytes in the destination buffer based on \c extra, - * the method should only use \c extra to decide how many bytes to store - * into the destination buffer for its ideal block size. - * - * On entry, \c src points to the source buffer; \c offsets is an array - * of \c slots 0-based offsets into \c src, where each offset is the - * start of a byte string, except for the last offset, which is the - * offset of the first byte past the end of the last byte string. (In - * other words, <code>offsets[0]</code> will be 0, the offset of the - * first byte of the first byte string in \c src, and - * <code>offsets[slots]</code> is the total length of all of the byte - * strings in the \c src buffer.) - * - * On entry, \c dst points to the destination buffer with a length - * of \c dst_len. If the WT_COMPRESSOR::pre_size method is specified, - * the destination buffer will be at least the size returned by that - * method; otherwise, the destination buffer will be at least as large - * as the length of the data to compress. - * - * After successful completion, the callback should return \c 0, and - * set \c result_slotsp to the number of byte strings encoded and - * \c result_lenp to the bytes needed for the encoded representation. - * - * There is no requirement the callback encode any or all of the byte - * strings passed by WiredTiger. If the callback does not encode any - * of the byte strings and compression should not be retried, the - * callback should set \c result_slotsp to 0. - * - * If the callback does not encode any of the byte strings and - * compression should be retried with additional byte strings, the - * callback must return \c EAGAIN. In that case, WiredTiger will - * accumulate more rows and repeat the call. - * - * If there are no more rows to accumulate or the callback indicates - * that it cannot be retried, WiredTiger writes the remaining rows - * using \c WT_COMPRESSOR::compress. - * - * On entry, \c final is zero if there are more rows to be written as - * part of this page (if there will be additional data provided to the - * callback), and non-zero if there are no more rows to be written as - * part of this page. If \c final is set and the callback fails to - * encode any rows, WiredTiger writes the remaining rows without further - * calls to the callback. If \c final is set and the callback encodes - * any number of rows, WiredTiger continues to call the callback until - * all of the rows are encoded or the callback fails to encode any rows. - * - * The WT_COMPRESSOR::compress_raw callback is intended for applications - * wanting to create disk blocks in specific sizes. - * WT_COMPRESSOR::compress_raw is not a replacement for - * WT_COMPRESSOR::compress: objects which WT_COMPRESSOR::compress_raw - * cannot handle (for example, overflow key or value items), or which - * WT_COMPRESSOR::compress_raw chooses not to compress for any reason - * (for example, if WT_COMPRESSOR::compress_raw callback chooses not to - * compress a small number of rows, but the page being written has no - * more rows to accumulate), will be passed to WT_COMPRESSOR::compress. - * - * The WT_COMPRESSOR::compress_raw callback is only called for objects - * where it is applicable, that is, for row-store and variable-length - * column-store objects, where both row-store key prefix compression - * and row-store and variable-length column-store dictionary compression - * are \b not configured. When WT_COMPRESSOR::compress_raw is not - * applicable, the WT_COMPRESSOR::compress callback is used instead. - * - * @param[in] page_max the configured maximum page size for this object - * @param[in] split_pct the configured page split size for this object - * @param[in] extra the count of the additional bytes - * @param[in] src the data to compress - * @param[in] offsets the byte offsets of the byte strings in src - * @param[in] slots the number of entries in offsets - * @param[in] dst the destination buffer - * @param[in] dst_len the length of the destination buffer - * @param[in] final non-zero if there are no more rows to accumulate - * @param[out] result_lenp the length of the compressed data - * @param[out] result_slotsp the number of byte offsets taken - * @returns zero for success, non-zero to indicate an error. - */ - int (*compress_raw)(WT_COMPRESSOR *compressor, WT_SESSION *session, - size_t page_max, int split_pct, size_t extra, - uint8_t *src, uint32_t *offsets, uint32_t slots, - uint8_t *dst, size_t dst_len, - int final, - size_t *result_lenp, uint32_t *result_slotsp); - - /*! * Callback to decompress a chunk of data. * * WT_COMPRESSOR::decompress takes a source buffer and a destination @@ -5291,282 +5233,282 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_CONN_READ_IO 1140 /*! connection: total write I/Os */ #define WT_STAT_CONN_WRITE_IO 1141 +/*! cursor: cached cursor count */ +#define WT_STAT_CONN_CURSOR_CACHED_COUNT 1142 /*! cursor: cursor close calls that result in cache */ -#define WT_STAT_CONN_CURSOR_CACHE 1142 +#define WT_STAT_CONN_CURSOR_CACHE 1143 /*! cursor: cursor create calls */ -#define WT_STAT_CONN_CURSOR_CREATE 1143 +#define WT_STAT_CONN_CURSOR_CREATE 1144 /*! cursor: cursor insert calls */ -#define WT_STAT_CONN_CURSOR_INSERT 1144 +#define WT_STAT_CONN_CURSOR_INSERT 1145 /*! cursor: cursor modify calls */ -#define WT_STAT_CONN_CURSOR_MODIFY 1145 +#define WT_STAT_CONN_CURSOR_MODIFY 1146 /*! cursor: cursor next calls */ -#define WT_STAT_CONN_CURSOR_NEXT 1146 +#define WT_STAT_CONN_CURSOR_NEXT 1147 /*! cursor: cursor operation restarted */ -#define WT_STAT_CONN_CURSOR_RESTART 1147 +#define WT_STAT_CONN_CURSOR_RESTART 1148 /*! cursor: cursor prev calls */ -#define WT_STAT_CONN_CURSOR_PREV 1148 +#define WT_STAT_CONN_CURSOR_PREV 1149 /*! cursor: cursor remove calls */ -#define WT_STAT_CONN_CURSOR_REMOVE 1149 +#define WT_STAT_CONN_CURSOR_REMOVE 1150 /*! cursor: cursor reserve calls */ -#define WT_STAT_CONN_CURSOR_RESERVE 1150 +#define WT_STAT_CONN_CURSOR_RESERVE 1151 /*! cursor: cursor reset calls */ -#define WT_STAT_CONN_CURSOR_RESET 1151 +#define WT_STAT_CONN_CURSOR_RESET 1152 /*! cursor: cursor search calls */ -#define WT_STAT_CONN_CURSOR_SEARCH 1152 +#define WT_STAT_CONN_CURSOR_SEARCH 1153 /*! cursor: cursor search near calls */ -#define WT_STAT_CONN_CURSOR_SEARCH_NEAR 1153 +#define WT_STAT_CONN_CURSOR_SEARCH_NEAR 1154 /*! cursor: cursor sweep buckets */ -#define WT_STAT_CONN_CURSOR_SWEEP_BUCKETS 1154 +#define WT_STAT_CONN_CURSOR_SWEEP_BUCKETS 1155 /*! cursor: cursor sweep cursors closed */ -#define WT_STAT_CONN_CURSOR_SWEEP_CLOSED 1155 +#define WT_STAT_CONN_CURSOR_SWEEP_CLOSED 1156 /*! cursor: cursor sweep cursors examined */ -#define WT_STAT_CONN_CURSOR_SWEEP_EXAMINED 1156 +#define WT_STAT_CONN_CURSOR_SWEEP_EXAMINED 1157 /*! cursor: cursor sweeps */ -#define WT_STAT_CONN_CURSOR_SWEEP 1157 +#define WT_STAT_CONN_CURSOR_SWEEP 1158 /*! cursor: cursor update calls */ -#define WT_STAT_CONN_CURSOR_UPDATE 1158 -/*! cursor: cursors currently cached */ -#define WT_STAT_CONN_CURSORS_CACHED 1159 +#define WT_STAT_CONN_CURSOR_UPDATE 1159 /*! cursor: cursors reused from cache */ #define WT_STAT_CONN_CURSOR_REOPEN 1160 +/*! cursor: open cursor count */ +#define WT_STAT_CONN_CURSOR_OPEN_COUNT 1161 /*! cursor: truncate calls */ -#define WT_STAT_CONN_CURSOR_TRUNCATE 1161 +#define WT_STAT_CONN_CURSOR_TRUNCATE 1162 /*! data-handle: connection data handles currently active */ -#define WT_STAT_CONN_DH_CONN_HANDLE_COUNT 1162 +#define WT_STAT_CONN_DH_CONN_HANDLE_COUNT 1163 /*! data-handle: connection sweep candidate became referenced */ -#define WT_STAT_CONN_DH_SWEEP_REF 1163 +#define WT_STAT_CONN_DH_SWEEP_REF 1164 /*! data-handle: connection sweep dhandles closed */ -#define WT_STAT_CONN_DH_SWEEP_CLOSE 1164 +#define WT_STAT_CONN_DH_SWEEP_CLOSE 1165 /*! data-handle: connection sweep dhandles removed from hash list */ -#define WT_STAT_CONN_DH_SWEEP_REMOVE 1165 +#define WT_STAT_CONN_DH_SWEEP_REMOVE 1166 /*! data-handle: connection sweep time-of-death sets */ -#define WT_STAT_CONN_DH_SWEEP_TOD 1166 +#define WT_STAT_CONN_DH_SWEEP_TOD 1167 /*! data-handle: connection sweeps */ -#define WT_STAT_CONN_DH_SWEEPS 1167 +#define WT_STAT_CONN_DH_SWEEPS 1168 /*! data-handle: session dhandles swept */ -#define WT_STAT_CONN_DH_SESSION_HANDLES 1168 +#define WT_STAT_CONN_DH_SESSION_HANDLES 1169 /*! data-handle: session sweep attempts */ -#define WT_STAT_CONN_DH_SESSION_SWEEPS 1169 +#define WT_STAT_CONN_DH_SESSION_SWEEPS 1170 /*! lock: checkpoint lock acquisitions */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_COUNT 1170 +#define WT_STAT_CONN_LOCK_CHECKPOINT_COUNT 1171 /*! lock: checkpoint lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_APPLICATION 1171 +#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_APPLICATION 1172 /*! lock: checkpoint lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_INTERNAL 1172 +#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_INTERNAL 1173 /*! * lock: commit timestamp queue lock application thread time waiting * (usecs) */ -#define WT_STAT_CONN_LOCK_COMMIT_TIMESTAMP_WAIT_APPLICATION 1173 +#define WT_STAT_CONN_LOCK_COMMIT_TIMESTAMP_WAIT_APPLICATION 1174 /*! lock: commit timestamp queue lock internal thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_COMMIT_TIMESTAMP_WAIT_INTERNAL 1174 +#define WT_STAT_CONN_LOCK_COMMIT_TIMESTAMP_WAIT_INTERNAL 1175 /*! lock: commit timestamp queue read lock acquisitions */ -#define WT_STAT_CONN_LOCK_COMMIT_TIMESTAMP_READ_COUNT 1175 +#define WT_STAT_CONN_LOCK_COMMIT_TIMESTAMP_READ_COUNT 1176 /*! lock: commit timestamp queue write lock acquisitions */ -#define WT_STAT_CONN_LOCK_COMMIT_TIMESTAMP_WRITE_COUNT 1176 +#define WT_STAT_CONN_LOCK_COMMIT_TIMESTAMP_WRITE_COUNT 1177 /*! lock: dhandle lock application thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_APPLICATION 1177 +#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_APPLICATION 1178 /*! lock: dhandle lock internal thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_INTERNAL 1178 +#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_INTERNAL 1179 /*! lock: dhandle read lock acquisitions */ -#define WT_STAT_CONN_LOCK_DHANDLE_READ_COUNT 1179 +#define WT_STAT_CONN_LOCK_DHANDLE_READ_COUNT 1180 /*! lock: dhandle write lock acquisitions */ -#define WT_STAT_CONN_LOCK_DHANDLE_WRITE_COUNT 1180 +#define WT_STAT_CONN_LOCK_DHANDLE_WRITE_COUNT 1181 /*! lock: metadata lock acquisitions */ -#define WT_STAT_CONN_LOCK_METADATA_COUNT 1181 +#define WT_STAT_CONN_LOCK_METADATA_COUNT 1182 /*! lock: metadata lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_METADATA_WAIT_APPLICATION 1182 +#define WT_STAT_CONN_LOCK_METADATA_WAIT_APPLICATION 1183 /*! lock: metadata lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_METADATA_WAIT_INTERNAL 1183 +#define WT_STAT_CONN_LOCK_METADATA_WAIT_INTERNAL 1184 /*! * lock: read timestamp queue lock application thread time waiting * (usecs) */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_APPLICATION 1184 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_APPLICATION 1185 /*! lock: read timestamp queue lock internal thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_INTERNAL 1185 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_INTERNAL 1186 /*! lock: read timestamp queue read lock acquisitions */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_READ_COUNT 1186 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_READ_COUNT 1187 /*! lock: read timestamp queue write lock acquisitions */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WRITE_COUNT 1187 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WRITE_COUNT 1188 /*! lock: schema lock acquisitions */ -#define WT_STAT_CONN_LOCK_SCHEMA_COUNT 1188 +#define WT_STAT_CONN_LOCK_SCHEMA_COUNT 1189 /*! lock: schema lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_APPLICATION 1189 +#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_APPLICATION 1190 /*! lock: schema lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_INTERNAL 1190 +#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_INTERNAL 1191 /*! * lock: table lock application thread time waiting for the table lock * (usecs) */ -#define WT_STAT_CONN_LOCK_TABLE_WAIT_APPLICATION 1191 +#define WT_STAT_CONN_LOCK_TABLE_WAIT_APPLICATION 1192 /*! * lock: table lock internal thread time waiting for the table lock * (usecs) */ -#define WT_STAT_CONN_LOCK_TABLE_WAIT_INTERNAL 1192 +#define WT_STAT_CONN_LOCK_TABLE_WAIT_INTERNAL 1193 /*! lock: table read lock acquisitions */ -#define WT_STAT_CONN_LOCK_TABLE_READ_COUNT 1193 +#define WT_STAT_CONN_LOCK_TABLE_READ_COUNT 1194 /*! lock: table write lock acquisitions */ -#define WT_STAT_CONN_LOCK_TABLE_WRITE_COUNT 1194 +#define WT_STAT_CONN_LOCK_TABLE_WRITE_COUNT 1195 /*! lock: txn global lock application thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_APPLICATION 1195 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_APPLICATION 1196 /*! lock: txn global lock internal thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_INTERNAL 1196 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_INTERNAL 1197 /*! lock: txn global read lock acquisitions */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_READ_COUNT 1197 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_READ_COUNT 1198 /*! lock: txn global write lock acquisitions */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WRITE_COUNT 1198 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WRITE_COUNT 1199 /*! log: busy returns attempting to switch slots */ -#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1199 +#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1200 /*! log: force archive time sleeping (usecs) */ -#define WT_STAT_CONN_LOG_FORCE_ARCHIVE_SLEEP 1200 +#define WT_STAT_CONN_LOG_FORCE_ARCHIVE_SLEEP 1201 /*! log: log bytes of payload data */ -#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1201 +#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1202 /*! log: log bytes written */ -#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1202 +#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1203 /*! log: log files manually zero-filled */ -#define WT_STAT_CONN_LOG_ZERO_FILLS 1203 +#define WT_STAT_CONN_LOG_ZERO_FILLS 1204 /*! log: log flush operations */ -#define WT_STAT_CONN_LOG_FLUSH 1204 +#define WT_STAT_CONN_LOG_FLUSH 1205 /*! log: log force write operations */ -#define WT_STAT_CONN_LOG_FORCE_WRITE 1205 +#define WT_STAT_CONN_LOG_FORCE_WRITE 1206 /*! log: log force write operations skipped */ -#define WT_STAT_CONN_LOG_FORCE_WRITE_SKIP 1206 +#define WT_STAT_CONN_LOG_FORCE_WRITE_SKIP 1207 /*! log: log records compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1207 +#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1208 /*! log: log records not compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1208 +#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1209 /*! log: log records too small to compress */ -#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1209 +#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1210 /*! log: log release advances write LSN */ -#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1210 +#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1211 /*! log: log scan operations */ -#define WT_STAT_CONN_LOG_SCANS 1211 +#define WT_STAT_CONN_LOG_SCANS 1212 /*! log: log scan records requiring two reads */ -#define WT_STAT_CONN_LOG_SCAN_REREADS 1212 +#define WT_STAT_CONN_LOG_SCAN_REREADS 1213 /*! log: log server thread advances write LSN */ -#define WT_STAT_CONN_LOG_WRITE_LSN 1213 +#define WT_STAT_CONN_LOG_WRITE_LSN 1214 /*! log: log server thread write LSN walk skipped */ -#define WT_STAT_CONN_LOG_WRITE_LSN_SKIP 1214 +#define WT_STAT_CONN_LOG_WRITE_LSN_SKIP 1215 /*! log: log sync operations */ -#define WT_STAT_CONN_LOG_SYNC 1215 +#define WT_STAT_CONN_LOG_SYNC 1216 /*! log: log sync time duration (usecs) */ -#define WT_STAT_CONN_LOG_SYNC_DURATION 1216 +#define WT_STAT_CONN_LOG_SYNC_DURATION 1217 /*! log: log sync_dir operations */ -#define WT_STAT_CONN_LOG_SYNC_DIR 1217 +#define WT_STAT_CONN_LOG_SYNC_DIR 1218 /*! log: log sync_dir time duration (usecs) */ -#define WT_STAT_CONN_LOG_SYNC_DIR_DURATION 1218 +#define WT_STAT_CONN_LOG_SYNC_DIR_DURATION 1219 /*! log: log write operations */ -#define WT_STAT_CONN_LOG_WRITES 1219 +#define WT_STAT_CONN_LOG_WRITES 1220 /*! log: logging bytes consolidated */ -#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1220 +#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1221 /*! log: maximum log file size */ -#define WT_STAT_CONN_LOG_MAX_FILESIZE 1221 +#define WT_STAT_CONN_LOG_MAX_FILESIZE 1222 /*! log: number of pre-allocated log files to create */ -#define WT_STAT_CONN_LOG_PREALLOC_MAX 1222 +#define WT_STAT_CONN_LOG_PREALLOC_MAX 1223 /*! log: pre-allocated log files not ready and missed */ -#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1223 +#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1224 /*! log: pre-allocated log files prepared */ -#define WT_STAT_CONN_LOG_PREALLOC_FILES 1224 +#define WT_STAT_CONN_LOG_PREALLOC_FILES 1225 /*! log: pre-allocated log files used */ -#define WT_STAT_CONN_LOG_PREALLOC_USED 1225 +#define WT_STAT_CONN_LOG_PREALLOC_USED 1226 /*! log: records processed by log scan */ -#define WT_STAT_CONN_LOG_SCAN_RECORDS 1226 +#define WT_STAT_CONN_LOG_SCAN_RECORDS 1227 /*! log: slot close lost race */ -#define WT_STAT_CONN_LOG_SLOT_CLOSE_RACE 1227 +#define WT_STAT_CONN_LOG_SLOT_CLOSE_RACE 1228 /*! log: slot close unbuffered waits */ -#define WT_STAT_CONN_LOG_SLOT_CLOSE_UNBUF 1228 +#define WT_STAT_CONN_LOG_SLOT_CLOSE_UNBUF 1229 /*! log: slot closures */ -#define WT_STAT_CONN_LOG_SLOT_CLOSES 1229 +#define WT_STAT_CONN_LOG_SLOT_CLOSES 1230 /*! log: slot join atomic update races */ -#define WT_STAT_CONN_LOG_SLOT_RACES 1230 +#define WT_STAT_CONN_LOG_SLOT_RACES 1231 /*! log: slot join calls atomic updates raced */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_RACE 1231 +#define WT_STAT_CONN_LOG_SLOT_YIELD_RACE 1232 /*! log: slot join calls did not yield */ -#define WT_STAT_CONN_LOG_SLOT_IMMEDIATE 1232 +#define WT_STAT_CONN_LOG_SLOT_IMMEDIATE 1233 /*! log: slot join calls found active slot closed */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_CLOSE 1233 +#define WT_STAT_CONN_LOG_SLOT_YIELD_CLOSE 1234 /*! log: slot join calls slept */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_SLEEP 1234 +#define WT_STAT_CONN_LOG_SLOT_YIELD_SLEEP 1235 /*! log: slot join calls yielded */ -#define WT_STAT_CONN_LOG_SLOT_YIELD 1235 +#define WT_STAT_CONN_LOG_SLOT_YIELD 1236 /*! log: slot join found active slot closed */ -#define WT_STAT_CONN_LOG_SLOT_ACTIVE_CLOSED 1236 +#define WT_STAT_CONN_LOG_SLOT_ACTIVE_CLOSED 1237 /*! log: slot joins yield time (usecs) */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_DURATION 1237 +#define WT_STAT_CONN_LOG_SLOT_YIELD_DURATION 1238 /*! log: slot transitions unable to find free slot */ -#define WT_STAT_CONN_LOG_SLOT_NO_FREE_SLOTS 1238 +#define WT_STAT_CONN_LOG_SLOT_NO_FREE_SLOTS 1239 /*! log: slot unbuffered writes */ -#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1239 +#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1240 /*! log: total in-memory size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_MEM 1240 +#define WT_STAT_CONN_LOG_COMPRESS_MEM 1241 /*! log: total log buffer size */ -#define WT_STAT_CONN_LOG_BUFFER_SIZE 1241 +#define WT_STAT_CONN_LOG_BUFFER_SIZE 1242 /*! log: total size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_LEN 1242 +#define WT_STAT_CONN_LOG_COMPRESS_LEN 1243 /*! log: written slots coalesced */ -#define WT_STAT_CONN_LOG_SLOT_COALESCED 1243 +#define WT_STAT_CONN_LOG_SLOT_COALESCED 1244 /*! log: yields waiting for previous log file close */ -#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1244 +#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1245 /*! perf: file system read latency histogram (bucket 1) - 10-49ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT50 1245 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT50 1246 /*! perf: file system read latency histogram (bucket 2) - 50-99ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT100 1246 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT100 1247 /*! perf: file system read latency histogram (bucket 3) - 100-249ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT250 1247 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT250 1248 /*! perf: file system read latency histogram (bucket 4) - 250-499ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT500 1248 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT500 1249 /*! perf: file system read latency histogram (bucket 5) - 500-999ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT1000 1249 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT1000 1250 /*! perf: file system read latency histogram (bucket 6) - 1000ms+ */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_GT1000 1250 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_GT1000 1251 /*! perf: file system write latency histogram (bucket 1) - 10-49ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT50 1251 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT50 1252 /*! perf: file system write latency histogram (bucket 2) - 50-99ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT100 1252 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT100 1253 /*! perf: file system write latency histogram (bucket 3) - 100-249ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT250 1253 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT250 1254 /*! perf: file system write latency histogram (bucket 4) - 250-499ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT500 1254 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT500 1255 /*! perf: file system write latency histogram (bucket 5) - 500-999ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT1000 1255 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT1000 1256 /*! perf: file system write latency histogram (bucket 6) - 1000ms+ */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_GT1000 1256 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_GT1000 1257 /*! perf: operation read latency histogram (bucket 1) - 100-249us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT250 1257 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT250 1258 /*! perf: operation read latency histogram (bucket 2) - 250-499us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT500 1258 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT500 1259 /*! perf: operation read latency histogram (bucket 3) - 500-999us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT1000 1259 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT1000 1260 /*! perf: operation read latency histogram (bucket 4) - 1000-9999us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT10000 1260 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT10000 1261 /*! perf: operation read latency histogram (bucket 5) - 10000us+ */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_GT10000 1261 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_GT10000 1262 /*! perf: operation write latency histogram (bucket 1) - 100-249us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT250 1262 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT250 1263 /*! perf: operation write latency histogram (bucket 2) - 250-499us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT500 1263 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT500 1264 /*! perf: operation write latency histogram (bucket 3) - 500-999us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT1000 1264 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT1000 1265 /*! perf: operation write latency histogram (bucket 4) - 1000-9999us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT10000 1265 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT10000 1266 /*! perf: operation write latency histogram (bucket 5) - 10000us+ */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_GT10000 1266 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_GT10000 1267 /*! reconciliation: fast-path pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1267 +#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1268 /*! reconciliation: page reconciliation calls */ -#define WT_STAT_CONN_REC_PAGES 1268 +#define WT_STAT_CONN_REC_PAGES 1269 /*! reconciliation: page reconciliation calls for eviction */ -#define WT_STAT_CONN_REC_PAGES_EVICTION 1269 +#define WT_STAT_CONN_REC_PAGES_EVICTION 1270 /*! reconciliation: pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE 1270 +#define WT_STAT_CONN_REC_PAGE_DELETE 1271 /*! reconciliation: split bytes currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1271 +#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1272 /*! reconciliation: split objects currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1272 -/*! session: open cursor count */ -#define WT_STAT_CONN_SESSION_CURSOR_OPEN 1273 +#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1273 /*! session: open session count */ #define WT_STAT_CONN_SESSION_OPEN 1274 /*! session: session query timestamp calls */ @@ -6095,89 +6037,81 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_DSRC_COMPRESS_WRITE_FAIL 2102 /*! compression: page written was too small to compress */ #define WT_STAT_DSRC_COMPRESS_WRITE_TOO_SMALL 2103 -/*! compression: raw compression call failed, additional data available */ -#define WT_STAT_DSRC_COMPRESS_RAW_FAIL_TEMPORARY 2104 -/*! compression: raw compression call failed, no additional data available */ -#define WT_STAT_DSRC_COMPRESS_RAW_FAIL 2105 -/*! compression: raw compression call succeeded */ -#define WT_STAT_DSRC_COMPRESS_RAW_OK 2106 /*! cursor: bulk-loaded cursor-insert calls */ -#define WT_STAT_DSRC_CURSOR_INSERT_BULK 2107 +#define WT_STAT_DSRC_CURSOR_INSERT_BULK 2104 /*! cursor: close calls that result in cache */ -#define WT_STAT_DSRC_CURSOR_CACHE 2108 +#define WT_STAT_DSRC_CURSOR_CACHE 2105 /*! cursor: create calls */ -#define WT_STAT_DSRC_CURSOR_CREATE 2109 +#define WT_STAT_DSRC_CURSOR_CREATE 2106 /*! cursor: cursor operation restarted */ -#define WT_STAT_DSRC_CURSOR_RESTART 2110 +#define WT_STAT_DSRC_CURSOR_RESTART 2107 /*! cursor: cursor-insert key and value bytes inserted */ -#define WT_STAT_DSRC_CURSOR_INSERT_BYTES 2111 +#define WT_STAT_DSRC_CURSOR_INSERT_BYTES 2108 /*! cursor: cursor-remove key bytes removed */ -#define WT_STAT_DSRC_CURSOR_REMOVE_BYTES 2112 +#define WT_STAT_DSRC_CURSOR_REMOVE_BYTES 2109 /*! cursor: cursor-update value bytes updated */ -#define WT_STAT_DSRC_CURSOR_UPDATE_BYTES 2113 +#define WT_STAT_DSRC_CURSOR_UPDATE_BYTES 2110 /*! cursor: cursors reused from cache */ -#define WT_STAT_DSRC_CURSOR_REOPEN 2114 +#define WT_STAT_DSRC_CURSOR_REOPEN 2111 /*! cursor: insert calls */ -#define WT_STAT_DSRC_CURSOR_INSERT 2115 +#define WT_STAT_DSRC_CURSOR_INSERT 2112 /*! cursor: modify calls */ -#define WT_STAT_DSRC_CURSOR_MODIFY 2116 +#define WT_STAT_DSRC_CURSOR_MODIFY 2113 /*! cursor: next calls */ -#define WT_STAT_DSRC_CURSOR_NEXT 2117 +#define WT_STAT_DSRC_CURSOR_NEXT 2114 +/*! cursor: open cursor count */ +#define WT_STAT_DSRC_CURSOR_OPEN_COUNT 2115 /*! cursor: prev calls */ -#define WT_STAT_DSRC_CURSOR_PREV 2118 +#define WT_STAT_DSRC_CURSOR_PREV 2116 /*! cursor: remove calls */ -#define WT_STAT_DSRC_CURSOR_REMOVE 2119 +#define WT_STAT_DSRC_CURSOR_REMOVE 2117 /*! cursor: reserve calls */ -#define WT_STAT_DSRC_CURSOR_RESERVE 2120 +#define WT_STAT_DSRC_CURSOR_RESERVE 2118 /*! cursor: reset calls */ -#define WT_STAT_DSRC_CURSOR_RESET 2121 +#define WT_STAT_DSRC_CURSOR_RESET 2119 /*! cursor: search calls */ -#define WT_STAT_DSRC_CURSOR_SEARCH 2122 +#define WT_STAT_DSRC_CURSOR_SEARCH 2120 /*! cursor: search near calls */ -#define WT_STAT_DSRC_CURSOR_SEARCH_NEAR 2123 +#define WT_STAT_DSRC_CURSOR_SEARCH_NEAR 2121 /*! cursor: truncate calls */ -#define WT_STAT_DSRC_CURSOR_TRUNCATE 2124 +#define WT_STAT_DSRC_CURSOR_TRUNCATE 2122 /*! cursor: update calls */ -#define WT_STAT_DSRC_CURSOR_UPDATE 2125 +#define WT_STAT_DSRC_CURSOR_UPDATE 2123 /*! reconciliation: dictionary matches */ -#define WT_STAT_DSRC_REC_DICTIONARY 2126 +#define WT_STAT_DSRC_REC_DICTIONARY 2124 /*! reconciliation: fast-path pages deleted */ -#define WT_STAT_DSRC_REC_PAGE_DELETE_FAST 2127 +#define WT_STAT_DSRC_REC_PAGE_DELETE_FAST 2125 /*! * reconciliation: internal page key bytes discarded using suffix * compression */ -#define WT_STAT_DSRC_REC_SUFFIX_COMPRESSION 2128 +#define WT_STAT_DSRC_REC_SUFFIX_COMPRESSION 2126 /*! reconciliation: internal page multi-block writes */ -#define WT_STAT_DSRC_REC_MULTIBLOCK_INTERNAL 2129 +#define WT_STAT_DSRC_REC_MULTIBLOCK_INTERNAL 2127 /*! reconciliation: internal-page overflow keys */ -#define WT_STAT_DSRC_REC_OVERFLOW_KEY_INTERNAL 2130 +#define WT_STAT_DSRC_REC_OVERFLOW_KEY_INTERNAL 2128 /*! reconciliation: leaf page key bytes discarded using prefix compression */ -#define WT_STAT_DSRC_REC_PREFIX_COMPRESSION 2131 +#define WT_STAT_DSRC_REC_PREFIX_COMPRESSION 2129 /*! reconciliation: leaf page multi-block writes */ -#define WT_STAT_DSRC_REC_MULTIBLOCK_LEAF 2132 +#define WT_STAT_DSRC_REC_MULTIBLOCK_LEAF 2130 /*! reconciliation: leaf-page overflow keys */ -#define WT_STAT_DSRC_REC_OVERFLOW_KEY_LEAF 2133 +#define WT_STAT_DSRC_REC_OVERFLOW_KEY_LEAF 2131 /*! reconciliation: maximum blocks required for a page */ -#define WT_STAT_DSRC_REC_MULTIBLOCK_MAX 2134 +#define WT_STAT_DSRC_REC_MULTIBLOCK_MAX 2132 /*! reconciliation: overflow values written */ -#define WT_STAT_DSRC_REC_OVERFLOW_VALUE 2135 +#define WT_STAT_DSRC_REC_OVERFLOW_VALUE 2133 /*! reconciliation: page checksum matches */ -#define WT_STAT_DSRC_REC_PAGE_MATCH 2136 +#define WT_STAT_DSRC_REC_PAGE_MATCH 2134 /*! reconciliation: page reconciliation calls */ -#define WT_STAT_DSRC_REC_PAGES 2137 +#define WT_STAT_DSRC_REC_PAGES 2135 /*! reconciliation: page reconciliation calls for eviction */ -#define WT_STAT_DSRC_REC_PAGES_EVICTION 2138 +#define WT_STAT_DSRC_REC_PAGES_EVICTION 2136 /*! reconciliation: pages deleted */ -#define WT_STAT_DSRC_REC_PAGE_DELETE 2139 -/*! session: cached cursor count */ -#define WT_STAT_DSRC_SESSION_CURSORS_CACHED 2140 +#define WT_STAT_DSRC_REC_PAGE_DELETE 2137 /*! session: object compaction */ -#define WT_STAT_DSRC_SESSION_COMPACT 2141 -/*! session: open cursor count */ -#define WT_STAT_DSRC_SESSION_CURSOR_OPEN 2142 +#define WT_STAT_DSRC_SESSION_COMPACT 2138 /*! transaction: update conflicts */ -#define WT_STAT_DSRC_TXN_UPDATE_CONFLICT 2143 +#define WT_STAT_DSRC_TXN_UPDATE_CONFLICT 2139 /*! * @} diff --git a/src/third_party/wiredtiger/src/log/log.c b/src/third_party/wiredtiger/src/log/log.c index 690c5841ac8..9e520084e3c 100644 --- a/src/third_party/wiredtiger/src/log/log.c +++ b/src/third_party/wiredtiger/src/log/log.c @@ -1348,6 +1348,7 @@ __log_newfile(WT_SESSION_IMPL *session, bool conn_open, bool *created) log->write_lsn = end_lsn; log->write_start_lsn = end_lsn; } + log->dirty_lsn = log->alloc_lsn; if (created != NULL) *created = create_log; return (0); @@ -2053,7 +2054,7 @@ __wt_log_release(WT_SESSION_IMPL *session, WT_LOGSLOT *slot, bool *freep) * responsible for freeing the slot in that case. Otherwise the * worker thread will free it. */ - if (!F_ISSET(slot, WT_SLOT_FLUSH | WT_SLOT_SYNC | WT_SLOT_SYNC_DIR)) { + if (!F_ISSET(slot, WT_SLOT_FLUSH | WT_SLOT_SYNC_FLAGS)) { if (freep != NULL) *freep = 0; slot->slot_state = WT_LOG_SLOT_WRITTEN; @@ -2090,6 +2091,16 @@ __wt_log_release(WT_SESSION_IMPL *session, WT_LOGSLOT *slot, bool *freep) if (F_ISSET(slot, WT_SLOT_CLOSEFH)) __wt_cond_signal(session, conn->log_file_cond); + if (F_ISSET(slot, WT_SLOT_SYNC_DIRTY) && !F_ISSET(slot, WT_SLOT_SYNC) && + (ret = __wt_fsync(session, log->log_fh, false)) != 0) { + /* + * Ignore ENOTSUP, but don't try again. + */ + if (ret != ENOTSUP) + WT_ERR(ret); + conn->log_dirty_max = 0; + } + /* * Try to consolidate calls to fsync to wait less. Acquire a spin lock * so that threads finishing writing to the log will wait while the diff --git a/src/third_party/wiredtiger/src/log/log_slot.c b/src/third_party/wiredtiger/src/log/log_slot.c index 2d9f1a04017..d7280f17c47 100644 --- a/src/third_party/wiredtiger/src/log/log_slot.c +++ b/src/third_party/wiredtiger/src/log/log_slot.c @@ -204,6 +204,38 @@ retry: } /* + * __log_slot_dirty_max_check -- + * If we've passed the maximum of dirty system pages, schedule an + * asynchronous sync that will be performed when this slot is written. + */ +static void +__log_slot_dirty_max_check(WT_SESSION_IMPL *session, WT_LOGSLOT *slot) +{ + WT_CONNECTION_IMPL *conn; + WT_LOG *log; + WT_LSN *current, *last_sync; + + if (S2C(session)->log_dirty_max == 0) + return; + + conn = S2C(session); + log = conn->log; + current = &slot->slot_release_lsn; + + if (__wt_log_cmp(&log->dirty_lsn, &log->sync_lsn) < 0) + last_sync = &log->sync_lsn; + else + last_sync = &log->dirty_lsn; + if (current->l.file == last_sync->l.file && + current->l.offset > last_sync->l.offset && + current->l.offset - last_sync->l.offset > conn->log_dirty_max) { + /* Schedule the asynchronous sync */ + F_SET(slot, WT_SLOT_SYNC_DIRTY); + log->dirty_lsn = slot->slot_release_lsn; + } +} + +/* * __log_slot_new -- * Find a free slot and switch it as the new active slot. * Must be called holding the slot lock. @@ -263,6 +295,7 @@ __log_slot_new(WT_SESSION_IMPL *session) */ log->active_slot = slot; log->pool_index = pool_i; + __log_slot_dirty_max_check(session, slot); return (0); } } diff --git a/src/third_party/wiredtiger/src/lsm/lsm_work_unit.c b/src/third_party/wiredtiger/src/lsm/lsm_work_unit.c index 98645456fc4..d1b827d1db5 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_work_unit.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_work_unit.c @@ -37,6 +37,7 @@ __lsm_copy_chunks(WT_SESSION_IMPL *session, /* Take a copy of the current state of the LSM tree. */ nchunks = old_chunks ? lsm_tree->nold_chunks : lsm_tree->nchunks; alloc = old_chunks ? lsm_tree->old_alloc : lsm_tree->chunk_alloc; + WT_ASSERT(session, alloc > 0 && nchunks > 0); /* * If the tree array of active chunks is larger than our current buffer, diff --git a/src/third_party/wiredtiger/src/os_common/os_fhandle.c b/src/third_party/wiredtiger/src/os_common/os_fhandle.c index 3e5ad67e031..c76bf8c3f14 100644 --- a/src/third_party/wiredtiger/src/os_common/os_fhandle.c +++ b/src/third_party/wiredtiger/src/os_common/os_fhandle.c @@ -128,16 +128,46 @@ __handle_search( } /* + * __open_verbose_file_type_tag -- + * Return a string describing a file type. + */ +static const char * +__open_verbose_file_type_tag(WT_FS_OPEN_FILE_TYPE file_type) +{ + + /* + * WT_FS_OPEN_FILE_TYPE is an enum and the switch exhaustively lists the + * cases, but clang, lint and gcc argue over whether or not the switch + * is exhaustive, or if a temporary variable inserted into the mix is + * set but never read. Break out of the switch, returning some value in + * all cases, just to shut everybody up. + */ + switch (file_type) { + case WT_FS_OPEN_FILE_TYPE_CHECKPOINT: + return ("checkpoint"); + case WT_FS_OPEN_FILE_TYPE_DATA: + return ("data"); + case WT_FS_OPEN_FILE_TYPE_DIRECTORY: + return ("directory"); + case WT_FS_OPEN_FILE_TYPE_LOG: + return ("log"); + case WT_FS_OPEN_FILE_TYPE_REGULAR: + break; + } + return ("regular"); +} + +/* * __open_verbose -- * Optionally output a verbose message on handle open. */ static inline int -__open_verbose( - WT_SESSION_IMPL *session, const char *name, int file_type, u_int flags) +__open_verbose(WT_SESSION_IMPL *session, + const char *name, WT_FS_OPEN_FILE_TYPE file_type, u_int flags) { WT_DECL_ITEM(tmp); WT_DECL_RET; - const char *file_type_tag, *sep; + const char *sep; if (!WT_VERBOSE_ISSET(session, WT_VERB_FILEOPS)) return (0); @@ -146,28 +176,6 @@ __open_verbose( * It's useful to track file opens when debugging platforms, take some * effort to output good tracking information. */ - - switch (file_type) { - case WT_FS_OPEN_FILE_TYPE_CHECKPOINT: - file_type_tag = "checkpoint"; - break; - case WT_FS_OPEN_FILE_TYPE_DATA: - file_type_tag = "data"; - break; - case WT_FS_OPEN_FILE_TYPE_DIRECTORY: - file_type_tag = "directory"; - break; - case WT_FS_OPEN_FILE_TYPE_LOG: - file_type_tag = "log"; - break; - case WT_FS_OPEN_FILE_TYPE_REGULAR: - file_type_tag = "regular"; - break; - default: - file_type_tag = "unknown open type"; - break; - } - WT_RET(__wt_scr_alloc(session, 0, &tmp)); sep = " ("; #define WT_FS_OPEN_VERBOSE_FLAG(f, name) \ @@ -188,7 +196,8 @@ __open_verbose( __wt_verbose(session, WT_VERB_FILEOPS, "%s: file-open: type %s%s", - name, file_type_tag, tmp->size == 0 ? "" : (char *)tmp->data); + name, __open_verbose_file_type_tag(file_type), + tmp->size == 0 ? "" : (char *)tmp->data); err: __wt_scr_free(session, &tmp); return (ret); diff --git a/src/third_party/wiredtiger/src/os_win/os_dlopen.c b/src/third_party/wiredtiger/src/os_win/os_dlopen.c index a4e7ebe4fbb..153dd86abe5 100644 --- a/src/third_party/wiredtiger/src/os_win/os_dlopen.c +++ b/src/third_party/wiredtiger/src/os_win/os_dlopen.c @@ -15,9 +15,9 @@ int __wt_dlopen(WT_SESSION_IMPL *session, const char *path, WT_DLH **dlhp) { - DWORD windows_error; WT_DECL_RET; WT_DLH *dlh; + DWORD windows_error; WT_RET(__wt_calloc_one(session, &dlh)); WT_ERR(__wt_strdup(session, path, &dlh->name)); @@ -80,8 +80,8 @@ __wt_dlsym(WT_SESSION_IMPL *session, int __wt_dlclose(WT_SESSION_IMPL *session, WT_DLH *dlh) { - DWORD windows_error; WT_DECL_RET; + DWORD windows_error; if (FreeLibrary(dlh->handle) == FALSE) { windows_error = __wt_getlasterror(); diff --git a/src/third_party/wiredtiger/src/os_win/os_fs.c b/src/third_party/wiredtiger/src/os_win/os_fs.c index 2eac258fc70..e0011461b0a 100644 --- a/src/third_party/wiredtiger/src/os_win/os_fs.c +++ b/src/third_party/wiredtiger/src/os_win/os_fs.c @@ -41,10 +41,10 @@ static int __win_fs_remove(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, const char *name, uint32_t flags) { - DWORD windows_error; WT_DECL_ITEM(name_wide); WT_DECL_RET; WT_SESSION_IMPL *session; + DWORD windows_error; WT_UNUSED(file_system); WT_UNUSED(flags); @@ -74,11 +74,11 @@ static int __win_fs_rename(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, const char *from, const char *to, uint32_t flags) { - DWORD windows_error; WT_DECL_ITEM(from_wide); WT_DECL_ITEM(to_wide); WT_DECL_RET; WT_SESSION_IMPL *session; + DWORD windows_error; WT_UNUSED(file_system); WT_UNUSED(flags); @@ -153,10 +153,10 @@ err: __wt_scr_free(session, &name_wide); static int __win_file_close(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session) { - DWORD windows_error; WT_DECL_RET; WT_FILE_HANDLE_WIN *win_fh; WT_SESSION_IMPL *session; + DWORD windows_error; win_fh = (WT_FILE_HANDLE_WIN *)file_handle; session = (WT_SESSION_IMPL *)wt_session; @@ -201,10 +201,10 @@ static int __win_file_lock( WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session, bool lock) { - DWORD windows_error; WT_DECL_RET; WT_FILE_HANDLE_WIN *win_fh; WT_SESSION_IMPL *session; + DWORD windows_error; win_fh = (WT_FILE_HANDLE_WIN *)file_handle; session = (WT_SESSION_IMPL *)wt_session; @@ -329,10 +329,10 @@ __win_file_size( static int __win_file_sync(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session) { - DWORD windows_error; WT_DECL_RET; WT_FILE_HANDLE_WIN *win_fh; WT_SESSION_IMPL *session; + DWORD windows_error; win_fh = (WT_FILE_HANDLE_WIN *)file_handle; session = (WT_SESSION_IMPL *)wt_session; @@ -466,13 +466,13 @@ __win_open_file(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, const char *name, WT_FS_OPEN_FILE_TYPE file_type, uint32_t flags, WT_FILE_HANDLE **file_handlep) { - DWORD dwCreationDisposition, windows_error; WT_CONNECTION_IMPL *conn; WT_DECL_ITEM(name_wide); WT_DECL_RET; WT_FILE_HANDLE *file_handle; WT_FILE_HANDLE_WIN *win_fh; WT_SESSION_IMPL *session; + DWORD dwCreationDisposition, windows_error; int desired_access, f; WT_UNUSED(file_system); diff --git a/src/third_party/wiredtiger/src/os_win/os_map.c b/src/third_party/wiredtiger/src/os_win/os_map.c index 2223a6037ef..c2a404720bc 100644 --- a/src/third_party/wiredtiger/src/os_win/os_map.c +++ b/src/third_party/wiredtiger/src/os_win/os_map.c @@ -16,11 +16,11 @@ int __wt_win_map(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session, void *mapped_regionp, size_t *lenp, void *mapped_cookiep) { - DWORD windows_error; WT_FILE_HANDLE_WIN *win_fh; WT_SESSION_IMPL *session; - size_t len; wt_off_t file_size; + DWORD windows_error; + size_t len; void *map, *mapped_cookie; win_fh = (WT_FILE_HANDLE_WIN *)file_handle; @@ -77,10 +77,10 @@ int __wt_win_unmap(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session, void *mapped_region, size_t length, void *mapped_cookie) { - DWORD windows_error; WT_DECL_RET; WT_FILE_HANDLE_WIN *win_fh; WT_SESSION_IMPL *session; + DWORD windows_error; win_fh = (WT_FILE_HANDLE_WIN *)file_handle; session = (WT_SESSION_IMPL *)wt_session; diff --git a/src/third_party/wiredtiger/src/reconcile/rec_write.c b/src/third_party/wiredtiger/src/reconcile/rec_write.c index d3caa23ed02..2b2026f87cc 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_write.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_write.c @@ -66,22 +66,6 @@ typedef struct { bool leave_dirty; /* - * Raw compression (don't get me started, as if normal reconciliation - * wasn't bad enough). If an application wants absolute control over - * what gets written to disk, we give it a list of byte strings and it - * gives us back an image that becomes a file block. Because we don't - * know the number of items we're storing in a block until we've done - * a lot of work, we turn off most compression: dictionary, copy-cell, - * prefix and row-store internal page suffix compression are all off. - */ - bool raw_compression; - uint32_t raw_max_slots; /* Raw compression array sizes */ - uint32_t *raw_entries; /* Raw compression slot entries */ - uint32_t *raw_offsets; /* Raw compression slot offsets */ - uint64_t *raw_recnos; /* Raw compression recno count */ - WT_ITEM raw_destination; /* Raw compression destination buffer */ - - /* * Track if reconciliation has seen any overflow items. If a leaf page * with no overflow items is written, the parent page's address cell is * set to the leaf-no-overflow type. This means we can delete the leaf @@ -94,10 +78,9 @@ typedef struct { * they contain overflow items. In other words, leaf-no-overflow is not * guaranteed to be set on every page that doesn't contain an overflow * item, only that if it is set, the page contains no overflow items. - * - * The reason is because of raw compression: there's no easy/fast way to - * figure out if the rows selected by raw compression included overflow - * items, and the optimization isn't worth another pass over the data. + * XXX + * This was originally done because raw compression couldn't do better, + * now that raw compression has been removed, we should do better. */ bool ovfl_items; @@ -121,17 +104,9 @@ typedef struct { * when the disk image we create exceeds the page type's maximum disk * image size. * - * First, the sizes of the page we're building. If WiredTiger is doing - * page layout, page_size is the same as page_size_orig. We accumulate - * a "page size" of raw data and when we reach that size, we split the - * page into multiple chunks, eventually compressing those chunks. When - * the application is doing page layout (raw compression is configured), - * page_size can continue to grow past page_size_orig, and we keep - * accumulating raw data until the raw compression callback accepts it. + * First, the target size of the page we're building. */ - uint32_t page_size; /* Set page size */ - uint32_t page_size_orig; /* Saved set page size */ - uint32_t max_raw_page_size; /* Max page size with raw compression */ + uint32_t page_size; /* Page size */ /* * Second, the split size: if we're doing the page layout, split to a @@ -814,61 +789,6 @@ err: __wt_page_out(session, &next); } /* - * __rec_raw_compression_config -- - * Configure raw compression. - */ -static inline bool -__rec_raw_compression_config(WT_SESSION_IMPL *session, - uint32_t flags, WT_PAGE *page, WT_SALVAGE_COOKIE *salvage) -{ - WT_BTREE *btree; - - btree = S2BT(session); - - /* Check if raw compression configured. */ - if (btree->compressor == NULL || - btree->compressor->compress_raw == NULL) - return (false); - - /* Only for row-store and variable-length column-store objects. */ - if (page->type == WT_PAGE_COL_FIX) - return (false); - - /* - * XXX - * Turn off if lookaside or update/restore are configured: those modes - * potentially write blocks without entries and raw compression isn't - * ready for that. - */ - if (LF_ISSET(WT_REC_LOOKASIDE | WT_REC_UPDATE_RESTORE)) - return (false); - - /* - * Raw compression cannot support dictionary compression. (Technically, - * we could still use the raw callback on column-store variable length - * internal pages with dictionary compression configured, because - * dictionary compression only applies to column-store leaf pages, but - * that seems an unlikely use case.) - */ - if (btree->dictionary != 0) - return (false); - - /* Raw compression cannot support prefix compression. */ - if (btree->prefix_compression) - return (false); - - /* - * Raw compression is also turned off during salvage: we can't allow - * pages to split during salvage, raw compression has no point if it - * can't manipulate the page size. - */ - if (salvage != NULL) - return (false); - - return (true); -} - -/* * __rec_init -- * Initialize the reconciliation structure. */ @@ -1003,11 +923,6 @@ __rec_init(WT_SESSION_IMPL *session, /* Track if the page can be marked clean. */ r->leave_dirty = false; - /* Raw compression. */ - r->raw_compression = - __rec_raw_compression_config(session, flags, page, salvage); - r->raw_destination.flags = WT_ITEM_ALIGNED; - /* Track overflow items. */ r->ovfl_items = false; @@ -1061,13 +976,9 @@ __rec_init(WT_SESSION_IMPL *session, * implement suffix compression for custom collators, we can add a * setting to the collator, configured when the collator is added, that * turns on suffix compression. - * - * The raw compression routines don't even consider suffix compression, - * but it doesn't hurt to confirm that. */ r->key_sfx_compress_conf = false; - if (btree->collator == NULL && - btree->internal_key_truncate && !r->raw_compression) + if (btree->collator == NULL && btree->internal_key_truncate) r->key_sfx_compress_conf = true; r->is_bulk_load = false; @@ -1128,11 +1039,6 @@ __rec_destroy(WT_SESSION_IMPL *session, void *reconcilep) return; *(WT_RECONCILE **)reconcilep = NULL; - __wt_free(session, r->raw_entries); - __wt_free(session, r->raw_offsets); - __wt_free(session, r->raw_recnos); - __wt_buf_free(session, &r->raw_destination); - __wt_buf_free(session, &r->chunkA.key); __wt_buf_free(session, &r->chunkA.min_key); __wt_buf_free(session, &r->chunkA.image); @@ -2167,8 +2073,7 @@ __rec_need_split(WT_RECONCILE *r, size_t len) len += r->supd_memsize; /* Check for the disk image crossing a boundary. */ - return (r->raw_compression ? - len > r->space_avail : WT_CHECK_CROSSING_BND(r, len)); + return (WT_CHECK_CROSSING_BND(r, len)); } /* @@ -2270,26 +2175,9 @@ __rec_split_init(WT_SESSION_IMPL *session, if (r->salvage != NULL) max = __rec_leaf_page_max(session, r); - /* - * Set the page sizes. If we're doing the page layout, the maximum page - * size is the same as the page size. If the application is doing page - * layout (raw compression is configured), we accumulate some amount of - * additional data because we don't know how well it will compress, and - * we don't want to increment our way up to the amount of data needed by - * the application to successfully compress to the target page size. - * Ideally accumulate data several times the page size without - * approaching the memory page maximum, but at least have data worth - * one page. - * - * There are cases when we grow the page size to accommodate large - * records, in those cases we split the pages once they have crossed - * the maximum size for a page with raw compression. - */ - r->page_size = r->page_size_orig = (uint32_t)max; - if (r->raw_compression) - r->max_raw_page_size = r->page_size = - (uint32_t)WT_MIN((uint64_t)r->page_size * 10, - WT_MAX((uint64_t)r->page_size, btree->maxmempage / 2)); + /* Set the page sizes. */ + r->page_size = (uint32_t)max; + /* * If we have to split, we want to choose a smaller page size for the * split pages, because otherwise we could end up splitting one large @@ -2320,14 +2208,12 @@ __rec_split_init(WT_SESSION_IMPL *session, * increasing the size of the last page written without decreasing the * penultimate page size beyond the minimum split size. * - * Finally, all this doesn't matter for fixed-size column-store pages, - * raw compression, and salvage. Fixed-size column store pages can - * split under (very) rare circumstances, but they're allocated at a - * fixed page size, never anything smaller. In raw compression, the - * underlying compression routine decides when we split, so it's not our - * problem. In salvage, as noted above, we can't split at all. - */ - if (r->raw_compression || r->salvage != NULL) { + * Finally, all this doesn't matter for fixed-size column-store pages + * and salvage. Fixed-size column store pages can split under (very) + * rare circumstances, but they're allocated at a fixed page size, never + * anything smaller. In salvage, as noted above, we can't split at all. + */ + if (r->salvage != NULL) { r->split_size = 0; r->space_avail = r->page_size - WT_PAGE_HEADER_BYTE_SIZE(btree); } else if (page->type == WT_PAGE_COL_FIX) { @@ -2420,34 +2306,6 @@ __rec_is_checkpoint(WT_SESSION_IMPL *session, WT_RECONCILE *r) } /* - * __rec_split_row_promote_cell -- - * Get a key from a cell for the purposes of promotion. - */ -static int -__rec_split_row_promote_cell(WT_SESSION_IMPL *session, - WT_RECONCILE *r, WT_PAGE_HEADER *dsk, WT_ITEM *key) -{ - WT_BTREE *btree; - WT_CELL *cell; - WT_CELL_UNPACK *kpack, _kpack; - - btree = S2BT(session); - kpack = &_kpack; - - /* - * The cell had better have a zero-length prefix and not be a copy cell; - * the first cell on a page cannot refer an earlier cell on the page. - */ - cell = WT_PAGE_HEADER_BYTE(btree, dsk); - __wt_cell_unpack(cell, kpack); - WT_ASSERT(session, - kpack->prefix == 0 && kpack->raw != WT_CELL_VALUE_COPY); - - WT_RET(__wt_cell_data_copy(session, r->page->type, kpack, key)); - return (0); -} - -/* * __rec_split_row_promote -- * Key promotion for a row-store. */ @@ -2752,390 +2610,6 @@ __rec_split_crossing_bnd( } /* - * __rec_split_raw -- - * Raw compression. - */ -static int -__rec_split_raw(WT_SESSION_IMPL *session, - WT_RECONCILE *r, size_t next_len, bool no_more_rows) -{ - WT_BM *bm; - WT_BTREE *btree; - WT_CELL *cell; - WT_CELL_UNPACK *unpack, _unpack; - WT_CHUNK *chunk, *next, *tmp; - WT_COMPRESSOR *compressor; - WT_DECL_RET; - WT_ITEM *dst; - WT_PAGE *page; - WT_PAGE_HEADER *dsk; - WT_SESSION *wt_session; - size_t corrected_page_size, extra_skip, len, result_len; - uint64_t recno; - uint32_t entry, i, max_image_slot, result_slots, slots; - uint8_t *next_start; - bool compressed, last_block; - - wt_session = (WT_SESSION *)session; - btree = S2BT(session); - bm = btree->bm; - - unpack = &_unpack; - compressor = btree->compressor; - dst = &r->raw_destination; - page = r->page; - compressed = false; - - chunk = r->cur_ptr; - if (r->prev_ptr == NULL) - r->prev_ptr = &r->chunkB; - next = r->prev_ptr; - - /* - * We can get here if the first key/value pair won't fit. - */ - if (r->entries == 0 && !__rec_need_split(r, 0)) - goto split_grow; - - /* - * Build arrays of offsets and cumulative counts of cells and rows in - * the page: the offset is the byte offset to the possible split-point - * (adjusted for an initial chunk that cannot be compressed), entries - * is the cumulative page entries covered by the byte offset, recnos is - * the cumulative rows covered by the byte offset. Allocate to handle - * both column- and row-store regardless of this page type, structures - * are potentially reused for subsequent reconciliations of different - * page types. - */ - if (r->entries >= r->raw_max_slots) { - __wt_free(session, r->raw_entries); - __wt_free(session, r->raw_offsets); - __wt_free(session, r->raw_recnos); - r->raw_max_slots = 0; - - i = r->entries + 100; - WT_RET(__wt_calloc_def(session, i, &r->raw_entries)); - WT_RET(__wt_calloc_def(session, i, &r->raw_offsets)); - WT_RET(__wt_calloc_def(session, i, &r->raw_recnos)); - r->raw_max_slots = i; - } - - /* - * Walk the disk image looking for places where we can split it, which - * requires setting the number of entries. - */ - dsk = chunk->image.mem; - dsk->u.entries = r->entries; - - /* - * We track the record number at each column-store split point, set an - * initial value. - */ - recno = WT_RECNO_OOB; - if (page->type == WT_PAGE_COL_VAR) - recno = chunk->recno; - - entry = max_image_slot = slots = 0; - WT_CELL_FOREACH(btree, dsk, cell, unpack, i) { - ++entry; - - /* - * Row-store pages can split at keys, but not at values, - * column-store pages can split at values. - */ - __wt_cell_unpack(cell, unpack); - switch (unpack->type) { - case WT_CELL_KEY: - case WT_CELL_KEY_OVFL: - case WT_CELL_KEY_SHORT: - break; - case WT_CELL_ADDR_DEL: - case WT_CELL_ADDR_INT: - case WT_CELL_ADDR_LEAF: - case WT_CELL_ADDR_LEAF_NO: - case WT_CELL_DEL: - case WT_CELL_VALUE: - case WT_CELL_VALUE_OVFL: - case WT_CELL_VALUE_SHORT: - if (page->type == WT_PAGE_COL_INT) { - recno = unpack->v; - break; - } - if (page->type == WT_PAGE_COL_VAR) { - recno += __wt_cell_rle(unpack); - break; - } - r->raw_entries[slots] = entry; - continue; - WT_ILLEGAL_VALUE(session, unpack->type); - } - - /* - * We can't compress the first 64B of the block (it must be - * written without compression), and a possible split point - * may appear in that 64B; keep it simple, ignore the first - * allocation size of data, anybody splitting smaller than - * that (as calculated before compression), is doing it wrong. - */ - if ((len = WT_PTRDIFF(cell, dsk)) > btree->allocsize) - r->raw_offsets[++slots] = - WT_STORE_SIZE(len - WT_BLOCK_COMPRESS_SKIP); - - if (page->type == WT_PAGE_COL_INT || - page->type == WT_PAGE_COL_VAR) - r->raw_recnos[slots] = recno; - r->raw_entries[slots] = entry; - - /* - * Don't create an image so large that any future update will - * cause a split in memory. - */ - if (max_image_slot == 0 && len > (size_t)r->max_raw_page_size) - max_image_slot = slots; - } - - /* - * If we haven't managed to find at least one split point, we're done, - * don't bother calling the underlying compression function. - */ - if (slots == 0) { - result_slots = 0; - goto no_slots; - } - - /* The slot at array's end is the total length of the data. */ - r->raw_offsets[++slots] = - WT_STORE_SIZE(WT_PTRDIFF(cell, dsk) - WT_BLOCK_COMPRESS_SKIP); - - /* - * Allocate a destination buffer. If there's a pre-size function, call - * it to determine the destination buffer's size, else the destination - * buffer is documented to be at least the source size. (We can't use - * the target page size, any single key/value could be larger than the - * page size. Don't bother figuring out a minimum, just use the source - * size.) - * - * The destination buffer needs to be large enough for the final block - * size, corrected for the requirements of the underlying block manager. - * If the final block size is 8KB, that's a multiple of 512B and so the - * underlying block manager is fine with it. But... we don't control - * what the pre_size method returns us as a required size, and we don't - * want to document the compress_raw method has to skip bytes in the - * buffer because that's confusing, so do something more complicated. - * First, find out how much space the compress_raw function might need, - * either the value returned from pre_size, or the initial source size. - * Add the compress-skip bytes, and then correct that value for the - * underlying block manager. As a result, we have a destination buffer - * that's large enough when calling the compress_raw method, and there - * are bytes in the header just for us. - */ - if (compressor->pre_size == NULL) - result_len = (size_t)r->raw_offsets[slots]; - else - WT_RET(compressor->pre_size(compressor, wt_session, - (uint8_t *)dsk + WT_BLOCK_COMPRESS_SKIP, - (size_t)r->raw_offsets[slots], &result_len)); - extra_skip = btree->kencryptor == NULL ? 0 : - btree->kencryptor->size_const + WT_ENCRYPT_LEN_SIZE; - - corrected_page_size = result_len + WT_BLOCK_COMPRESS_SKIP; - WT_RET(bm->write_size(bm, session, &corrected_page_size)); - WT_RET(__wt_buf_init(session, dst, corrected_page_size)); - - /* - * Copy the header bytes into the destination buffer, then call the - * compression function. - */ - memcpy(dst->mem, dsk, WT_BLOCK_COMPRESS_SKIP); - ret = compressor->compress_raw(compressor, wt_session, - r->page_size_orig, btree->split_pct, - WT_BLOCK_COMPRESS_SKIP + extra_skip, - (uint8_t *)dsk + WT_BLOCK_COMPRESS_SKIP, r->raw_offsets, - max_image_slot == 0 ? slots : max_image_slot, - (uint8_t *)dst->mem + WT_BLOCK_COMPRESS_SKIP, - result_len, - no_more_rows || max_image_slot != 0, - &result_len, &result_slots); - switch (ret) { - case EAGAIN: - /* - * The compression function wants more rows, accumulate and - * retry if possible. - * - * First, reset the resulting slots count, just in case the - * compression function modified it before giving up. - */ - result_slots = 0; - - /* - * If the image is too large and there are more rows to gather, - * act as if the compression engine gave up on this chunk of - * data. That doesn't make sense (we flagged the engine that we - * wouldn't give it any more rows, but it's a possible return). - */ - if (no_more_rows || max_image_slot == 0) - break; - /* FALLTHROUGH */ - case 0: - /* - * If the compression function returned zero result slots, it's - * giving up and we write the original data. (This is a pretty - * bad result: we've not done compression on a block much larger - * than the maximum page size, but once compression gives up, - * there's not much else we can do.) - * - * If the compression function returned non-zero result slots, - * we were successful and have a block to write. - */ - if (result_slots == 0) { - WT_STAT_DATA_INCR(session, compress_raw_fail); - - /* - * If there are no more rows, we can write the original - * data from the original buffer, else take all but the - * last row of the original data (the last row has to be - * set as the key for the next block). - */ - if (!no_more_rows) - result_slots = slots - 1; - } else { - WT_STAT_DATA_INCR(session, compress_raw_ok); - - /* - * If there are more rows and the compression function - * consumed all of the current data, there are problems: - * First, with row-store objects, we're potentially - * skipping updates, we must have a key for the next - * block so we know with what block a skipped update is - * associated. Second, if the compression function - * compressed all of the data, we're not pushing it - * hard enough (unless we got lucky and gave it exactly - * the right amount to work with, which is unlikely). - * Handle both problems by accumulating more data any - * time we're not writing the last block and compression - * ate all of the rows. - */ - if (result_slots == slots && !no_more_rows) - result_slots = 0; - else { - /* - * Finalize the compressed disk image's - * information. - */ - dst->size = result_len + WT_BLOCK_COMPRESS_SKIP; - - compressed = true; - } - } - break; - default: - return (ret); - } - -no_slots: - /* - * Check for the last block we're going to write: if no more rows and - * we failed to compress anything, or we compressed everything, it's - * the last block. - */ - last_block = no_more_rows && - (result_slots == 0 || result_slots == slots); - - if (!last_block && result_slots != 0) { - /* - * Writing the current (possibly compressed), chunk. - * Finalize the current chunk's information. - */ - chunk->image.size = (size_t) - r->raw_offsets[result_slots] + WT_BLOCK_COMPRESS_SKIP; - chunk->entries = r->raw_entries[result_slots - 1]; - - /* Move any remnant to the next chunk. */ - len = WT_PTRDIFF(r->first_free, - (uint8_t *)dsk + chunk->image.size); - WT_ASSERT(session, len > 0); - WT_RET(__rec_split_chunk_init( - session, r, next, chunk->image.memsize)); - next_start = WT_PAGE_HEADER_BYTE(btree, next->image.mem); - (void)memcpy(next_start, r->first_free - len, len); - - /* Set the key for the next chunk. */ - switch (page->type) { - case WT_PAGE_COL_INT: - next->recno = r->raw_recnos[result_slots]; - break; - case WT_PAGE_COL_VAR: - next->recno = r->raw_recnos[result_slots - 1]; - break; - case WT_PAGE_ROW_INT: - case WT_PAGE_ROW_LEAF: - next->recno = WT_RECNO_OOB; - /* - * Confirm there was uncompressed data remaining - * in the buffer, we're about to read it for the - * next chunk's initial key. - */ - WT_RET(__rec_split_row_promote_cell( - session, r, next->image.mem, &next->key)); - break; - } - - /* Update the tracking information. */ - r->entries -= r->raw_entries[result_slots - 1]; - r->first_free = next_start + len; - r->space_avail += r->raw_offsets[result_slots]; - WT_ASSERT(session, r->first_free + r->space_avail <= - (uint8_t *)next->image.mem + next->image.memsize); - } else if (no_more_rows) { - /* - * No more rows to accumulate, writing the entire chunk. - * Finalize the current chunk's information. - */ - chunk->image.size = WT_PTRDIFF32(r->first_free, dsk); - chunk->entries = r->entries; - - /* Clear the tracking information. */ - r->entries = 0; - r->first_free = NULL; - r->space_avail = 0; - } else { - /* - * Compression failed, there are more rows to accumulate and the - * compression function wants to try again; increase the size of - * the "page" and try again after we accumulate some more rows. - */ - WT_STAT_DATA_INCR(session, compress_raw_fail_temporary); - goto split_grow; - } - - /* Write the chunk. */ - WT_RET(__rec_split_write(session, r, - r->cur_ptr, compressed ? dst : NULL, last_block)); - - /* Switch chunks. */ - tmp = r->prev_ptr; - r->prev_ptr = r->cur_ptr; - r->cur_ptr = tmp; - - /* - * We got called because there wasn't enough room in the buffer for the - * next key and we might or might not have written a block. In any case, - * make sure the next key fits into the buffer. - */ - if (r->space_avail < next_len) { -split_grow: /* - * Double the page size and make sure we accommodate at least - * one more record. The reason for the latter is that we may - * be here because there's a large key/value pair that won't - * fit in our initial page buffer, even at its expanded size. - */ - r->page_size *= 2; - return (__rec_split_grow(session, r, r->page_size + next_len)); - } - return (0); -} - -/* * __rec_split_finish_process_prev -- * If the two split chunks together fit in a single page, merge them into * one. If they do not fit in a single page but the last is smaller than @@ -3237,30 +2711,10 @@ __rec_split_finish_process_prev(WT_SESSION_IMPL *session, WT_RECONCILE *r) static int __rec_split_finish(WT_SESSION_IMPL *session, WT_RECONCILE *r) { - WT_BTREE *btree; - size_t data_size; - - btree = S2BT(session); - /* - * We're done reconciling, write the final page. Call raw compression - * until/unless there's not enough data to compress. - */ - if (r->entries != 0 && r->raw_compression) { - while (r->entries != 0) { - data_size = - WT_PTRDIFF(r->first_free, r->cur_ptr->image.mem); - if (data_size <= btree->allocsize) - break; - WT_RET(__rec_split_raw(session, r, 0, true)); - } - if (r->entries == 0) - return (0); - } - - /* - * We may arrive here with no entries to write if the page was entirely - * empty or if nothing on the page was visible to us. + * We're done reconciling, write the final page. We may arrive here with + * no entries to write if the page was entirely empty or if nothing on + * the page was visible to us. * * Pages with skipped or not-yet-globally visible updates aren't really * empty; otherwise, the page is truly empty and we will merge it into @@ -3274,8 +2728,8 @@ __rec_split_finish(WT_SESSION_IMPL *session, WT_RECONCILE *r) WT_PTRDIFF32(r->first_free, r->cur_ptr->image.mem); r->cur_ptr->entries = r->entries; - /* If not raw compression, potentially reconsider a previous chunk. */ - if (!r->raw_compression && r->prev_ptr != NULL) + /* Potentially reconsider a previous chunk. */ + if (r->prev_ptr != NULL) WT_RET(__rec_split_finish_process_prev(session, r)); /* Write the remaining data/last page. */ @@ -3915,26 +3369,21 @@ __wt_bulk_insert_row(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk) cursor->value.data, cursor->value.size, (uint64_t)0)); /* Boundary: split or write the page. */ - if (r->raw_compression) { - if (key->len + val->len > r->space_avail) - WT_RET(__rec_split_raw( - session, r, key->len + val->len, false)); - } else - if (WT_CROSSING_SPLIT_BND(r, key->len + val->len)) { - /* - * Turn off prefix compression until a full key written - * to the new page, and (unless already working with an - * overflow key), rebuild the key without compression. - */ - if (r->key_pfx_compress_conf) { - r->key_pfx_compress = false; - if (!ovfl_key) - WT_RET(__rec_cell_build_leaf_key( - session, r, NULL, 0, &ovfl_key)); - } - WT_RET(__rec_split_crossing_bnd( - session, r, key->len + val->len)); + if (WT_CROSSING_SPLIT_BND(r, key->len + val->len)) { + /* + * Turn off prefix compression until a full key written to the + * new page, and (unless already working with an overflow key), + * rebuild the key without compression. + */ + if (r->key_pfx_compress_conf) { + r->key_pfx_compress = false; + if (!ovfl_key) + WT_RET(__rec_cell_build_leaf_key( + session, r, NULL, 0, &ovfl_key)); } + WT_RET(__rec_split_crossing_bnd( + session, r, key->len + val->len)); + } /* Copy the key/value pair onto the page. */ __rec_copy_incr(session, r, key); @@ -4083,12 +3532,8 @@ __wt_bulk_insert_var( r, cbulk->last.data, cbulk->last.size, cbulk->rle)); /* Boundary: split or write the page. */ - if (r->raw_compression) { - if (val->len > r->space_avail) - WT_RET(__rec_split_raw(session, r, val->len, false)); - } else - if (WT_CROSSING_SPLIT_BND(r, val->len)) - WT_RET(__rec_split_crossing_bnd(session, r, val->len)); + if (WT_CROSSING_SPLIT_BND(r, val->len)) + WT_RET(__rec_split_crossing_bnd(session, r, val->len)); /* Copy the value onto the page. */ if (btree->dictionary) @@ -4224,14 +3669,8 @@ __rec_col_int(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_REF *pageref) WT_CHILD_RELEASE_ERR(session, hazard, ref); /* Boundary: split or write the page. */ - if (__rec_need_split(r, val->len)) { - if (r->raw_compression) - WT_ERR(__rec_split_raw( - session, r, val->len, false)); - else - WT_ERR(__rec_split_crossing_bnd( - session, r, val->len)); - } + if (__rec_need_split(r, val->len)) + WT_ERR(__rec_split_crossing_bnd(session, r, val->len)); /* Copy the value onto the page. */ __rec_copy_incr(session, r, val); @@ -4273,14 +3712,8 @@ __rec_col_merge(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) addr->addr, addr->size, __rec_vtype(addr), r->recno); /* Boundary: split or write the page. */ - if (__rec_need_split(r, val->len)) { - if (r->raw_compression) - WT_RET(__rec_split_raw( - session, r, val->len, false)); - else - WT_RET(__rec_split_crossing_bnd( - session, r, val->len)); - } + if (__rec_need_split(r, val->len)) + WT_RET(__rec_split_crossing_bnd(session, r, val->len)); /* Copy the value onto the page. */ __rec_copy_incr(session, r, val); @@ -4547,12 +3980,8 @@ __rec_col_var_helper(WT_SESSION_IMPL *session, WT_RECONCILE *r, session, r, value->data, value->size, rle)); /* Boundary: split or write the page. */ - if (__rec_need_split(r, val->len)) { - if (r->raw_compression) - WT_RET(__rec_split_raw(session, r, val->len, false)); - else - WT_RET(__rec_split_crossing_bnd(session, r, val->len)); - } + if (__rec_need_split(r, val->len)) + WT_RET(__rec_split_crossing_bnd(session, r, val->len)); /* Copy the value onto the page. */ if (!deleted && !overflow_type && btree->dictionary) @@ -5253,25 +4682,20 @@ __rec_row_int(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) /* Boundary: split or write the page. */ if (__rec_need_split(r, key->len + val->len)) { - if (r->raw_compression) - WT_ERR(__rec_split_raw( - session, r, key->len + val->len, false)); - else { - /* - * In one path above, we copied address blocks - * from the page rather than building the actual - * key. In that case, we have to build the key - * now because we are about to promote it. - */ - if (key_onpage_ovfl) { - WT_ERR(__wt_buf_set(session, r->cur, - WT_IKEY_DATA(ikey), ikey->size)); - key_onpage_ovfl = false; - } - - WT_ERR(__rec_split_crossing_bnd( - session, r, key->len + val->len)); + /* + * In one path above, we copied address blocks from the + * page rather than building the actual key. In that + * case, we have to build the key now because we are + * about to promote it. + */ + if (key_onpage_ovfl) { + WT_ERR(__wt_buf_set(session, r->cur, + WT_IKEY_DATA(ikey), ikey->size)); + key_onpage_ovfl = false; } + + WT_ERR(__rec_split_crossing_bnd( + session, r, key->len + val->len)); } /* Copy the key and value onto the page. */ @@ -5322,14 +4746,9 @@ __rec_row_merge(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) addr->addr, addr->size, __rec_vtype(addr), WT_RECNO_OOB); /* Boundary: split or write the page. */ - if (__rec_need_split(r, key->len + val->len)) { - if (r->raw_compression) - WT_RET(__rec_split_raw( - session, r, key->len + val->len, false)); - else - WT_RET(__rec_split_crossing_bnd( - session, r, key->len + val->len)); - } + if (__rec_need_split(r, key->len + val->len)) + WT_RET(__rec_split_crossing_bnd( + session, r, key->len + val->len)); /* Copy the key and value onto the page. */ __rec_copy_incr(session, r, key); @@ -5664,40 +5083,31 @@ build: /* Boundary: split or write the page. */ if (__rec_need_split(r, key->len + val->len)) { - if (r->raw_compression) - WT_ERR(__rec_split_raw( - session, r, key->len + val->len, false)); - else { - /* - * If we copied address blocks from the page - * rather than building the actual key, we have - * to build the key now because we are about to - * promote it. - */ - if (key_onpage_ovfl) { - WT_ERR(__wt_dsk_cell_data_ref(session, - WT_PAGE_ROW_LEAF, kpack, r->cur)); - WT_NOT_READ(key_onpage_ovfl, false); - } - - /* - * Turn off prefix compression until a full key - * written to the new page, and (unless already - * working with an overflow key), rebuild the - * key without compression. - */ - if (r->key_pfx_compress_conf) { - r->key_pfx_compress = false; - if (!ovfl_key) - WT_ERR( - __rec_cell_build_leaf_key( - session, r, NULL, 0, - &ovfl_key)); - } + /* + * If we copied address blocks from the page rather than + * building the actual key, we have to build the key now + * because we are about to promote it. + */ + if (key_onpage_ovfl) { + WT_ERR(__wt_dsk_cell_data_ref(session, + WT_PAGE_ROW_LEAF, kpack, r->cur)); + WT_NOT_READ(key_onpage_ovfl, false); + } - WT_ERR(__rec_split_crossing_bnd( - session, r, key->len + val->len)); + /* + * Turn off prefix compression until a full key written + * to the new page, and (unless already working with an + * overflow key), rebuild the key without compression. + */ + if (r->key_pfx_compress_conf) { + r->key_pfx_compress = false; + if (!ovfl_key) + WT_ERR(__rec_cell_build_leaf_key( + session, r, NULL, 0, &ovfl_key)); } + + WT_ERR(__rec_split_crossing_bnd( + session, r, key->len + val->len)); } /* Copy the key/value pair onto the page. */ @@ -5806,28 +5216,20 @@ __rec_row_leaf_insert(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins) /* Boundary: split or write the page. */ if (__rec_need_split(r, key->len + val->len)) { - if (r->raw_compression) - WT_RET(__rec_split_raw( - session, r, key->len + val->len, false)); - else { - /* - * Turn off prefix compression until a full key - * written to the new page, and (unless already - * working with an overflow key), rebuild the - * key without compression. - */ - if (r->key_pfx_compress_conf) { - r->key_pfx_compress = false; - if (!ovfl_key) - WT_RET( - __rec_cell_build_leaf_key( - session, r, NULL, 0, - &ovfl_key)); - } - - WT_RET(__rec_split_crossing_bnd( - session, r, key->len + val->len)); + /* + * Turn off prefix compression until a full key written + * to the new page, and (unless already working with an + * overflow key), rebuild the key without compression. + */ + if (r->key_pfx_compress_conf) { + r->key_pfx_compress = false; + if (!ovfl_key) + WT_RET(__rec_cell_build_leaf_key( + session, r, NULL, 0, &ovfl_key)); } + + WT_RET(__rec_split_crossing_bnd( + session, r, key->len + val->len)); } /* Copy the key/value pair onto the page. */ diff --git a/src/third_party/wiredtiger/src/session/session_api.c b/src/third_party/wiredtiger/src/session/session_api.c index c413fb21f32..0c0cb8f5b43 100644 --- a/src/third_party/wiredtiger/src/session/session_api.c +++ b/src/third_party/wiredtiger/src/session/session_api.c @@ -493,7 +493,7 @@ __session_open_cursor_int(WT_SESSION_IMPL *session, const char *uri, case 'b': if (WT_PREFIX_MATCH(uri, "backup:")) WT_RET(__wt_curbackup_open( - session, uri, cfg, cursorp)); + session, uri, other, cfg, cursorp)); break; case 's': if (WT_PREFIX_MATCH(uri, "statistics:")) @@ -570,10 +570,11 @@ __session_open_cursor(WT_SESSION *wt_session, WT_CURSOR *cursor; WT_DECL_RET; WT_SESSION_IMPL *session; - bool statjoin; + bool dup_backup, statjoin; cursor = *cursorp = NULL; + dup_backup = false; session = (WT_SESSION_IMPL *)wt_session; SESSION_API_CALL(session, open_cursor, config, cfg); @@ -589,11 +590,19 @@ __session_open_cursor(WT_SESSION *wt_session, if ((ret = __wt_cursor_cache_get( session, uri, to_dup, cfg, &cursor)) == 0) goto done; + + /* + * Detect if we're duplicating a backup cursor specifically. + * That needs special handling. + */ + if (to_dup != NULL && strcmp(to_dup->uri, "backup:") == 0) + dup_backup = true; WT_ERR_NOTFOUND_OK(ret); if (to_dup != NULL) { uri = to_dup->uri; - if (!WT_PREFIX_MATCH(uri, "colgroup:") && + if (!WT_PREFIX_MATCH(uri, "backup:") && + !WT_PREFIX_MATCH(uri, "colgroup:") && !WT_PREFIX_MATCH(uri, "index:") && !WT_PREFIX_MATCH(uri, "file:") && !WT_PREFIX_MATCH(uri, "lsm:") && @@ -605,10 +614,10 @@ __session_open_cursor(WT_SESSION *wt_session, } WT_ERR(__session_open_cursor_int(session, uri, NULL, - statjoin ? to_dup : NULL, cfg, &cursor)); + statjoin || dup_backup ? to_dup : NULL, cfg, &cursor)); done: - if (to_dup != NULL && !statjoin) + if (to_dup != NULL && !statjoin && !dup_backup) WT_ERR(__wt_cursor_dup_position(to_dup, cursor)); *cursorp = cursor; diff --git a/src/third_party/wiredtiger/src/support/hazard.c b/src/third_party/wiredtiger/src/support/hazard.c index eb65c00741c..2a216ff1d24 100644 --- a/src/third_party/wiredtiger/src/support/hazard.c +++ b/src/third_party/wiredtiger/src/support/hazard.c @@ -322,7 +322,8 @@ hazard_get_reference( * Return if there's a hazard pointer to the page in the system. */ WT_HAZARD * -__wt_hazard_check(WT_SESSION_IMPL *session, WT_REF *ref) +__wt_hazard_check(WT_SESSION_IMPL *session, + WT_REF *ref, WT_SESSION_IMPL **sessionp) { WT_CONNECTION_IMPL *conn; WT_HAZARD *hp; @@ -366,6 +367,8 @@ __wt_hazard_check(WT_SESSION_IMPL *session, WT_REF *ref) if (hp->ref == ref) { WT_STAT_CONN_INCRV(session, cache_hazard_walks, walk_cnt); + if (sessionp != NULL) + *sessionp = s; goto done; } } @@ -408,18 +411,22 @@ bool __wt_hazard_check_assert(WT_SESSION_IMPL *session, void *ref, bool waitfor) { WT_HAZARD *hp; + WT_SESSION_IMPL *s; int i; + s = NULL; for (i = 0;;) { - if ((hp = __wt_hazard_check(session, ref)) == NULL) + if ((hp = __wt_hazard_check(session, ref, &s)) == NULL) return (true); if (!waitfor || ++i > 100) break; __wt_sleep(0, 10000); } __wt_errx(session, - "hazard pointer reference to discarded object: (%p: %s, line %d)", - (void *)hp->ref, hp->func, hp->line); + "hazard pointer reference to discarded object: " + "(%p: session %p name %s: %s, line %d)", + (void *)hp->ref, (void *)s, + s->name == NULL ? "UNKNOWN" : s->name, hp->func, hp->line); return (false); } diff --git a/src/third_party/wiredtiger/src/support/mtx_rw.c b/src/third_party/wiredtiger/src/support/mtx_rw.c index 959405dee50..9b7c3d441bd 100644 --- a/src/third_party/wiredtiger/src/support/mtx_rw.c +++ b/src/third_party/wiredtiger/src/support/mtx_rw.c @@ -171,13 +171,14 @@ void __wt_readlock(WT_SESSION_IMPL *session, WT_RWLOCK *l) { WT_RWLOCK new, old; - uint64_t time_start, time_stop; - int64_t **stats; + uint64_t time_diff, time_start, time_stop; + int64_t *session_stats, **stats; int16_t writers_active; uint8_t ticket; int pause_cnt; bool set_stats; + session_stats = NULL; /* -Wconditional-uninitialized */ stats = NULL; /* -Wconditional-uninitialized */ time_start = time_stop = 0; /* -Wconditional-uninitialized */ @@ -243,6 +244,7 @@ stall: __wt_cond_wait(session, if (set_stats) { stats = (int64_t **)S2C(session)->stats; stats[session->stat_bucket][l->stat_read_count_off]++; + session_stats = (int64_t *)&(session->stats); time_start = __wt_clock(session); } /* Wait for our group to start. */ @@ -260,12 +262,15 @@ stall: __wt_cond_wait(session, } if (set_stats) { time_stop = __wt_clock(session); + time_diff = WT_CLOCKDIFF_US(time_stop, time_start); if (F_ISSET(session, WT_SESSION_INTERNAL)) stats[session->stat_bucket][l->stat_int_usecs_off] += - (int64_t)WT_CLOCKDIFF_US(time_stop, time_start); - else + (int64_t)time_diff; + else { stats[session->stat_bucket][l->stat_app_usecs_off] += - (int64_t)WT_CLOCKDIFF_US(time_stop, time_start); + (int64_t)time_diff; + } + session_stats[l->stat_session_usecs_off] += (int64_t)time_diff; } /* @@ -371,12 +376,13 @@ void __wt_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *l) { WT_RWLOCK new, old; - uint64_t time_start, time_stop; - int64_t **stats; + uint64_t time_diff, time_start, time_stop; + int64_t *session_stats, **stats; uint8_t ticket; int pause_cnt; bool set_stats; + session_stats = NULL; /* -Wconditional-uninitialized */ stats = NULL; /* -Wconditional-uninitialized */ time_start = time_stop = 0; /* -Wconditional-uninitialized */ @@ -407,6 +413,7 @@ __wt_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *l) if (set_stats) { stats = (int64_t **)S2C(session)->stats; stats[session->stat_bucket][l->stat_write_count_off]++; + session_stats = (int64_t *)&(session->stats); time_start = __wt_clock(session); } /* @@ -433,12 +440,14 @@ __wt_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *l) } if (set_stats) { time_stop = __wt_clock(session); + time_diff = WT_CLOCKDIFF_US(time_stop, time_start); if (F_ISSET(session, WT_SESSION_INTERNAL)) stats[session->stat_bucket][l->stat_int_usecs_off] += - (int64_t)WT_CLOCKDIFF_US(time_stop, time_start); + (int64_t)time_diff; else stats[session->stat_bucket][l->stat_app_usecs_off] += - (int64_t)WT_CLOCKDIFF_US(time_stop, time_start); + (int64_t)time_diff; + session_stats[l->stat_session_usecs_off] += (int64_t)time_diff; } /* diff --git a/src/third_party/wiredtiger/src/support/scratch.c b/src/third_party/wiredtiger/src/support/scratch.c index a0f7de3179f..fda96c8efe2 100644 --- a/src/third_party/wiredtiger/src/support/scratch.c +++ b/src/third_party/wiredtiger/src/support/scratch.c @@ -128,7 +128,7 @@ __wt_buf_set_printable_format(WT_SESSION_IMPL *session, WT_DECL_RET; WT_PACK pack; const uint8_t *p, *end; - const char *retp, *sep; + const char *sep; p = (const uint8_t *)buffer; end = p + size; @@ -188,9 +188,13 @@ err: __wt_scr_free(session, &tmp); if (ret == 0) return ((const char *)buf->data); - retp = "failed to create printable output"; - __wt_err(session, ret, "%s", retp); - return (retp); + /* + * The byte string may not match the format (it happens if a formatted, + * internal row-store key is truncated, and then passed here by a page + * debugging routine). Our current callers aren't interested in error + * handling in such cases, return a byte string instead. + */ + return (__wt_buf_set_printable(session, buffer, size, buf)); } /* diff --git a/src/third_party/wiredtiger/src/support/stat.c b/src/third_party/wiredtiger/src/support/stat.c index a2019cd3aac..cee1f270d6d 100644 --- a/src/third_party/wiredtiger/src/support/stat.c +++ b/src/third_party/wiredtiger/src/support/stat.c @@ -107,9 +107,6 @@ static const char * const __stats_dsrc_desc[] = { "compression: compressed pages written", "compression: page written failed to compress", "compression: page written was too small to compress", - "compression: raw compression call failed, additional data available", - "compression: raw compression call failed, no additional data available", - "compression: raw compression call succeeded", "cursor: bulk-loaded cursor-insert calls", "cursor: close calls that result in cache", "cursor: create calls", @@ -121,6 +118,7 @@ static const char * const __stats_dsrc_desc[] = { "cursor: insert calls", "cursor: modify calls", "cursor: next calls", + "cursor: open cursor count", "cursor: prev calls", "cursor: remove calls", "cursor: reserve calls", @@ -143,9 +141,7 @@ static const char * const __stats_dsrc_desc[] = { "reconciliation: page reconciliation calls", "reconciliation: page reconciliation calls for eviction", "reconciliation: pages deleted", - "session: cached cursor count", "session: object compaction", - "session: open cursor count", "transaction: update conflicts", }; @@ -293,9 +289,6 @@ __wt_stat_dsrc_clear_single(WT_DSRC_STATS *stats) stats->compress_write = 0; stats->compress_write_fail = 0; stats->compress_write_too_small = 0; - stats->compress_raw_fail_temporary = 0; - stats->compress_raw_fail = 0; - stats->compress_raw_ok = 0; stats->cursor_insert_bulk = 0; stats->cursor_cache = 0; stats->cursor_create = 0; @@ -307,6 +300,7 @@ __wt_stat_dsrc_clear_single(WT_DSRC_STATS *stats) stats->cursor_insert = 0; stats->cursor_modify = 0; stats->cursor_next = 0; + /* not clearing cursor_open_count */ stats->cursor_prev = 0; stats->cursor_remove = 0; stats->cursor_reserve = 0; @@ -329,9 +323,7 @@ __wt_stat_dsrc_clear_single(WT_DSRC_STATS *stats) stats->rec_pages = 0; stats->rec_pages_eviction = 0; stats->rec_page_delete = 0; - /* not clearing session_cursors_cached */ stats->session_compact = 0; - /* not clearing session_cursor_open */ stats->txn_update_conflict = 0; } @@ -480,9 +472,6 @@ __wt_stat_dsrc_aggregate_single( to->compress_write += from->compress_write; to->compress_write_fail += from->compress_write_fail; to->compress_write_too_small += from->compress_write_too_small; - to->compress_raw_fail_temporary += from->compress_raw_fail_temporary; - to->compress_raw_fail += from->compress_raw_fail; - to->compress_raw_ok += from->compress_raw_ok; to->cursor_insert_bulk += from->cursor_insert_bulk; to->cursor_cache += from->cursor_cache; to->cursor_create += from->cursor_create; @@ -494,6 +483,7 @@ __wt_stat_dsrc_aggregate_single( to->cursor_insert += from->cursor_insert; to->cursor_modify += from->cursor_modify; to->cursor_next += from->cursor_next; + to->cursor_open_count += from->cursor_open_count; to->cursor_prev += from->cursor_prev; to->cursor_remove += from->cursor_remove; to->cursor_reserve += from->cursor_reserve; @@ -517,9 +507,7 @@ __wt_stat_dsrc_aggregate_single( to->rec_pages += from->rec_pages; to->rec_pages_eviction += from->rec_pages_eviction; to->rec_page_delete += from->rec_page_delete; - to->session_cursors_cached += from->session_cursors_cached; to->session_compact += from->session_compact; - to->session_cursor_open += from->session_cursor_open; to->txn_update_conflict += from->txn_update_conflict; } @@ -700,10 +688,6 @@ __wt_stat_dsrc_aggregate( to->compress_write_fail += WT_STAT_READ(from, compress_write_fail); to->compress_write_too_small += WT_STAT_READ(from, compress_write_too_small); - to->compress_raw_fail_temporary += - WT_STAT_READ(from, compress_raw_fail_temporary); - to->compress_raw_fail += WT_STAT_READ(from, compress_raw_fail); - to->compress_raw_ok += WT_STAT_READ(from, compress_raw_ok); to->cursor_insert_bulk += WT_STAT_READ(from, cursor_insert_bulk); to->cursor_cache += WT_STAT_READ(from, cursor_cache); to->cursor_create += WT_STAT_READ(from, cursor_create); @@ -715,6 +699,7 @@ __wt_stat_dsrc_aggregate( to->cursor_insert += WT_STAT_READ(from, cursor_insert); to->cursor_modify += WT_STAT_READ(from, cursor_modify); to->cursor_next += WT_STAT_READ(from, cursor_next); + to->cursor_open_count += WT_STAT_READ(from, cursor_open_count); to->cursor_prev += WT_STAT_READ(from, cursor_prev); to->cursor_remove += WT_STAT_READ(from, cursor_remove); to->cursor_reserve += WT_STAT_READ(from, cursor_reserve); @@ -744,10 +729,7 @@ __wt_stat_dsrc_aggregate( to->rec_pages += WT_STAT_READ(from, rec_pages); to->rec_pages_eviction += WT_STAT_READ(from, rec_pages_eviction); to->rec_page_delete += WT_STAT_READ(from, rec_page_delete); - to->session_cursors_cached += - WT_STAT_READ(from, session_cursors_cached); to->session_compact += WT_STAT_READ(from, session_compact); - to->session_cursor_open += WT_STAT_READ(from, session_cursor_open); to->txn_update_conflict += WT_STAT_READ(from, txn_update_conflict); } @@ -894,6 +876,7 @@ static const char * const __stats_connection_desc[] = { "connection: total fsync I/Os", "connection: total read I/Os", "connection: total write I/Os", + "cursor: cached cursor count", "cursor: cursor close calls that result in cache", "cursor: cursor create calls", "cursor: cursor insert calls", @@ -911,8 +894,8 @@ static const char * const __stats_connection_desc[] = { "cursor: cursor sweep cursors examined", "cursor: cursor sweeps", "cursor: cursor update calls", - "cursor: cursors currently cached", "cursor: cursors reused from cache", + "cursor: open cursor count", "cursor: truncate calls", "data-handle: connection data handles currently active", "data-handle: connection sweep candidate became referenced", @@ -1025,7 +1008,6 @@ static const char * const __stats_connection_desc[] = { "reconciliation: pages deleted", "reconciliation: split bytes currently awaiting free", "reconciliation: split objects currently awaiting free", - "session: open cursor count", "session: open session count", "session: session query timestamp calls", "session: table alter failed calls", @@ -1303,6 +1285,7 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->fsync_io = 0; stats->read_io = 0; stats->write_io = 0; + /* not clearing cursor_cached_count */ stats->cursor_cache = 0; stats->cursor_create = 0; stats->cursor_insert = 0; @@ -1320,8 +1303,8 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->cursor_sweep_examined = 0; stats->cursor_sweep = 0; stats->cursor_update = 0; - /* not clearing cursors_cached */ stats->cursor_reopen = 0; + /* not clearing cursor_open_count */ stats->cursor_truncate = 0; /* not clearing dh_conn_handle_count */ stats->dh_sweep_ref = 0; @@ -1434,7 +1417,6 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->rec_page_delete = 0; /* not clearing rec_split_stashed_bytes */ /* not clearing rec_split_stashed_objects */ - /* not clearing session_cursor_open */ /* not clearing session_open */ stats->session_query_ts = 0; /* not clearing session_table_alter_fail */ @@ -1758,6 +1740,7 @@ __wt_stat_connection_aggregate( to->fsync_io += WT_STAT_READ(from, fsync_io); to->read_io += WT_STAT_READ(from, read_io); to->write_io += WT_STAT_READ(from, write_io); + to->cursor_cached_count += WT_STAT_READ(from, cursor_cached_count); to->cursor_cache += WT_STAT_READ(from, cursor_cache); to->cursor_create += WT_STAT_READ(from, cursor_create); to->cursor_insert += WT_STAT_READ(from, cursor_insert); @@ -1776,8 +1759,8 @@ __wt_stat_connection_aggregate( WT_STAT_READ(from, cursor_sweep_examined); to->cursor_sweep += WT_STAT_READ(from, cursor_sweep); to->cursor_update += WT_STAT_READ(from, cursor_update); - to->cursors_cached += WT_STAT_READ(from, cursors_cached); to->cursor_reopen += WT_STAT_READ(from, cursor_reopen); + to->cursor_open_count += WT_STAT_READ(from, cursor_open_count); to->cursor_truncate += WT_STAT_READ(from, cursor_truncate); to->dh_conn_handle_count += WT_STAT_READ(from, dh_conn_handle_count); to->dh_sweep_ref += WT_STAT_READ(from, dh_sweep_ref); @@ -1949,7 +1932,6 @@ __wt_stat_connection_aggregate( WT_STAT_READ(from, rec_split_stashed_bytes); to->rec_split_stashed_objects += WT_STAT_READ(from, rec_split_stashed_objects); - to->session_cursor_open += WT_STAT_READ(from, session_cursor_open); to->session_open += WT_STAT_READ(from, session_open); to->session_query_ts += WT_STAT_READ(from, session_query_ts); to->session_table_alter_fail += diff --git a/src/third_party/wiredtiger/src/txn/txn.c b/src/third_party/wiredtiger/src/txn/txn.c index 20e469d0262..4ddc75afe6c 100644 --- a/src/third_party/wiredtiger/src/txn/txn.c +++ b/src/third_party/wiredtiger/src/txn/txn.c @@ -992,9 +992,8 @@ __wt_txn_prepare(WT_SESSION_IMPL *session, const char *cfg[]) WT_STAT_CONN_INCR(session, txn_prepared_updates_count); break; case WT_TXN_OP_REF_DELETE: - op->u.ref->page_del->timestamp = ts; - WT_PUBLISH(op->u.ref->page_del->prepare_state, - WT_PREPARE_INPROGRESS); + __wt_txn_op_apply_prepare_state( + session, op->u.ref, false); break; case WT_TXN_OP_TRUNCATE_COL: case WT_TXN_OP_TRUNCATE_ROW: diff --git a/src/third_party/wiredtiger/src/utilities/util_list.c b/src/third_party/wiredtiger/src/utilities/util_list.c index 54e3cd61f8b..72ae405ff87 100644 --- a/src/third_party/wiredtiger/src/utilities/util_list.c +++ b/src/third_party/wiredtiger/src/utilities/util_list.c @@ -69,6 +69,8 @@ list_get_allocsize(WT_SESSION *session, const char *key, size_t *allocsize) int tret; char *config; + *allocsize = 0; + wt_api = session->connection->get_extension_api(session->connection); if ((ret = wt_api->metadata_search(wt_api, session, key, &config)) != 0) return (util_err( @@ -78,10 +80,9 @@ list_get_allocsize(WT_SESSION *session, const char *key, size_t *allocsize) return (util_err( session, ret, "WT_EXTENSION_API.config_parser_open")); if ((ret = parser->get(parser, "allocation_size", &szvalue)) != 0) { - if (ret == WT_NOTFOUND) { - *allocsize = 0; + if (ret == WT_NOTFOUND) ret = 0; - } else + else ret = util_err(session, ret, "WT_CONFIG_PARSER.get"); if ((tret = parser->close(parser)) != 0) (void)util_err(session, tret, "WT_CONFIG_PARSER.close"); diff --git a/src/third_party/wiredtiger/test/bloom/Makefile.am b/src/third_party/wiredtiger/test/bloom/Makefile.am index 81a21f59882..6bba3a5e33f 100644 --- a/src/third_party/wiredtiger/test/bloom/Makefile.am +++ b/src/third_party/wiredtiger/test/bloom/Makefile.am @@ -14,4 +14,4 @@ TESTS = $(noinst_PROGRAMS) LOG_COMPILER = $(TEST_WRAPPER) clean-local: - rm -rf WiredTiger* *.core + rm -rf WiredTiger* core.* *.core diff --git a/src/third_party/wiredtiger/test/checkpoint/Makefile.am b/src/third_party/wiredtiger/test/checkpoint/Makefile.am index 2b5ba800c9c..da7b85ec0b2 100644 --- a/src/third_party/wiredtiger/test/checkpoint/Makefile.am +++ b/src/third_party/wiredtiger/test/checkpoint/Makefile.am @@ -12,4 +12,4 @@ t_LDFLAGS = -static TESTS = smoke.sh clean-local: - rm -rf WT_TEST *.core + rm -rf WT_TEST core.* *.core diff --git a/src/third_party/wiredtiger/test/checkpoint/smoke.sh b/src/third_party/wiredtiger/test/checkpoint/smoke.sh index 39b1f428c2c..2f1d4345ad7 100755 --- a/src/third_party/wiredtiger/test/checkpoint/smoke.sh +++ b/src/third_party/wiredtiger/test/checkpoint/smoke.sh @@ -6,9 +6,6 @@ set -e echo "checkpoint: 3 mixed tables" $TEST_WRAPPER ./t -T 3 -t m -# We are done unless long tests are enabled. -test "$TESTUTIL_ENABLE_LONG_TESTS" = "1" || exit 0 - echo "checkpoint: 6 column-store tables" $TEST_WRAPPER ./t -T 6 -t c diff --git a/src/third_party/wiredtiger/test/csuite/Makefile.am b/src/third_party/wiredtiger/test/csuite/Makefile.am index 6d6e893beb8..362d0775a88 100644 --- a/src/third_party/wiredtiger/test/csuite/Makefile.am +++ b/src/third_party/wiredtiger/test/csuite/Makefile.am @@ -132,4 +132,4 @@ TESTS = $(all_TESTS) LOG_COMPILER = $(TEST_WRAPPER) clean-local: - rm -rf WT_TEST.* *.core + rm -rf WT_TEST.* core.* *.core diff --git a/src/third_party/wiredtiger/test/csuite/README b/src/third_party/wiredtiger/test/csuite/README new file mode 100644 index 00000000000..c1c42372ec3 --- /dev/null +++ b/src/third_party/wiredtiger/test/csuite/README @@ -0,0 +1,12 @@ +The 'test/csuite' directory includes a collection of sanity tests written in C. +They are expected to be executed as part of 'make check' testing. + +Each sub directory of 'test/csuite' include source code of one csuite test, and +should have one corresponding task in the Evergreen (CI system) configuration file. +See 'test/evergreen.yml' for details. + +When a new csuite test is introduced, a corresponding new Evergreen task should be +crafted and put into the Evergreen configuration file. We have a utility program +'test/evergreen/evg_cfg.py' to help with identifying and auto-generating the +Evergreen configuration for new or missing csuite tests. The program checking +has been bound into developer workflow through 'dist/s_evergreen'. diff --git a/src/third_party/wiredtiger/test/csuite/random_abort/main.c b/src/third_party/wiredtiger/test/csuite/random_abort/main.c index e2f2820a63a..e4c50cedddd 100644 --- a/src/third_party/wiredtiger/test/csuite/random_abort/main.c +++ b/src/third_party/wiredtiger/test/csuite/random_abort/main.c @@ -348,7 +348,7 @@ main(int argc, char *argv[]) testutil_check(__wt_snprintf( buf, sizeof(buf),"%s/%s", home, fname)); while (stat(buf, &sb) != 0) - sleep(1); + testutil_sleep_wait(1, pid); ++i; } sleep(timeout); diff --git a/src/third_party/wiredtiger/test/csuite/rwlock/main.c b/src/third_party/wiredtiger/test/csuite/rwlock/main.c index f69628dca40..45ff418c7cc 100644 --- a/src/third_party/wiredtiger/test/csuite/rwlock/main.c +++ b/src/third_party/wiredtiger/test/csuite/rwlock/main.c @@ -55,10 +55,6 @@ main(int argc, char *argv[]) pthread_t dump_id, id[MAX_THREADS]; int i; - /* Ignore unless requested */ - if (!testutil_is_flag_set("TESTUTIL_ENABLE_LONG_TESTS")) - return (EXIT_SUCCESS); - opts = &_opts; memset(opts, 0, sizeof(*opts)); opts->nthreads = 100; @@ -110,7 +106,8 @@ thread_rwlock(void *arg) opts->conn->open_session(opts->conn, NULL, NULL, &wt_session)); session = (WT_SESSION_IMPL *)wt_session; - printf("Running rwlock thread\n"); + if (opts->verbose) + printf("Running rwlock thread\n"); for (i = 1; i <= opts->nops; ++i) { writelock = (i % READS_PER_WRITE == 0); @@ -152,7 +149,7 @@ thread_rwlock(void *arg) __wt_readunlock(session, &rwlock); #endif - if (i % 10000 == 0) { + if (opts->verbose && i % 10000 == 0) { printf("%s", session->id == 20 ? ".\n" : "."); fflush(stdout); } @@ -164,20 +161,24 @@ thread_rwlock(void *arg) } void * -thread_dump(void *arg) { - WT_UNUSED(arg); +thread_dump(void *arg) +{ + TEST_OPTS *opts; + + opts = arg; while (running) { sleep(1); - printf("\n" - "rwlock { current %" PRIu8 ", next %" PRIu8 - ", reader %" PRIu8 ", readers_active %" PRIu32 - ", readers_queued %" PRIu8 " }\n", - rwlock.u.s.current, - rwlock.u.s.next, - rwlock.u.s.reader, - rwlock.u.s.readers_active, - rwlock.u.s.readers_queued); + if (opts->verbose) + printf("\n" + "rwlock { current %" PRIu8 ", next %" PRIu8 + ", reader %" PRIu8 ", readers_active %" PRIu32 + ", readers_queued %" PRIu8 " }\n", + rwlock.u.s.current, + rwlock.u.s.next, + rwlock.u.s.reader, + rwlock.u.s.readers_active, + rwlock.u.s.readers_queued); } return (NULL); diff --git a/src/third_party/wiredtiger/test/csuite/schema_abort/main.c b/src/third_party/wiredtiger/test/csuite/schema_abort/main.c index 63e457fe705..81cf528dded 100644 --- a/src/third_party/wiredtiger/test/csuite/schema_abort/main.c +++ b/src/third_party/wiredtiger/test/csuite/schema_abort/main.c @@ -1096,7 +1096,7 @@ main(int argc, char *argv[]) testutil_check(__wt_snprintf( statname, sizeof(statname), "%s/%s", home, ckpt_file)); while (stat(statname, &sb) != 0) - sleep(1); + testutil_sleep_wait(1, pid); sleep(timeout); sa.sa_handler = SIG_DFL; testutil_checksys(sigaction(SIGCHLD, &sa, NULL)); diff --git a/src/third_party/wiredtiger/test/csuite/time_shift_test.sh b/src/third_party/wiredtiger/test/csuite/time_shift_test.sh index ae06fd03f36..9c14ae86ae7 100755 --- a/src/third_party/wiredtiger/test/csuite/time_shift_test.sh +++ b/src/third_party/wiredtiger/test/csuite/time_shift_test.sh @@ -11,13 +11,6 @@ set -e # assumption that other factors of the environment will influence the execution # time by less than 20%. - -# need to enable long tests to run test_rwlock -export TESTUTIL_ENABLE_LONG_TESTS=1 - -# We will run only when long tests are enabled. -test "$TESTUTIL_ENABLE_LONG_TESTS" = "1" || exit 0 - EXIT_SUCCESS=0 EXIT_FAILURE=1 diff --git a/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c b/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c index 837b3f400b2..765c49ad2ce 100644 --- a/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c +++ b/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c @@ -676,7 +676,7 @@ main(int argc, char *argv[]) testutil_check(__wt_snprintf( statname, sizeof(statname), "%s/%s", home, ckpt_file)); while (stat(statname, &sb) != 0) - sleep(1); + testutil_sleep_wait(1, pid); sleep(timeout); sa.sa_handler = SIG_DFL; testutil_checksys(sigaction(SIGCHLD, &sa, NULL)); diff --git a/src/third_party/wiredtiger/test/csuite/wt2246_col_append/main.c b/src/third_party/wiredtiger/test/csuite/wt2246_col_append/main.c index fca3d8fcd00..2757f991c2a 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2246_col_append/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2246_col_append/main.c @@ -79,9 +79,6 @@ page_init(uint64_t n) } } -/* - * TODO: Platform specific? - */ static void onsig(int signo) { @@ -101,10 +98,6 @@ main(int argc, char *argv[]) uint64_t i, id; char buf[100]; - /* Ignore unless requested */ - if (!testutil_is_flag_set("TESTUTIL_ENABLE_LONG_TESTS")) - return (EXIT_SUCCESS); - opts = &_opts; memset(opts, 0, sizeof(*opts)); opts->table_type = TABLE_ROW; diff --git a/src/third_party/wiredtiger/test/csuite/wt2323_join_visibility/main.c b/src/third_party/wiredtiger/test/csuite/wt2323_join_visibility/main.c index 0de304e61bd..0b99df76cf3 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2323_join_visibility/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2323_join_visibility/main.c @@ -92,10 +92,6 @@ main(int argc, char *argv[]) TEST_OPTS *opts, _opts; const char *tablename; - /* Ignore unless requested */ - if (!testutil_is_flag_set("TESTUTIL_ENABLE_LONG_TESTS")) - return (EXIT_SUCCESS); - opts = &_opts; sharedopts = &_sharedopts; memset(opts, 0, sizeof(*opts)); @@ -318,7 +314,8 @@ thread_insert(void *arg) return (NULL); } -static void *thread_join(void *arg) +static void * +thread_join(void *arg) { SHARED_OPTS *sharedopts; TEST_OPTS *opts; diff --git a/src/third_party/wiredtiger/test/csuite/wt2535_insert_race/main.c b/src/third_party/wiredtiger/test/csuite/wt2535_insert_race/main.c index 966258163f6..19620cec894 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2535_insert_race/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2535_insert_race/main.c @@ -38,6 +38,8 @@ void *thread_insert_race(void *); +static uint64_t ready_counter; + int main(int argc, char *argv[]) { @@ -49,14 +51,10 @@ main(int argc, char *argv[]) uint64_t current_value; int i; - /* Ignore unless requested */ - if (!testutil_is_flag_set("TESTUTIL_ENABLE_LONG_TESTS")) - return (EXIT_SUCCESS); - opts = &_opts; memset(opts, 0, sizeof(*opts)); - opts->nthreads = 10; - opts->nrecords = 1000; + opts->nthreads = 20; + opts->nrecords = 100000; opts->table_type = TABLE_ROW; testutil_check(testutil_parse_opts(argc, argv, opts)); testutil_make_work_dir(opts->home); @@ -119,16 +117,25 @@ thread_insert_race(void *arg) WT_CURSOR *cursor; WT_DECL_RET; WT_SESSION *session; - uint64_t i, value; + uint64_t i, value, ready_counter_local; opts = (TEST_OPTS *)arg; conn = opts->conn; + printf("Running insert thread\n"); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); testutil_check(session->open_cursor( session, opts->uri, NULL, NULL, &cursor)); - printf("Running insert thread\n"); + /* Wait until all the threads are ready to go. */ + (void)__wt_atomic_add64(&ready_counter, 1); + for (;; __wt_yield()) { + WT_ORDERED_READ(ready_counter_local, ready_counter); + if (ready_counter_local >= opts->nthreads) + break; + } + for (i = 0; i < opts->nrecords; ++i) { testutil_check( session->begin_transaction(session, "isolation=snapshot")); diff --git a/src/third_party/wiredtiger/test/csuite/wt2834_join_bloom_fix/main.c b/src/third_party/wiredtiger/test/csuite/wt2834_join_bloom_fix/main.c index 58df56b50b1..a8d44bf3dab 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2834_join_bloom_fix/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2834_join_bloom_fix/main.c @@ -59,10 +59,6 @@ main(int argc, char *argv[]) char posturi[256]; const char *tablename; - /* Ignore unless requested */ - if (!testutil_is_flag_set("TESTUTIL_ENABLE_LONG_TESTS")) - return (EXIT_SUCCESS); - opts = &_opts; memset(opts, 0, sizeof(*opts)); testutil_check(testutil_parse_opts(argc, argv, opts)); diff --git a/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c b/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c index 93fb94836c0..80911ddfd2d 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c @@ -82,10 +82,6 @@ main(int argc, char *argv[]) int i, nfail; const char *tablename; - /* Ignore unless requested */ - if (!testutil_is_flag_set("TESTUTIL_ENABLE_LONG_TESTS")) - return (EXIT_SUCCESS); - opts = &_opts; sharedopts = &_sharedopts; memset(opts, 0, sizeof(*opts)); diff --git a/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c b/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c index 10c0571c9f6..2020b4dcecd 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c @@ -379,7 +379,7 @@ run_check_subtest_range(TEST_OPTS *opts, const char *debugger, bool close_test) low = 0; high = MAX_OP_RANGE; mid = (low + high) / 2; - while (mid != low) { + while (low < mid - 5 || high > mid + 5) { run_check_subtest(opts, debugger, mid, close_test, &nresults); if (nresults > cutoff) @@ -401,8 +401,9 @@ run_check_subtest_range(TEST_OPTS *opts, const char *debugger, bool close_test) got_failure = false; got_success = false; - for (nops = mid - 10; nops < mid + 10; nops++) { - for (i = 0; i < TESTS_PER_OP_VALUE; i++) { + for (i = 0; + i < TESTS_PER_OP_VALUE && (!got_failure || !got_success); i++) + for (nops = mid - 10; nops < mid + 10; nops++) { run_check_subtest(opts, debugger, nops, close_test, &nresults); if (nresults > cutoff) @@ -410,7 +411,7 @@ run_check_subtest_range(TEST_OPTS *opts, const char *debugger, bool close_test) else got_success = true; } - } + /* * Check that it really ran with a crossover point. */ @@ -647,10 +648,6 @@ main(int argc, char *argv[]) uint64_t nresults; const char *debugger; - /* Ignore unless requested */ - if (!testutil_is_flag_set("TESTUTIL_ENABLE_LONG_TESTS")) - return (EXIT_SUCCESS); - opts = &_opts; memset(opts, 0, sizeof(*opts)); debugger = NULL; diff --git a/src/third_party/wiredtiger/test/csuite/wt3338_partial_update/main.c b/src/third_party/wiredtiger/test/csuite/wt3338_partial_update/main.c index fb246f87da1..1d59fab8583 100644 --- a/src/third_party/wiredtiger/test/csuite/wt3338_partial_update/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt3338_partial_update/main.c @@ -32,13 +32,13 @@ * Test case description: Smoke-test the partial update construction. */ -#define DEBUG 0 +#define DEBUG 0 -#define DATASIZE 1024 +#define DATASIZE 1024 +#define MAX_MODIFY_ENTRIES 37 /* Maximum modify vectors */ -#define MAX_MODIFY_ENTRIES 37 -static WT_MODIFY entries[MAX_MODIFY_ENTRIES]; /* Entries vector */ -static int nentries; /* Entries count */ +static WT_MODIFY entries[1000]; /* Entries vector */ +static int nentries; /* Entries count */ /* * The replacement bytes array is 2x the maximum replacement string so we can @@ -213,11 +213,11 @@ slow_apply_api(WT_ITEM *orig) } /* - * diff -- - * Diff the two results. + * compare -- + * Compare two results. */ static void -diff(WT_ITEM *local, WT_ITEM *library) +compare(WT_ITEM *local, WT_ITEM *library) { #if DEBUG if (local->size != library->size || @@ -233,60 +233,108 @@ diff(WT_ITEM *local, WT_ITEM *library) } /* - * modify_init -- - * Initialize the buffers to a known state. + * modify_run + * Run some tests: + * 1. Create an initial value, a copy and a fake cursor to use with the + * WiredTiger routines. Generate a set of modify vectors and apply them to + * the item stored in the cursor using the modify apply API. Also apply the + * same modify vector to one of the copies using a helper routine written + * to test the modify API. The final value generated with the modify API + * and the helper routine should match. + * + * 2. Use the initial value and the modified value generated above as + * inputs into the calculate-modify API to generate a set of modify + * vectors. Apply this generated vector to the initial value using the + * modify apply API to obtain a final value. The final value generated + * should match the modified value that was used as input to the + * calculate-modify API. */ static void -modify_init(WT_ITEM *local, WT_ITEM *library) +modify_run(bool verbose) { + WT_CURSOR *cursor, _cursor; + WT_DECL_RET; + WT_ITEM *localA, _localA, *localB, _localB; size_t len; + int i, j; - len = (size_t)(__wt_random(&rnd) % MAX_REPL_BYTES); - testutil_check(__wt_buf_set(NULL, local, modify_repl, len)); - testutil_check(__wt_buf_set(NULL, library, modify_repl, len)); -} + /* Initialize the RNG. */ + __wt_random_init_seed(NULL, &rnd); -static int nruns = 1000; + /* Set up replacement information. */ + modify_repl_init(); -/* - * modify_run - * Run some tests. - */ -static void -modify_run(WT_CURSOR *cursor, WT_ITEM *local, bool verbose) -{ - int i, j; + /* We need three WT_ITEMs, one of them part of a fake cursor. */ + localA = &_localA; + memset(&_localA, 0, sizeof(_localA)); + localB = &_localB; + memset(&_localB, 0, sizeof(_localB)); + cursor = &_cursor; + memset(&_cursor, 0, sizeof(_cursor)); + cursor->value_format = "u"; - for (i = 0; i < nruns; ++i) { - modify_init(local, &cursor->value); +#define NRUNS 10000 + for (i = 0; i < NRUNS; ++i) { + /* Create an initial value. */ + len = (size_t)(__wt_random(&rnd) % MAX_REPL_BYTES); + testutil_check(__wt_buf_set(NULL, localA, modify_repl, len)); for (j = 0; j < 1000; ++j) { + /* Copy the current value into the second item. */ + testutil_check(__wt_buf_set( + NULL, localB, localA->data, localA->size)); + + /* + * Create a random set of modify vectors, run the + * underlying library modification function, then + * compare the result against our implementation + * of modify. + */ modify_build(); - - slow_apply_api(local); + testutil_check(__wt_buf_set( + NULL, &cursor->value, localA->data, localA->size)); testutil_check(__wt_modify_apply_api( NULL, cursor, entries, nentries)); - - diff(local, &cursor->value); + slow_apply_api(localA); + compare(localA, &cursor->value); + + /* + * Call the WiredTiger function to build a modification + * vector for the change, and repeat the test using the + * WiredTiger modification vector, then compare results + * against our implementation of modify. + */ + nentries = WT_ELEMENTS(entries); + ret = wiredtiger_calc_modify(NULL, + localB, localA, + WT_MAX(localB->size, localA->size) + 100, + entries, &nentries); + if (ret == WT_NOTFOUND) + continue; + testutil_check(ret); + testutil_check(__wt_buf_set( + NULL, &cursor->value, localB->data, localB->size)); + testutil_check(__wt_modify_apply_api( + NULL, cursor, entries, nentries)); + compare(localA, &cursor->value); } if (verbose) { - printf("%d (%d%%)\r", i, (i * 100) / nruns); + printf("%d (%d%%)\r", i, (i * 100) / NRUNS); fflush(stdout); } } if (verbose) - printf("\n"); + printf("%d (100%%)\n", i); + + __wt_buf_free(NULL, localA); + __wt_buf_free(NULL, localB); + __wt_buf_free(NULL, &cursor->value); } int main(int argc, char *argv[]) { TEST_OPTS *opts, _opts; - WT_CURSOR *cursor, _cursor; - WT_ITEM *local, _local; - - if (testutil_is_flag_set("TESTUTIL_ENABLE_LONG_TESTS")) - nruns = 10000; opts = &_opts; memset(opts, 0, sizeof(*opts)); @@ -295,24 +343,8 @@ main(int argc, char *argv[]) testutil_check( wiredtiger_open(opts->home, NULL, "create", &opts->conn)); - /* Initialize the RNG. */ - __wt_random_init_seed(NULL, &rnd); - - /* Set up replacement information. */ - modify_repl_init(); - - /* We need two items, one of them hooked into fake cursor. */ - local = &_local; - memset(&_local, 0, sizeof(_local)); - cursor = &_cursor; - memset(&_cursor, 0, sizeof(_cursor)); - cursor->value_format = "u"; - /* Run the test. */ - modify_run(cursor, local, opts->verbose); - - __wt_buf_free(NULL, local); - __wt_buf_free(NULL, &cursor->value); + modify_run(opts->verbose); testutil_cleanup(opts); return (EXIT_SUCCESS); diff --git a/src/third_party/wiredtiger/test/csuite/wt4156_metadata_salvage/main.c b/src/third_party/wiredtiger/test/csuite/wt4156_metadata_salvage/main.c index b813a50c458..53fdfe16bd6 100644 --- a/src/third_party/wiredtiger/test/csuite/wt4156_metadata_salvage/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt4156_metadata_salvage/main.c @@ -424,9 +424,6 @@ static void wt_open_corrupt(const char *) static void wt_open_corrupt(const char *sfx) { -#ifdef HAVE_ATTACH - WT_UNUSED(sfx); -#else WT_CONNECTION *conn; WT_DECL_RET; char buf[1024]; @@ -446,7 +443,6 @@ wt_open_corrupt(const char *sfx) fprintf(stderr, "OPEN_CORRUPT: wiredtiger_open returned %d\n", ret); testutil_assert(ret == WT_TRY_SALVAGE || ret == 0); -#endif exit (EXIT_SUCCESS); } diff --git a/src/third_party/wiredtiger/test/csuite/wt4333_handle_locks/main.c b/src/third_party/wiredtiger/test/csuite/wt4333_handle_locks/main.c index 6ddad5c9063..25dd9e515db 100644 --- a/src/third_party/wiredtiger/test/csuite/wt4333_handle_locks/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt4333_handle_locks/main.c @@ -315,62 +315,51 @@ main(int argc, char *argv[]) static const struct { u_int workers; u_int uris; + bool cache_cursors; } runs[] = { - { 1, 1}, - { 8, 1}, - { 16, 1}, - { 16, WT_ELEMENTS(uri_list)}, - {200, 100}, - {300, 100}, - {200, WT_ELEMENTS(uri_list)}, - {600, WT_ELEMENTS(uri_list)}, + { 1, 1, false}, + { 1, 1, true}, + { 8, 1, false}, + { 8, 1, true}, + { 16, 1, false}, + { 16, 1, true}, + { 16, WT_ELEMENTS(uri_list), false}, + { 16, WT_ELEMENTS(uri_list), true}, + {200, 100, false}, + {200, 100, true}, + {200, WT_ELEMENTS(uri_list), false}, + {200, WT_ELEMENTS(uri_list), true}, + {300, 100, false}, + {300, 100, true}, + {600, WT_ELEMENTS(uri_list), false}, + {600, WT_ELEMENTS(uri_list), true}, }; + WT_RAND_STATE rnd; u_int i, n; int ch; - bool run_long; (void)testutil_set_progname(argv); + __wt_random_init_seed(NULL, &rnd); - run_long = false; - while ((ch = __wt_getopt(argv[0], argc, argv, "av")) != EOF) { + while ((ch = __wt_getopt(argv[0], argc, argv, "v")) != EOF) { switch (ch) { - case 'a': - run_long = true; - break; case 'v': verbose = true; break; default: - fprintf(stderr, "usage: %s -a", argv[0]); + fprintf(stderr, "usage: %s [-v]\n", argv[0]); return (EXIT_FAILURE); } } - /* Ignore unless requested */ - if (!run_long && - !testutil_is_flag_set("TESTUTIL_ENABLE_LONG_TESTS")) - return (EXIT_SUCCESS); - (void)signal(SIGALRM, on_alarm); - /* - * This test takes 2 minutes per slot in the runs table, only do the - * first 2 and last 2 slots, unless specifically requested. - */ - n = WT_ELEMENTS(runs); - for (i = 0; i < 2; ++i) { - workers = runs[i].workers; - uris = runs[i].uris; - run(true); - run(false); - } - if (!run_long) - i = n - 2; - for (; i < n; ++i) { - workers = runs[i].workers; - uris = runs[i].uris; - run(true); - run(false); + /* Each test in the table runs for a minute, run 5 tests at random. */ + for (i = 0; i < 5; ++i) { + n = __wt_random(&rnd) % WT_ELEMENTS(runs); + workers = runs[n].workers; + uris = runs[n].uris; + run(runs[n].cache_cursors); } uri_teardown(); diff --git a/src/third_party/wiredtiger/test/cursor_order/Makefile.am b/src/third_party/wiredtiger/test/cursor_order/Makefile.am index c98cf1fa047..f5330b57d3b 100644 --- a/src/third_party/wiredtiger/test/cursor_order/Makefile.am +++ b/src/third_party/wiredtiger/test/cursor_order/Makefile.am @@ -12,4 +12,4 @@ cursor_order_LDFLAGS = -static TESTS = $(noinst_PROGRAMS) clean-local: - rm -rf WT_TEST *.core + rm -rf WT_TEST core.* *.core diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml index 481fccddba9..be111c6432c 100644 --- a/src/third_party/wiredtiger/test/evergreen.yml +++ b/src/third_party/wiredtiger/test/evergreen.yml @@ -97,7 +97,7 @@ tasks: set -o errexit set -o verbose - ${test_env_vars|} TESTUTIL_ENABLE_LONG_TESTS=1 ${make_command|make} VERBOSE=1 check ${smp_command|} 2>&1 + ${test_env_vars|} ${make_command|make} VERBOSE=1 check ${smp_command|} 2>&1 # Start of normal make check test tasks @@ -621,7 +621,7 @@ tasks: set -o errexit set -o verbose - ${test_env_vars|} TESTUTIL_ENABLE_LONG_TESTS=1 $(pwd)/test_rwlock 2>&1 + ${test_env_vars|} $(pwd)/test_rwlock 2>&1 - name: csuite-wt2246-col-append-test depends_on: @@ -635,7 +635,7 @@ tasks: set -o errexit set -o verbose - ${test_env_vars|} TESTUTIL_ENABLE_LONG_TESTS=1 $(pwd)/test_wt2246_col_append 2>&1 + ${test_env_vars|} $(pwd)/test_wt2246_col_append 2>&1 - name: csuite-wt2323-join-visibility-test depends_on: @@ -649,7 +649,7 @@ tasks: set -o errexit set -o verbose - ${test_env_vars|} TESTUTIL_ENABLE_LONG_TESTS=1 $(pwd)/test_wt2323_join_visibility 2>&1 + ${test_env_vars|} $(pwd)/test_wt2323_join_visibility 2>&1 - name: csuite-wt2535-insert-race-test depends_on: @@ -663,7 +663,7 @@ tasks: set -o errexit set -o verbose - ${test_env_vars|} TESTUTIL_ENABLE_LONG_TESTS=1 $(pwd)/test_wt2535_insert_race 2>&1 + ${test_env_vars|} $(pwd)/test_wt2535_insert_race 2>&1 - name: csuite-wt2834-join-bloom-fix-test depends_on: @@ -677,7 +677,7 @@ tasks: set -o errexit set -o verbose - ${test_env_vars|} TESTUTIL_ENABLE_LONG_TESTS=1 $(pwd)/test_wt2834_join_bloom_fix 2>&1 + ${test_env_vars|} $(pwd)/test_wt2834_join_bloom_fix 2>&1 - name: csuite-wt2853-perf-test depends_on: @@ -691,7 +691,7 @@ tasks: set -o errexit set -o verbose - ${test_env_vars|} TESTUTIL_ENABLE_LONG_TESTS=1 $(pwd)/test_wt2853_perf 2>&1 + ${test_env_vars|} $(pwd)/test_wt2853_perf 2>&1 - name: csuite-wt2909-checkpoint-integrity-test depends_on: @@ -705,7 +705,7 @@ tasks: set -o errexit set -o verbose - ${test_env_vars|} TESTUTIL_ENABLE_LONG_TESTS=1 $(pwd)/test_wt2909_checkpoint_integrity 2>&1 + ${test_env_vars|} $(pwd)/test_wt2909_checkpoint_integrity 2>&1 - name: csuite-wt3338-partial-update-test depends_on: @@ -719,7 +719,7 @@ tasks: set -o errexit set -o verbose - ${test_env_vars|} TESTUTIL_ENABLE_LONG_TESTS=1 $(pwd)/test_wt3338_partial_update 2>&1 + ${test_env_vars|} $(pwd)/test_wt3338_partial_update 2>&1 - name: csuite-wt4333-handle-locks-test depends_on: @@ -733,10 +733,12 @@ tasks: set -o errexit set -o verbose - ${test_env_vars|} TESTUTIL_ENABLE_LONG_TESTS=1 $(pwd)/test_wt4333_handle_locks 2>&1 + ${test_env_vars|} $(pwd)/test_wt4333_handle_locks 2>&1 # End of csuite test tasks + # Start of Python unit test tasks + - name: unit-test depends_on: - name: compile @@ -749,21 +751,141 @@ tasks: set -o errexit set -o verbose - if [ "Windows_NT" = "$OS" ]; then - ${test_env_vars|} python ./test/suite/run.py -v 2 ${smp_command|} 2>&1 - elif [ "$(uname -s)" == "Darwin" ]; then - # Avoid /usr/bin/python, at least on macOS: with System Integrity - # Protection enabled, it ignores DYLD_LIBRARY_PATH and hence - # doesn't find the WiredTiger library in the local tree. - ${test_env_vars|} python ./test/suite/run.py -v 2 ${smp_command|} 2>&1 - else # Ubuntu - # Change directory to where the local installed 'wt' binary is located, - # to avoid libtool generated 'wt' script from being selected by run.py, - # which invokes relink_command that tries to changing to a non-existed - # /data/mci/<uniq> directory, as 'make' is done by a separate 'compile' task. - cd .libs - ${python_test_env_vars|} python ../test/suite/run.py -v 2 ${smp_command|} 2>&1 - fi + # Only Windows and OS X variants are expected to run this task + # + # Avoid /usr/bin/python, at least on macOS: with System Integrity + # Protection enabled, it ignores DYLD_LIBRARY_PATH and hence + # doesn't find the WiredTiger library in the local tree. + ${test_env_vars|} python ./test/suite/run.py -v 2 ${smp_command|} 2>&1 + + # Break out Python unit tests into multiple buckets/tasks based on test name and runtime + # The test/suite/run.py script can work out test names by casting each command argument + # with "test_" prefix and "*.py" postfix. + # + # One example: + # "test/suite/run.py [ab]" will be translated to testing "test_a*.py" and "test_b*.py" + + - name: unit-test-bucket00 + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - command: shell.exec + params: + working_dir: "wiredtiger" + script: | + set -o errexit + set -o verbose + + cd .libs + ${python_test_env_vars|} python ../test/suite/run.py [ab] -v 2 ${smp_command|} 2>&1 + + - name: unit-test-bucket01 + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - command: shell.exec + params: + working_dir: "wiredtiger" + script: | + set -o errexit + set -o verbose + + cd .libs + ${python_test_env_vars|} python ../test/suite/run.py [c] -v 2 ${smp_command|} 2>&1 + + - name: unit-test-bucket02 + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - command: shell.exec + params: + working_dir: "wiredtiger" + script: | + set -o errexit + set -o verbose + + cd .libs + ${python_test_env_vars|} python ../test/suite/run.py [defg] -v 2 ${smp_command|} 2>&1 + + - name: unit-test-bucket03 + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - command: shell.exec + params: + working_dir: "wiredtiger" + script: | + set -o errexit + set -o verbose + + cd .libs + ${python_test_env_vars|} python ../test/suite/run.py [hijk] -v 2 ${smp_command|} 2>&1 + + - name: unit-test-bucket04 + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - command: shell.exec + params: + working_dir: "wiredtiger" + script: | + set -o errexit + set -o verbose + + cd .libs + ${python_test_env_vars|} python ../test/suite/run.py [lmnopq] -v 2 ${smp_command|} 2>&1 + + - name: unit-test-bucket05 + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - command: shell.exec + params: + working_dir: "wiredtiger" + script: | + set -o errexit + set -o verbose + + cd .libs + ${python_test_env_vars|} python ../test/suite/run.py [rs] -v 2 ${smp_command|} 2>&1 + + - name: unit-test-bucket06 + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - command: shell.exec + params: + working_dir: "wiredtiger" + script: | + set -o errexit + set -o verbose + + cd .libs + ${python_test_env_vars|} python ../test/suite/run.py [t] -v 2 ${smp_command|} 2>&1 + + - name: unit-test-bucket07 + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - command: shell.exec + params: + working_dir: "wiredtiger" + script: | + set -o errexit + set -o verbose + + cd .libs + ${python_test_env_vars|} python ../test/suite/run.py [uvwxyz] -v 2 ${smp_command|} 2>&1 + + # End of Python unit test tasks - name: compile-windows-alt depends_on: @@ -884,7 +1006,14 @@ buildvariants: - name: csuite-wt2909-checkpoint-integrity-test - name: csuite-wt3338-partial-update-test - name: csuite-wt4333-handle-locks-test - - name: unit-test + - name: unit-test-bucket00 + - name: unit-test-bucket01 + - name: unit-test-bucket02 + - name: unit-test-bucket03 + - name: unit-test-bucket04 + - name: unit-test-bucket05 + - name: unit-test-bucket06 + - name: unit-test-bucket07 - name: fops - name: large-scale-test diff --git a/src/third_party/wiredtiger/test/evergreen/csuite_test_evg_task.template b/src/third_party/wiredtiger/test/evergreen/csuite_test_evg_task.template new file mode 100644 index 00000000000..42d08c34fa0 --- /dev/null +++ b/src/third_party/wiredtiger/test/evergreen/csuite_test_evg_task.template @@ -0,0 +1,14 @@ + - name: {{task_name}} + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - command: shell.exec + params: + working_dir: "wiredtiger/test/csuite" + script: | + set -o errexit + set -o verbose + + ${test_env_vars|} $(pwd)/{{test_dir}}/test_{{test_dir}} 2>&1 + diff --git a/src/third_party/wiredtiger/test/evergreen/evg_cfg.py b/src/third_party/wiredtiger/test/evergreen/evg_cfg.py new file mode 100755 index 00000000000..0096219619b --- /dev/null +++ b/src/third_party/wiredtiger/test/evergreen/evg_cfg.py @@ -0,0 +1,331 @@ +#! /usr/bin/env python3 + +""" +This program provides an CLI interface to check and generate Evergreen configuration. +""" + +import os +import sys +import re +import subprocess + +try: + import docopt +except Exception as e: + modules = "docopt" + print("ERROR [%s]: %s" % (sys.argv[0], e)) + print("Use pip to install the required library:") + print(" pip install %s" % modules) + sys.exit(0) + +TEST_TYPES = ('make_check', 'csuite') +EVG_CFG_FILE = "test/evergreen.yml" +CSUITE_TEST_DIR = "test/csuite" +MAKE_CHECK_TEST_TMPLT = "test/evergreen/make_check_test_evg_task.template" +CSUITE_TEST_TMPLT = "test/evergreen/csuite_test_evg_task.template" +MAKE_CHECK_TEST_SEARCH_STR = " # End of normal make check test tasks" +CSUITE_TEST_SEARCH_STR = " # End of csuite test tasks" + +# This list of sub directories will be skipped from checking. +# They are not expected to trigger any 'make check' testing. +make_check_subdir_skips = [ + "api/leveldb", # no need to test + "test/csuite", # csuite has its own set of Evergreen tasks, skip the checking here +] + +prog=sys.argv[0] +PROGNAME = os.path.basename(prog) +DESCRIPTION = 'Evergreen configuration helper 0.1' +USAGE = """ +Evergreen configuration helper. + +Usage: + {progname} check [-t <test_type>] [-v] + {progname} generate [-t <test_type>] [-v] + {progname} (-h | --help) + +Options: + -h --help Show this screen. + -t TEST_TYPE The type of test to be checked/generated. + -v Enable verbose logging. + check Check if any missing tests that should be added into Evergreen configuration. + generate Generate Evergreen configuration for missing tests. +""".format(progname=PROGNAME) + +verbose = False + +def debug(msg): + """ A wrapper to print function with checking of verbose flag """ + + if verbose is True: + print(msg) + +def run(cmd): + """ Run a shell command and return the output """ + + if isinstance(cmd, str): + cmd = cmd.split() + + try: + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT).strip().decode() + except Exception as e: + sys.exit("ERROR [%s]: %s" % (prog, e)) + + return output + +def find_tests_missing_evg_cfg(test_type, dirs, evg_cfg_file): + """ + Check the list of 'make check' directories to find out those + that are missing from the Evergreen configuration file. + + The existing Evergreen configuration is expected to have + included one task for each applicable 'make check' directory. + Newly added 'make check' directories that involve real tests + should be identified by this function. + """ + + if not dirs: + sys.exit("\nNo %s directory is found ..." % test_type) + + assert os.path.isfile(evg_cfg_file), "'%s' does not exist" % evg_cfg_file + + with open(evg_cfg_file, 'r') as f: + evg_cfg = f.readlines() + + debug('\n') + missing_tests = {} + for d in dirs: + # Figure out the Evergreen task name from the directory name + + if test_type == 'make_check': + # The Evergreen task name for each 'make check' test is worked out from directory name + # E.g. for 'make check' directory 'test/cursor_order', the corresponding Evergreen + # task name will be 'cursor-order-test'. + + dir_wo_test_prefix = d[len("test/"):] if d.startswith("test/") else d + evg_task_name = dir_wo_test_prefix.replace('/', '-').replace('_', '-') + '-test' + debug("Evergreen task name for make check directory '%s' is: %s" % (d, evg_task_name)) + + elif test_type == 'csuite': + # The Evergreen task name for each 'csuite' test is worked out from sub directory name + # E.g. for 'test/csuite' sub directory 'wt3184_dup_index_collator', the corresponding + # Evergreen task name will be 'csuite-wt3184-dup-index-collator-test'. + + evg_task_name = 'csuite-' + d.replace('_', '-') + '-test' + debug("Evergreen task name for csuite sub directory '%s' is: %s" % (d, evg_task_name)) + + else: + sys.exit("Unsupported test_type '%s'" % test_type) + + # Check if the Evergreen task name exists in current Evergreen configuration + if evg_task_name in str(evg_cfg): + # Match found + continue + else: + # Missing task/test found + missing_tests.update({evg_task_name: d}) + print("Task '%s' (for directory '%s') is missing in %s!" % + (evg_task_name, d, evg_cfg_file)) + + return missing_tests + +def get_make_check_dirs(): + """ + Figure out the 'make check' directories that are applicable for testing + Directories with Makefile.am containing 'TESTS =' are the ones require test. + Skip a few known directories that do not require test or covered separately. + """ + + # Make sure we are under the repo top level directory + os.chdir(run('git rev-parse --show-toplevel')) + + # Search keyword in Makefile.am to identify directories that involve test configuration. + # Need to use subprocess 'shell=True' to get the expected shell command output. + cmd = "find . -name Makefile.am -exec grep -H -e '^TESTS =' {} \; | cut -d: -f1 | cut -c3-" + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True) + mkfiles_with_tests = p.stdout.readlines() + + # Need some string manipulation work here against the subprocess output. + # Cast elements to string, and strip the ending from the string to get directory names. + ending = '/Makefile.am\n' + dirs_with_tests = [d.decode('utf-8')[:-len(ending)] for d in mkfiles_with_tests] + debug("dirs_with_tests: %s" % dirs_with_tests) + + # Remove directories in the skip list + make_check_dirs = [d for d in dirs_with_tests if d not in make_check_subdir_skips] + debug("\nThe list of 'make check' dirs that should be included in Evergreen configuration:\n %s" + % make_check_dirs) + + return make_check_dirs + +def get_csuite_dirs(): + """ + Figure out the 'make check' directories that are applicable for testing + Loop through the list of sub directories under test/csuite/ and skip those WT_TEST.* directories + """ + + assert os.path.isdir(CSUITE_TEST_DIR), "'%s' does not exist" % CSUITE_TEST_DIR + + # Retrieve all sub directories under 'test/csuite' directory + subdirs = [x[1] for x in os.walk(CSUITE_TEST_DIR)][0] + + # Remove directories with name starting with 'WT_TEST' or '.' + regex = re.compile(r'^(WT_TEST|\.)') + csuite_dirs = [d for d in subdirs if not regex.search(d)] + debug("\nThe list of 'csuite' dirs that should be included in Evergreen configuration:\n %s" + % csuite_dirs) + + return csuite_dirs + +def check_missing_tests(test_type): + """ + Check to see if any tests are missing from the Evergreen configuration. + Loop through the list of directories in 'Make.subdirs' file and skip a few known + directories that do not require any test. + """ + + # Retrieve the directories that are applicable for testing based on test type + if test_type == 'make_check': + test_dirs = get_make_check_dirs() + elif test_type == 'csuite': + test_dirs = get_csuite_dirs() + else: + sys.exit("Unsupported test type '%s'" % test_type) + + return find_tests_missing_evg_cfg(test_type, test_dirs, EVG_CFG_FILE) + +def get_evg_task_template(test_type): + """ Retrieve the Evergreen task template based on test type """ + + if test_type == 'make_check': + template_file = MAKE_CHECK_TEST_TMPLT + elif test_type == 'csuite': + template_file = CSUITE_TEST_TMPLT + else: + sys.exit("Unsupported test type '%s'" % test_type) + + assert os.path.isfile(template_file), "'%s' does not exist" % template_file + + with open(template_file, 'r') as f: + template = f.read() + + return template + +def get_search_string(test_type): + """ Retrieve the search string based on test_type """ + + if test_type == 'make_check': + search_str = MAKE_CHECK_TEST_SEARCH_STR + elif test_type == 'csuite': + search_str = CSUITE_TEST_SEARCH_STR + else: + sys.exit("Unsupported test type '%s'" % test_type) + + return search_str + +def generate_evg_cfg_for_missing_tests(test_type, missing_tests): + """ + Generate the Evergreen configuration for the missing tests based on test type. + Will apply the newly generated changes to the Evergreen configuration file directly. + """ + + if not missing_tests: + sys.exit("No missing test is found, exiting ...") + debug("missing_tests: %s" % missing_tests) + + template = get_evg_task_template(test_type) + + evg_cfg_to_add = '' + for task, dir in missing_tests.items(): + # Replace variables in the template with valid values for each missing test + evg_cfg_to_add += template.replace('{{task_name}}', task).replace('{{test_dir}}', dir) + + print("\nBelow Evergreen configuration snippet will be added into existing %s file: \n\n%s" + % (EVG_CFG_FILE, evg_cfg_to_add)) + + assert os.path.isfile(EVG_CFG_FILE), "'%s' does not exist" % EVG_CFG_FILE + + search_str = get_search_string(test_type) + debug("search_str: '%s'" % search_str) + + with open(EVG_CFG_FILE, 'r') as f: + evg_cfg = f.read() + # Insert the new Evergreen configuration for missing tests. + # Use the search string to help locating the position for insert. + new_evg_cfg = evg_cfg.replace(search_str, evg_cfg_to_add + search_str) + + # Write the changes back to the file + with open(EVG_CFG_FILE, 'w') as f: + f.write(new_evg_cfg) + +def evg_cfg(action, test_type): + """ + The main entry function that calls different functions based on action type and test type + """ + + # Make sure the program is run under a checkout of wiredtiger repository + # We could get different string outputs when running 'git config remote.origin.url': + # - 'git@github.com:wiredtiger/wiredtiger.git' (if run locally) + # - 'ssh://git@github.com/wiredtiger/wiredtiger.git' (if run through SSH) + output = run('git config remote.origin.url') + if not 'github.com' in output or not 'wiredtiger.git' in output: + sys.exit("ERROR [%s]: need to run this script inside a wiredtiger repo" % prog) + + # Change directory to repo top level + os.chdir(run('git rev-parse --show-toplevel')) + + missing_tests = {} + if action == 'check': + if test_type in TEST_TYPES: + missing_tests = check_missing_tests(test_type) + elif test_type == 'all': + # Check each of the test type + for t in TEST_TYPES: + # Aggregate the missing tests for each test type together here + missing_tests.update(check_missing_tests(t)) + else: + sys.exit("Unsupported test type '%s'" % test_type) + + # If any missing test is identified, prompt users to run the 'generate' action + # which will auto-generate the Evergreen configuration for those missing tests. + if missing_tests: + prompt = ("\n*** Some tests are missing in Evergreen configuration ***\nPlease\n" + + "\t1) Run '{prog} generate' to generate and apply the Evergreen changes.\n" + + "\t2) Run 'git diff' to see the detail of the Evergreen changes.\n" + + "\t3) Trigger Evergreen patch build to verify the changes before merging.\n" + ).format(prog=prog) + print(prompt) + sys.exit(1) + + elif action == 'generate': + if test_type in TEST_TYPES: + missing_tests = check_missing_tests(test_type) + generate_evg_cfg_for_missing_tests(test_type, missing_tests) + elif test_type == 'all': + # Check each of the test type + for t in TEST_TYPES: + missing_tests = check_missing_tests(t) + generate_evg_cfg_for_missing_tests(t, missing_tests) + else: + sys.exit("Unsupported action type '%s'" % action) + +if __name__ == '__main__': + + args = docopt.docopt(USAGE, version=DESCRIPTION) + + verbose = args['-v'] + debug('\nargs:%s' % args) + + action = None + if args['check']: + action = 'check' + elif args['generate']: + action = 'generate' + assert action in ('check', 'generate') + + test_type = args.get('-t', None) + # If test type is not provided, assuming 'all' types need to be checked + if test_type is None: + test_type = 'all' + + evg_cfg(action, test_type) diff --git a/src/third_party/wiredtiger/test/evergreen/make_check_test_evg_task.template b/src/third_party/wiredtiger/test/evergreen/make_check_test_evg_task.template new file mode 100644 index 00000000000..762bcfff29d --- /dev/null +++ b/src/third_party/wiredtiger/test/evergreen/make_check_test_evg_task.template @@ -0,0 +1,15 @@ + - name: {{task_name}} + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - func: "compile wiredtiger" + - command: shell.exec + params: + working_dir: "wiredtiger" + script: | + set -o errexit + set -o verbose + + ${test_env_vars|} ${make_command|make} VERBOSE=1 check -C {{test_dir}} ${smp_command|} 2>&1 + diff --git a/src/third_party/wiredtiger/test/fops/Makefile.am b/src/third_party/wiredtiger/test/fops/Makefile.am index f8a76de82bc..519f6315445 100644 --- a/src/third_party/wiredtiger/test/fops/Makefile.am +++ b/src/third_party/wiredtiger/test/fops/Makefile.am @@ -15,4 +15,4 @@ TESTS = $(noinst_PROGRAMS) LOG_COMPILER = $(TEST_WRAPPER) clean-local: - rm -rf WT_TEST *.core + rm -rf WT_TEST core.* *.core diff --git a/src/third_party/wiredtiger/test/format/Makefile.am b/src/third_party/wiredtiger/test/format/Makefile.am index 5d946e5b63d..3a74cb0c104 100644 --- a/src/third_party/wiredtiger/test/format/Makefile.am +++ b/src/third_party/wiredtiger/test/format/Makefile.am @@ -45,4 +45,4 @@ refresh: TESTS = smoke.sh clean-local: - rm -rf RUNDIR s_dumpcmp *.core + rm -rf RUNDIR s_dumpcmp core.* *.core diff --git a/src/third_party/wiredtiger/test/format/config.c b/src/third_party/wiredtiger/test/format/config.c index 9a32a96423d..41dae7fe364 100644 --- a/src/third_party/wiredtiger/test/format/config.c +++ b/src/third_party/wiredtiger/test/format/config.c @@ -392,12 +392,9 @@ config_compression(const char *conf_name) */ switch (mmrand(NULL, 1, 20)) { #ifdef HAVE_BUILTIN_EXTENSION_LZ4 - case 1: case 2: /* 10% lz4 */ + case 1: case 2: case 3: /* 15% lz4 */ cstr = "lz4"; break; - case 3: /* 5% lz4-no-raw */ - cstr = "lz4-noraw"; - break; #endif #ifdef HAVE_BUILTIN_EXTENSION_SNAPPY case 4: case 5: case 6: case 7: /* 30% snappy */ @@ -409,12 +406,9 @@ config_compression(const char *conf_name) case 10: case 11: case 12: case 13: /* 20% zlib */ cstr = "zlib"; break; - case 14: /* 5% zlib-no-raw */ - cstr = "zlib-noraw"; - break; #endif #ifdef HAVE_BUILTIN_EXTENSION_ZSTD - case 15: case 16: case 17: /* 15% zstd */ + case 14: case 15: case 16: case 17: /* 20% zstd */ cstr = "zstd"; break; #endif @@ -1138,16 +1132,14 @@ config_map_compression(const char *s, u_int *vp) *vp = COMPRESS_NONE; else if (strcmp(s, "lz4") == 0) *vp = COMPRESS_LZ4; - else if (strcmp(s, "lz4-noraw") == 0) - *vp = COMPRESS_LZ4_NO_RAW; - else if (strcmp(s, "lzo") == 0) - *vp = COMPRESS_LZO; + else if (strcmp(s, "lz4-noraw") == 0) /* CONFIG compatibility */ + *vp = COMPRESS_LZ4; else if (strcmp(s, "snappy") == 0) *vp = COMPRESS_SNAPPY; else if (strcmp(s, "zlib") == 0) *vp = COMPRESS_ZLIB; - else if (strcmp(s, "zlib-noraw") == 0) - *vp = COMPRESS_ZLIB_NO_RAW; + else if (strcmp(s, "zlib-noraw") == 0) /* CONFIG compatibility */ + *vp = COMPRESS_ZLIB; else if (strcmp(s, "zstd") == 0) *vp = COMPRESS_ZSTD; else diff --git a/src/third_party/wiredtiger/test/format/config.h b/src/third_party/wiredtiger/test/format/config.h index a64a808d968..44961edfbda 100644 --- a/src/third_party/wiredtiger/test/format/config.h +++ b/src/third_party/wiredtiger/test/format/config.h @@ -58,7 +58,7 @@ typedef struct { } CONFIG; #define COMPRESSION_LIST \ - "(none | lz4 | lz4-noraw | snappy | zlib | zlib-noraw | zstd)" + "(none | lz4 | snappy | zlib | zstd)" static CONFIG c[] = { { "abort", diff --git a/src/third_party/wiredtiger/test/format/format.h b/src/third_party/wiredtiger/test/format/format.h index 8c4132d137e..e9063674476 100644 --- a/src/third_party/wiredtiger/test/format/format.h +++ b/src/third_party/wiredtiger/test/format/format.h @@ -62,8 +62,6 @@ #define HELIUM_PATH \ EXTPATH "datasources/helium/.libs/libwiredtiger_helium.so" -#define LZO_PATH ".libs/lzo_compress.so" - #undef M #define M(v) ((v) * WT_MILLION) /* Million */ #undef KILOBYTE @@ -257,12 +255,9 @@ typedef struct { #define COMPRESS_NONE 1 #define COMPRESS_LZ4 2 -#define COMPRESS_LZ4_NO_RAW 3 -#define COMPRESS_LZO 4 -#define COMPRESS_SNAPPY 5 -#define COMPRESS_ZLIB 6 -#define COMPRESS_ZLIB_NO_RAW 7 -#define COMPRESS_ZSTD 8 +#define COMPRESS_SNAPPY 3 +#define COMPRESS_ZLIB 4 +#define COMPRESS_ZSTD 5 u_int c_compression_flag; /* Compression flag value */ u_int c_logging_compression_flag; /* Log compression flag value */ diff --git a/src/third_party/wiredtiger/test/format/wts.c b/src/third_party/wiredtiger/test/format/wts.c index b62885f0369..0c22e98bd3c 100644 --- a/src/third_party/wiredtiger/test/format/wts.c +++ b/src/third_party/wiredtiger/test/format/wts.c @@ -45,21 +45,12 @@ compressor(uint32_t compress_flag) case COMPRESS_LZ4: p ="lz4"; break; - case COMPRESS_LZ4_NO_RAW: - p ="lz4-noraw"; - break; - case COMPRESS_LZO: - p ="LZO1B-6"; - break; case COMPRESS_SNAPPY: p ="snappy"; break; case COMPRESS_ZLIB: p ="zlib"; break; - case COMPRESS_ZLIB_NO_RAW: - p ="zlib-noraw"; - break; case COMPRESS_ZSTD: p ="zstd"; break; @@ -269,10 +260,9 @@ wts_open(const char *home, bool set_api, WT_CONNECTION **connp) /* Extensions. */ CONFIG_APPEND(p, ",extensions=[" - "\"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\"],", + "\"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\"],", g.c_reverse ? REVERSE_PATH : "", access(LZ4_PATH, R_OK) == 0 ? LZ4_PATH : "", - access(LZO_PATH, R_OK) == 0 ? LZO_PATH : "", access(ROTN_PATH, R_OK) == 0 ? ROTN_PATH : "", access(SNAPPY_PATH, R_OK) == 0 ? SNAPPY_PATH : "", access(ZLIB_PATH, R_OK) == 0 ? ZLIB_PATH : "", diff --git a/src/third_party/wiredtiger/test/huge/Makefile.am b/src/third_party/wiredtiger/test/huge/Makefile.am index 894bff5eace..402e8efc81e 100644 --- a/src/third_party/wiredtiger/test/huge/Makefile.am +++ b/src/third_party/wiredtiger/test/huge/Makefile.am @@ -13,4 +13,4 @@ t_LDFLAGS = -static TESTS = smoke.sh clean-local: - rm -rf WT_TEST *.core + rm -rf WT_TEST core.* *.core diff --git a/src/third_party/wiredtiger/test/manydbs/Makefile.am b/src/third_party/wiredtiger/test/manydbs/Makefile.am index ff5985cf2a4..66310350d71 100644 --- a/src/third_party/wiredtiger/test/manydbs/Makefile.am +++ b/src/third_party/wiredtiger/test/manydbs/Makefile.am @@ -14,4 +14,4 @@ TESTS = $(noinst_PROGRAMS) LOG_COMPILER = $(TEST_WRAPPER) clean-local: - rm -rf WT_TEST *.core + rm -rf WT_TEST core.* *.core diff --git a/src/third_party/wiredtiger/test/packing/Makefile.am b/src/third_party/wiredtiger/test/packing/Makefile.am index c9128100cc3..0f6f59d2cff 100644 --- a/src/third_party/wiredtiger/test/packing/Makefile.am +++ b/src/third_party/wiredtiger/test/packing/Makefile.am @@ -11,4 +11,4 @@ LDFLAGS = -static TESTS = smoke.sh clean-local: - rm -rf *.core + rm -rf core.* *.core diff --git a/src/third_party/wiredtiger/test/readonly/Makefile.am b/src/third_party/wiredtiger/test/readonly/Makefile.am index 84092e76f02..7dd08586122 100644 --- a/src/third_party/wiredtiger/test/readonly/Makefile.am +++ b/src/third_party/wiredtiger/test/readonly/Makefile.am @@ -13,4 +13,4 @@ t_LDFLAGS = -static TESTS = smoke.sh clean-local: - rm -rf WT_RD* *.core + rm -rf WT_RD* core.* *.core diff --git a/src/third_party/wiredtiger/test/salvage/Makefile.am b/src/third_party/wiredtiger/test/salvage/Makefile.am index a3c49b9c41a..b9ba9c35d72 100644 --- a/src/third_party/wiredtiger/test/salvage/Makefile.am +++ b/src/third_party/wiredtiger/test/salvage/Makefile.am @@ -14,4 +14,4 @@ TESTS = $(noinst_PROGRAMS) LOG_COMPILER = $(TEST_WRAPPER) clean-local: - rm -rf WiredTiger* __slvg* *.core + rm -rf WiredTiger* __slvg* core.* *.core diff --git a/src/third_party/wiredtiger/test/suite/README b/src/third_party/wiredtiger/test/suite/README new file mode 100644 index 00000000000..e63fd6dfc64 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/README @@ -0,0 +1,13 @@ +The test/suite directory includes a collection of Python unit tests +that are expected to be executed when code change is introduced in this repo. + +These Python tests are broken down and grouped into multiple buckets/tasks +in Evergreen (CI system) configuration. See test/evergreen.yml for details. + +There is a plan to implement a mechanism to auto-group tests into buckets/tasks +based on history runtime of each test, and generate the Evergreen configuration +dynamically before each Evergreen build variant run, so that no mental overhead +is required when new tests is introduced into test/suite. (WT-4441) + +Before the above mentioned mechansim is put into place, please double check +test/evergreen.yml and test run logs to make sure new test are covered. diff --git a/src/third_party/wiredtiger/test/suite/test_alter04.py b/src/third_party/wiredtiger/test/suite/test_alter04.py new file mode 100644 index 00000000000..97978922848 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_alter04.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2018 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +from wtscenario import make_scenarios + +# test_alter04.py +# Smoke-test the session alter operations. +# This test confirms os_cache_dirty_max and os_cache_max. +class test_alter04(wttest.WiredTigerTestCase): + name = "alter04" + entries = 100 + cache_alter=('1M', '100K') + # Settings for os_cache[_dirty]_max. + types = [ + ('file', dict(uri='file:', use_cg=False, use_index=False)), + ('lsm', dict(uri='lsm:', use_cg=False, use_index=False)), + ('table-cg', dict(uri='table:', use_cg=True, use_index=False)), + ('table-index', dict(uri='table:', use_cg=False, use_index=True)), + ('table-simple', dict(uri='table:', use_cg=False, use_index=False)), + ] + sizes = [ + ('default', dict(ocreate='')), + ('1M', dict(ocreate='1M')), + ('200K', dict(ocreate='200K')), + ] + reopen = [ + ('no-reopen', dict(reopen=False)), + ('reopen', dict(reopen=True)), + ] + settings = [ + ('cache', dict(setting='os_cache_max')), + ('cache_dirty', dict(setting='os_cache_dirty_max')), + ] + scenarios = make_scenarios(types, sizes, reopen, settings) + + def verify_metadata(self, metastr): + if metastr == '': + return + cursor = self.session.open_cursor('metadata:', None, None) + # + # Walk through all the metadata looking for the entries that are + # the file URIs for components of the table. + # + found = False + while True: + ret = cursor.next() + if ret != 0: + break + key = cursor.get_key() + check_meta = ((key.find("lsm:") != -1 or key.find("file:") != -1) \ + and key.find(self.name) != -1) + if check_meta: + value = cursor[key] + found = True + self.assertTrue(value.find(metastr) != -1) + cursor.close() + self.assertTrue(found == True) + + # Alter: Change the setting after creation + def test_alter04_cache(self): + uri = self.uri + self.name + create_params = 'key_format=i,value_format=i,' + complex_params = '' + # + # If we're not explicitly setting the parameter, then don't + # modify create_params to test using the default. + # + if self.ocreate != '': + new_param = '%s=%s' % (self.setting, self.ocreate) + create_params += '%s,' % new_param + complex_params += '%s,' % new_param + else: + # NOTE: This is hard-coding the default value. If the default + # changes then this will fail and need to be fixed. + new_param = '%s=0' % self.setting + + cgparam = '' + if self.use_cg or self.use_index: + cgparam = 'columns=(k,v),' + if self.use_cg: + cgparam += 'colgroups=(g0),' + + self.session.create(uri, create_params + cgparam) + # Add in column group or index settings. + if self.use_cg: + cgparam = 'columns=(v),' + suburi = 'colgroup:' + self.name + ':g0' + self.session.create(suburi, complex_params + cgparam) + if self.use_index: + suburi = 'index:' + self.name + ':i0' + self.session.create(suburi, complex_params + cgparam) + + # Put some data in table. + c = self.session.open_cursor(uri, None) + for k in range(self.entries): + c[k+1] = 1 + c.close() + + # Verify the string in the metadata + self.verify_metadata(new_param) + + # Run through all combinations of the alter commands + # for all allowed settings. + for a in self.cache_alter: + alter_param = '%s=%s' % (self.setting, a) + self.session.alter(uri, alter_param) + if self.reopen: + self.reopen_conn() + special = self.use_cg or self.use_index + if not special: + self.verify_metadata(alter_param) + else: + self.session.alter(suburi, alter_param) + self.verify_metadata(alter_param) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_backup04.py b/src/third_party/wiredtiger/test/suite/test_backup04.py index 13b2436d7ad..ba39d4aebfb 100644 --- a/src/third_party/wiredtiger/test/suite/test_backup04.py +++ b/src/third_party/wiredtiger/test/suite/test_backup04.py @@ -37,7 +37,7 @@ from wtthread import op_thread # test_backup04.py # Utilities: wt backup -# Test cursor backup with target URIs +# Test incremental cursor backup. class test_backup_target(wttest.WiredTigerTestCase, suite_subprocess): dir='backup.dir' # Backup directory name logmax="100K" diff --git a/src/third_party/wiredtiger/test/suite/test_backup07.py b/src/third_party/wiredtiger/test/suite/test_backup07.py index b15ab274a8b..b2b6fcc80de 100644 --- a/src/third_party/wiredtiger/test/suite/test_backup07.py +++ b/src/third_party/wiredtiger/test/suite/test_backup07.py @@ -51,16 +51,14 @@ class test_backup07(wttest.WiredTigerTestCase, suite_subprocess): return 'cache_size=1G,log=(archive=false,enabled,file_max=%s)' % \ self.logmax - # Run background inserts while running checkpoints and incremental backups - # repeatedly. + # Run background inserts while running checkpoints repeatedly. def test_backup07(self): log2 = "WiredTigerLog.0000000002" self.session.create(self.uri, "key_format=S,value_format=S") # Insert small amounts of data at a time stopping just after we - # cross into log file 2. That way we can add more operations into - # log file 2 during the full backup. + # cross into log file 2. loop = 0 c = self.session.open_cursor(self.uri) while not os.path.exists(log2): @@ -74,9 +72,7 @@ class test_backup07(wttest.WiredTigerTestCase, suite_subprocess): # Test a potential bug in full backups and creates. # We allow creates during backup because the file doesn't exist # when the backup metadata is created on cursor open and the newly - # created file is not in the cursor list. However, if using logging - # and the create and inserts/updates appear in a log file copied, - # then currently there will be an error opening the backup directory. + # created file is not in the cursor list. # Open up the backup cursor, create and add data to a new table # and then copy the files. @@ -94,13 +90,14 @@ class test_backup07(wttest.WiredTigerTestCase, suite_subprocess): c.close() self.session.log_flush('sync=on') - # Now copy the files returned by the backup cursor. This will - # include the log file that has updates for the newly created table. + # Now copy the files returned by the backup cursor. This should not + # include the newly created table. while True: ret = bkup_c.next() if ret != 0: break newfile = bkup_c.get_key() + self.assertNotEqual(newfile, self.newuri) sz = os.path.getsize(newfile) self.pr('Copy from: ' + newfile + ' (' + str(sz) + ') to ' + self.dir) shutil.copy(newfile, self.dir) diff --git a/src/third_party/wiredtiger/test/suite/test_backup10.py b/src/third_party/wiredtiger/test/suite/test_backup10.py new file mode 100644 index 00000000000..27838834ace --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_backup10.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2018 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +import os, shutil +from helper import compare_files +from suite_subprocess import suite_subprocess +from wtdataset import simple_key +from wtscenario import make_scenarios + +# test_backup10.py +# Test cursor backup with a duplicate backup cursor. +class test_backup10(wttest.WiredTigerTestCase, suite_subprocess): + dir='backup.dir' # Backup directory name + logmax="100K" + newuri="table:newtable" + uri="table:test" + nops=100 + + pfx = 'test_backup' + + scenarios = make_scenarios([ + ('archiving', dict(archive='true')), + ('not-archiving', dict(archive='false')), + ]) + + # Create a large cache, otherwise this test runs quite slowly. + def conn_config(self): + return 'cache_size=1G,log=(archive=%s,' % self.archive + \ + 'enabled,file_max=%s)' % self.logmax + + # Run background inserts while running checkpoints repeatedly. + def test_backup10(self): + log2 = "WiredTigerLog.0000000002" + log3 = "WiredTigerLog.0000000003" + + self.session.create(self.uri, "key_format=S,value_format=S") + + # Insert small amounts of data at a time stopping after we + # cross into log file 2. + loop = 0 + c = self.session.open_cursor(self.uri) + while not os.path.exists(log2): + for i in range(0, self.nops): + num = i + (loop * self.nops) + key = 'key' + str(num) + val = 'value' + str(num) + c[key] = val + loop += 1 + + # Open up the backup cursor. This causes a new log file to be created. + # That log file is not part of the list returned. + os.mkdir(self.dir) + bkup_c = self.session.open_cursor('backup:', None, None) + + # Add some data that will appear in log file 3. + for i in range(0, self.nops): + num = i + (loop * self.nops) + key = 'key' + str(num) + val = 'value' + str(num) + c[key] = val + loop += 1 + c.close() + self.session.log_flush('sync=on') + + # Now copy the files returned by the backup cursor. + orig_logs = [] + while True: + ret = bkup_c.next() + if ret != 0: + break + newfile = bkup_c.get_key() + self.assertNotEqual(newfile, self.newuri) + sz = os.path.getsize(newfile) + self.pr('Copy from: ' + newfile + ' (' + str(sz) + ') to ' + self.dir) + shutil.copy(newfile, self.dir) + if "WiredTigerLog" in newfile: + orig_logs.append(newfile) + self.assertEqual(ret, wiredtiger.WT_NOTFOUND) + + # Now open a duplicate backup cursor. + config = 'target=("log:")' + dupc = self.session.open_cursor(None, bkup_c, config) + dup_logs = [] + while True: + ret = dupc.next() + if ret != 0: + break + newfile = dupc.get_key() + self.assertTrue("WiredTigerLog" in newfile) + sz = os.path.getsize(newfile) + if (newfile not in orig_logs): + self.pr('DUP: Copy from: ' + newfile + ' (' + str(sz) + ') to ' + self.dir) + shutil.copy(newfile, self.dir) + # Record all log files returned for later verification. + dup_logs.append(newfile) + self.assertEqual(ret, wiredtiger.WT_NOTFOUND) + + # We expect that the duplicate logs are a superset of the + # original logs. And we expect the difference to be the + # addition of log file 3 only. + orig_set = set(orig_logs) + dup_set = set(dup_logs) + self.assertTrue(dup_set.issuperset(orig_set)) + diff = dup_set.difference(orig_set) + self.assertEqual(len(diff), 1) + self.assertTrue(log3 in dup_set) + + # Test a few error cases now. + # - We cannot make multiple duplcate backup cursors. + # - We cannot duplicate the duplicate backup cursor. + # - We must use the log target. + msg = "/already a duplicate backup cursor open/" + # Test multiple duplicate backup cursors. + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda:self.assertEquals(self.session.open_cursor(None, + bkup_c, config), 0), msg) + # Test duplicate of duplicate backup cursor. + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda:self.assertEquals(self.session.open_cursor(None, + dupc, config), 0), msg) + + dupc.close() + + # Test we must use the log target. + msg = "/must be for logs/" + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda:self.assertEquals(self.session.open_cursor(None, + bkup_c, None), 0), msg) + + # Open duplicate backup cursor again now that the first + # one is closed. Test every log file returned is the same + # as the first time. + dupc = self.session.open_cursor(None, bkup_c, config) + while True: + ret = dupc.next() + if ret != 0: + break + newfile = dupc.get_key() + self.assertTrue("WiredTigerLog" in newfile) + self.assertTrue(newfile in dup_logs) + self.assertEqual(ret, wiredtiger.WT_NOTFOUND) + + dupc.close() + bkup_c.close() + + # After the full backup, open and recover the backup database. + backup_conn = self.wiredtiger_open(self.dir) + backup_conn.close() + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_bug019.py b/src/third_party/wiredtiger/test/suite/test_bug019.py index c25afa692cb..fd68578ce42 100644 --- a/src/third_party/wiredtiger/test/suite/test_bug019.py +++ b/src/third_party/wiredtiger/test/suite/test_bug019.py @@ -94,14 +94,15 @@ class test_bug019(wttest.WiredTigerTestCase): older = newer self.session.checkpoint() - # Wait for up to 30 seconds for pre-allocate to drop in an idle system + # Wait for a long time for pre-allocate to drop in an idle system # it should usually be fast, but on slow systems can take time. - for sleepcount in range(1,30): + max_wait_time = 90 + for sleepcount in range(1,max_wait_time): new_prealloc = self.get_prealloc_stat() if new_prealloc < self.max_prealloc: break time.sleep(1.0) - self.assertTrue(sleepcount < 30) + self.assertTrue(sleepcount < max_wait_time) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_calc_modify.py b/src/third_party/wiredtiger/test/suite/test_calc_modify.py new file mode 100644 index 00000000000..a52096a3125 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_calc_modify.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2018 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import random, string +import wiredtiger, wttest + +r = random.Random(42) # Make things repeatable + +# test_calc_modify.py +# Test the wiredtiger_calc_modify API +# +# Try many combinations of: +# - data size +# - data randomness ('a' * N, repeated patterns, uniform random) +# - number and type of modifications (add, remove, replace) +# - space between the modifications +# +# Check that wiredtiger_calc_modify finds a set of modifies when the edit +# difference is under the specified limits, and that applying those +# modifications produces the expected result. If the edit difference is +# larger than the limits, it okay for the call to fail. +class test_calc_modify(wttest.WiredTigerTestCase): + uri = 'table:test_calc_modify' + + # operation types + ADD = 1 + REMOVE = 2 + REPLACE = 3 + + def mkstring(self, size, repeat_size=1): + pattern = ''.join(r.choice(string.ascii_letters + string.digits) for _ in xrange(repeat_size)) + return (pattern * ((size + repeat_size - 1) / repeat_size))[:size] + + def one_test(self, c, k, oldsz, repeatsz, nmod, maxdiff): + oldv = self.mkstring(oldsz, repeatsz) + + offsets = sorted(r.sample(xrange(oldsz), nmod)) + modsizes = sorted(r.sample(xrange(maxdiff), nmod + 1)) + lengths = [modsizes[i+1] - modsizes[i] for i in xrange(nmod)] + modtypes = [r.choice((self.ADD, self.REMOVE, self.REPLACE)) for _ in xrange(nmod)] + + self.pr("offsets: %s" % offsets) + self.pr("modsizes: %s" % modsizes) + self.pr("lengths: %s" % lengths) + self.pr("modtypes: %s" % modtypes) + + orig = oldv + newv = '' + for i in xrange(nmod): + if i > 0 and offsets[i] - offsets[i - 1] < maxdiff: + continue + newv += orig[:offsets[i]] + orig = orig[offsets[i]:] + if modtypes[i] == self.ADD: + newv += self.mkstring(lengths[i], r.randint(1, lengths[i])) + elif modtypes[i] == self.REMOVE: + orig = orig[lengths[i]:] + elif modtypes[i] == self.REPLACE: + newv += self.mkstring(lengths[i], r.randint(1, lengths[i])) + orig = orig[lengths[i]:] + newv += orig + + self.pr("oldv: %s" % oldv) + self.pr("newv: %s" % newv) + try: + mods = wiredtiger.wiredtiger_calc_modify(None, oldv, newv, max(maxdiff, nmod * 64), nmod) + self.pr("calculated mods: %s" % mods) + except wiredtiger.WiredTigerError: + # When the data repeats, the algorithm can register the "wrong" repeated sequence. Retry... + mods = wiredtiger.wiredtiger_calc_modify(None, oldv, newv, nmod * (64 + repeatsz), nmod) + self.pr("calculated mods (round 2): %s" % mods) + self.assertIsNotNone(mods) + + c[k] = oldv + self.session.begin_transaction('isolation=snapshot') + c.set_key(k) + c.modify(mods) + self.session.commit_transaction() + self.assertEqual(c[k], newv) + + def test_calc_modify(self): + self.session.create(self.uri, 'key_format=i,value_format=u') + c = self.session.open_cursor(self.uri) + for k in xrange(1000): + size = r.randint(1000, 10000) + repeats = r.randint(1, size) + nmods = r.randint(1, 10) + maxdiff = r.randint(64, size / 10) + self.pr("size %s, repeats %s, nmods %s, maxdiff %s" % (size, repeats, nmods, maxdiff)) + self.one_test(c, k, size, repeats, nmods, maxdiff) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_compress01.py b/src/third_party/wiredtiger/test/suite/test_compress01.py index aa6f5104216..d675ea25b8b 100644 --- a/src/third_party/wiredtiger/test/suite/test_compress01.py +++ b/src/third_party/wiredtiger/test/suite/test_compress01.py @@ -27,14 +27,13 @@ # OTHER DEALINGS IN THE SOFTWARE. # # test_compress01.py -# Basic block compression operations +# Smoke-test compression # -import os, run import wiredtiger, wttest from wtscenario import make_scenarios -# Test basic compression +# Smoke-test compression class test_compress01(wttest.WiredTigerTestCase): types = [ @@ -43,8 +42,12 @@ class test_compress01(wttest.WiredTigerTestCase): ] compress = [ ('nop', dict(compress='nop')), + ('lz4', dict(compress='lz4')), + ('lz4-noraw', dict(compress='lz4')), # API compatibility test ('snappy', dict(compress='snappy')), - ('none', dict(compress=None)), + ('zlib', dict(compress='zlib')), + ('zlib-noraw', dict(compress='zlib')), # API compatibility test + ('zstd', dict(compress='zstd')), ] scenarios = make_scenarios(types, compress) @@ -58,13 +61,9 @@ class test_compress01(wttest.WiredTigerTestCase): # Create a table, add keys with both big and small values, then verify them. def test_compress(self): - # Use relatively small leaf pages to force big values to be overflow # items, but still large enough that we get some compression action. params = 'key_format=S,value_format=S,leaf_page_max=4096' - if self.compress != None: - params += ',block_compressor=' + self.compress - self.session.create(self.uri, params) cursor = self.session.open_cursor(self.uri, None) for idx in xrange(1,self.nrecords): diff --git a/src/third_party/wiredtiger/test/suite/test_cursor16.py b/src/third_party/wiredtiger/test/suite/test_cursor16.py index 8eeb46b8de1..1a3b4db2470 100755 --- a/src/third_party/wiredtiger/test/suite/test_cursor16.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor16.py @@ -44,7 +44,7 @@ class test_cursor16(wttest.WiredTigerTestCase): # Returns the number of cursors cached def cached_stats(self): stat_cursor = self.session.open_cursor('statistics:', None, None) - cache = stat_cursor[stat.conn.cursors_cached][2] + cache = stat_cursor[stat.conn.cursor_cached_count][2] stat_cursor.close() return cache diff --git a/src/third_party/wiredtiger/test/suite/test_encrypt05.py b/src/third_party/wiredtiger/test/suite/test_encrypt05.py deleted file mode 100644 index fa5ed483462..00000000000 --- a/src/third_party/wiredtiger/test/suite/test_encrypt05.py +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env python -# -# Public Domain 2014-2018 MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# -# This is free and unencumbered software released into the public domain. -# -# Anyone is free to copy, modify, publish, use, compile, sell, or -# distribute this software, either in source code form or as a compiled -# binary, for any purpose, commercial or non-commercial, and by any -# means. -# -# In jurisdictions that recognize copyright laws, the author or authors -# of this software dedicate any and all copyright interest in the -# software to the public domain. We make this dedication for the benefit -# of the public at large and to the detriment of our heirs and -# successors. We intend this dedication to be an overt act of -# relinquishment in perpetuity of all present and future rights to this -# software under copyright law. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -# -# test_encrypt05.py -# Test raw compression with encryption -# - -import os, run, random -import wiredtiger, wttest -from wtscenario import make_scenarios - -# Test raw compression with encryption -class test_encrypt05(wttest.WiredTigerTestCase): - - encrypt = [ - ('rotn', dict( sys_encrypt='rotn', sys_encrypt_args=',keyid=11', - file_encrypt='rotn', file_encrypt_args=',keyid=13')), - ] - compress = [ - ('zlib', dict(log_compress='zlib', block_compress='zlib')), - ] - scenarios = make_scenarios(encrypt, compress) - - nrecords = 500 - bigvalue = 'a' * 500 # we use values that will definitely give compression - - def conn_extensions(self, extlist): - extlist.skip_if_missing = True - extlist.extension('encryptors', self.sys_encrypt) - extlist.extension('encryptors', self.file_encrypt) - extlist.extension('compressors', self.block_compress) - extlist.extension('compressors', self.log_compress) - - def conn_config(self): - encarg = 'encryption=(name={0}{1}),'.format( - self.sys_encrypt, self.sys_encrypt_args) - comparg = '' - if self.log_compress != None: - comparg='log=(compressor={0}),'.format(self.log_compress) - return encarg + comparg - - def getvalue(self, r, n): - if n < len(self.bigvalue): - return self.bigvalue[0: n] - else: - diff = n - len(self.bigvalue) - rchr = ''.join(chr(r.randint(1, 255)) for i in range(diff)) - return self.bigvalue + rchr - - # Create a table, add key/values with specific lengths, then verify them. - def test_encrypt(self): - params = 'key_format=S,value_format=S' - if self.file_encrypt != None: - params += ',encryption=(name=' + self.file_encrypt + \ - self.file_encrypt_args + ')' - if self.block_compress != None: - params += ',block_compressor=' + self.block_compress - # Explicitly set max size for leaf page - params += ',leaf_page_max=8KB' - - # n is the length of the value. This range is experimentally chosen - # to be near an edge case for an 8K leaf size for raw compression. - # We can fit about 10-11 records of this size on the page. We let - # the size creep up to the edge case. The compressor is trying to - # maximize the number of records that can fit on the fixed size - # page, and the calculation is modulated by the encryptor's need for - # a constant buffer growth. - for n in xrange(1045, 1060, 1): - uri='table:test_encrypt05-' + str(n) - self.session.create(uri, params) - r = random.Random() - r.seed(0) - cursor = self.session.open_cursor(uri, None) - for idx in xrange(1,self.nrecords): - key = str(idx) - cursor.set_key(key) - cursor.set_value(self.getvalue(r, n)) - cursor.insert() - cursor.close() - - # Force the cache to disk, so we read - # compressed/encrypted pages from disk. - self.reopen_conn() - - cursor = self.session.open_cursor(uri, None) - r.seed(0) - for idx in xrange(1,self.nrecords): - key = str(idx) - cursor.set_key(key) - self.assertEqual(cursor.search(), 0) - self.assertEquals(cursor.get_value(), self.getvalue(r, n)) - cursor.close() - -if __name__ == '__main__': - wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_log03.py b/src/third_party/wiredtiger/test/suite/test_log03.py new file mode 100755 index 00000000000..e93232a14aa --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_log03.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2018 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import os, shutil +import helper, wiredtiger, wttest +from wiredtiger import stat + +# test_log03.py +# test configuration for log.dirty_max +class test_log03(wttest.WiredTigerTestCase): + """ + Test log.dirty_max + """ + + homedir = 'HOME' + uri = 'table:test_log03' + nentries = 20000 + + # Tests need to setup the connection in their own way. + def setUpConnectionOpen(self, dir): + return None + + def setUpSessionOpen(self, conn): + return None + + def populate(self): + big_str = 'A' * 10000 + self.session.create(self.uri, "key_format=S,value_format=S") + cursor = self.session.open_cursor(self.uri) + for i in range(self.nentries): + cursor[str(i)] = big_str + cursor.close() + + def fsync_stat(self): + cursor = self.session.open_cursor('statistics:', None, None) + result = cursor[stat.conn.fsync_io][2] + cursor.close() + return result + + def with_log_sync(self, log_size, dirty_pct): + config = "cache_size=1G,create,statistics=(fast),log=(enabled" + config += ",file_max=" + str(log_size) + "M" + config += ",os_cache_dirty_pct=" + str(dirty_pct) + config += "),transaction_sync=(enabled=false,method=none)" + #self.tty('CONFIG: ' + config) + + # Recreate a home directory each time so we have the log + # starting at zero. That makes our calculations easier. + shutil.rmtree(self.homedir, ignore_errors=True) + os.mkdir(self.homedir) + self.conn = self.wiredtiger_open(self.homedir, config) + self.session = self.conn.open_session(None) + self.populate() + result = self.fsync_stat() + self.session.close() + self.conn.close() + return result + + def test_dirty_max(self): + # With this test, we have a baseline of syncs performed for 12M + # log files. Then we set dirty_max to values that are half, + # a third, a quarter and a fifth of the log file, and we would + # expect an increase of syncs each time. The number of syncs + # produced turns out to be a little variable, so we've picked + # conservative increases. + baseline = self.with_log_sync(12, 0) + #self.tty('baseline: ' + str(baseline)) + + for dirty_pct,increase in [50, 8], [33, 16], [25, 24], [20, 32]: + result = self.with_log_sync(12, dirty_pct) + #self.tty('tried: ' + str(dirty_pct) + ', got: ' + str(result)) + self.assertGreater(result, baseline + increase) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_stat08.py b/src/third_party/wiredtiger/test/suite/test_stat08.py new file mode 100644 index 00000000000..96f0b29b04e --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_stat08.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2018 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest + +# test_stat08.py +# Session statistics for bytes read into the cache. +class test_stat08(wttest.WiredTigerTestCase): + + nentries = 350000 + conn_config = 'cache_size=50MB,statistics=(all)' + entry_value = "abcde" * 40 + BYTES_READ = 4000 + READ_TIME = 4003 + session_stats = { BYTES_READ : "session: bytes read into cache", \ + READ_TIME : "session: page read from disk to cache time (usecs)"} + + def check_stats(self, cur, k): + exp_desc = self.session_stats[k] + cur.set_key(k) + cur.search() + [desc, pvalue, value] = cur.get_values() + self.pr(' stat: \'%s\', \'%s\', \'%s\'' % (desc, pvalue, str(value))) + self.assertEqual(desc, exp_desc ) + if k is self.BYTES_READ or k is self.READ_TIME: + self.assertTrue(value > 0) + + def test_session_stats(self): + self.session = self.conn.open_session() + self.session.create("table:test_stat08", + "key_format=i,value_format=S") + cursor = self.session.open_cursor('table:test_stat08', None, None) + # Write the entries. + for i in range(0, self.nentries): + cursor[i] = self.entry_value + cursor.reset() + + # Read the entries. + i = 0 + for key, value in cursor: + i = i + 1 + cursor.reset() + + # Now check the session statistics for bytes read into the cache. + stat_cur = self.session.open_cursor('statistics:session', None, None) + for k in self.session_stats: + self.check_stats(stat_cur, k) + + # Session stats cursor reset should set all the stats values to zero. + stat_cur.reset() + while stat_cur.next() == 0: + [desc, pvalue, value] = stat_cur.get_values() + self.assertTrue(value == 0) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/thread/Makefile.am b/src/third_party/wiredtiger/test/thread/Makefile.am index 58b715d4a80..182effa3201 100644 --- a/src/third_party/wiredtiger/test/thread/Makefile.am +++ b/src/third_party/wiredtiger/test/thread/Makefile.am @@ -12,4 +12,4 @@ t_LDFLAGS = -static TESTS = smoke.sh clean-local: - rm -rf WT_TEST __stats *.core + rm -rf WT_TEST __stats core.* *.core diff --git a/src/third_party/wiredtiger/test/utility/misc.c b/src/third_party/wiredtiger/test/utility/misc.c index 2cc7ad8a94b..8d5605208cf 100644 --- a/src/third_party/wiredtiger/test/utility/misc.c +++ b/src/third_party/wiredtiger/test/utility/misc.c @@ -215,6 +215,21 @@ testutil_is_flag_set(const char *flag) return (enable_long_tests); } +/* + * testutil_print_command_line -- + * Print command line arguments for csuite tests. + */ +void +testutil_print_command_line(int argc, char * const *argv) +{ + int i; + + printf("Running test command: "); + for (i = 0; i < argc; i++) + printf("%s ", argv[i]); + printf("\n"); +} + #ifndef _WIN32 /* * testutil_sleep_wait -- diff --git a/src/third_party/wiredtiger/test/utility/parse_opts.c b/src/third_party/wiredtiger/test/utility/parse_opts.c index 0bd724528c1..5bc92633f79 100644 --- a/src/third_party/wiredtiger/test/utility/parse_opts.c +++ b/src/third_party/wiredtiger/test/utility/parse_opts.c @@ -46,6 +46,8 @@ testutil_parse_opts(int argc, char * const *argv, TEST_OPTS *opts) opts->progname = testutil_set_progname(argv); + testutil_print_command_line(argc, argv); + while ((ch = __wt_getopt(opts->progname, argc, argv, "A:dh:n:o:pR:T:t:vW:")) != EOF) switch (ch) { diff --git a/src/third_party/wiredtiger/test/utility/test_util.h b/src/third_party/wiredtiger/test/utility/test_util.h index d321452c494..e756f5b4225 100644 --- a/src/third_party/wiredtiger/test/utility/test_util.h +++ b/src/third_party/wiredtiger/test/utility/test_util.h @@ -249,6 +249,7 @@ void testutil_cleanup(TEST_OPTS *); bool testutil_is_flag_set(const char *); void testutil_make_work_dir(const char *); int testutil_parse_opts(int, char * const *, TEST_OPTS *); +void testutil_print_command_line(int argc, char * const *argv); void testutil_progress(TEST_OPTS *, const char *); #ifndef _WIN32 void testutil_sleep_wait(uint32_t, pid_t); |