diff options
| author | Luke Chen <luke.chen@mongodb.com> | 2021-11-15 13:57:48 +1100 |
|---|---|---|
| committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2021-11-15 03:26:55 +0000 |
| commit | 810151fafee6c4b648a19c24c5ab9416a6cc5d17 (patch) | |
| tree | 5765f5bfbe265196f1d5a3f849e45f2cd4eaa12a | |
| parent | 1ffa2d6dd11f62fba37669b15f39b98fc782f4c3 (diff) | |
| download | mongo-810151fafee6c4b648a19c24c5ab9416a6cc5d17.tar.gz | |
Import wiredtiger: 6f517e60fbc9c8e33032e75d683f53ae62ca531c from branch mongodb-5.0
ref: 8fc3ec7e84..6f517e60fb
for: 5.0.5
WT-3445 Add multiple tables to format tester.
WT-5008 Migrate Jenkins "wiredtiger-perf-btree" job to Evergreen
WT-5010 Migrate Jenkins "wiredtiger-perf-checkpoint" job to Evergreen
WT-5011 Migrate Jenkins "wiredtiger-perf-log-consolidated" job to Evergreen
WT-5012 Migrate wiredtiger-perf-evict to Evergreen
WT-5013 Migrate Jenkins "wiredtiger-perf-stress" job to Evergreen
WT-5580 Dump the wtperf command into logs in Evergreen wtperf test
WT-6001 Avoid reading the page into cache if it needs to be rewritten
WT-6022 NVRAM cache
WT-6116 Turn non-timestamp testing back on
WT-7694 Fixes to have bucket prefixes properly used in object names.
WT-7820 Retrieve the on-disk durable timestamp to compare with newer update timestamp
WT-7845 Add an oldest_id for tiered and its metadata.
WT-7912 Fix prefix search near optimisation to handle scenarios where the key range is split across pages.
WT-8004 Create a read order for the architecture guide
WT-8030 Add sanity checks related to eviction trigger setting
WT-8046 Keep non persistent configuration settings between wiredtiger_open calls in test/format
WT-8065 Update documentation about cursor and prefix_key configuration
WT-8076 Modify tiered_abort csuite test to work with cmake
WT-8114 Revert allow setting the prepare timestamp smaller than or equal to the latest active read timestamp with roundup prepare config
WT-8151 Make durable_start_ts equal to stop_ts if stop_ts is less than durable_start_ts and greater than start_ts
WT-8157 Fix format-abort-recovery-stress-test timeout condition
WT-8163 Consider more eviction scenarios to give up checkpoint-cleanup
WT-8167 Remove unused getters from the cppsuite code
WT-8173 Fix row column store node in architecture guide diagram
WT-8178 Push JSON output generated from wtperf_run.py to Cedar/Evergreen
WT-8179 Push out.json generated from wtperf_run.py to Atlas
WT-8188 Use compact progress stats in compact related tests
WT-8189 Add useful information for compact analysis to WT verbose messaging
WT-8192 Extend the verbose API to support the assignment of verbosity levels per event category
WT-8194 Fix ASAN leaks when tcmalloc is enabled
WT-8196 Add warning options to the cppsuite files
WT-8221 Compare write generation number before performing RTS
WT-8224 Fix coverity complains about unused variable
WT-8228 Add variable-length column store support to the compact tests
WT-8232 Fix hs18 test to use release evict cursor to evict the page
WT-8233 Fix data-validation-stress-test-checkpoint ignoring failures
WT-8237 Unassign NULL value to config after freeing, it stays unused.
WT-8241 Skip value return for largest key
WT-8253 Fix disk space issue in checkpoint stress tests
WT-8254 Implementation of new WiredTiger internal API to define verbose messages with an associated severity level
WT-8255 Create a Python suite test to assert legacy uses of the verbose interface still work as intended
WT-8256 Create new tests capturing different verbose configuration scenarios
WT-8270 Updating the time window clear obsolete stage of reconciliation to correctly consider global visibility.
WT-8271 Capture git branch and commit status in perf test result output
WT-8275 Streamline compact stats
WT-8280 Temporarily disable prefix assert
WT-8281 Fix out of order handling with history store stop timestamp
WT-8283 Use the new verbose API in compact files
WT-8284 Improve verbose logging for transactions when rollback is required
WT-8285 Simplify CMakes use of third party libraries
WT-8286 Create stress test for prefix search
WT-8291 Call _exit() instead of exit() without an exec
WT-8294 Push the results from performance tests to one collection
WT-8297 timestamp order check fired on logged table in recovery
WT-8298 Switch Evergreen performance tests to use ubuntu2004-large instances
WT-8314 Fix the block cache returning blocks for the wrong table
WT-8316 format backward compatibility mode fixes
WT-8317 Never store overflow keys on internal pages
WT-8318 Add format.sh support for a directory of CONFIG files
WT-8321 Update timeout for smoke tests to 60 minutes
WT-8331 change format to allow quote characters
WT-8335 Support compiling both a static and shared WiredTiger library in CMake
WT-8336 Disable unsafe-loop-optimizations flag when compiling c++ files
WT-8337 format direct I/O test fails to turn off backups
WT-8339 Add missing comma to wtperf monitor header
WT-8342 Coverity: CID 121074: UNINTENDED_INTEGER_DIVISION in src/support/float.c
WT-8345 API to configure cache_max_wait_ms at the session level
WT-8346 Migrate Jenkins "wiredtiger-perf-stress" remaining tests to Evergreen
WT-8347 Slipup in test_checkpoint modify changes
WT-8349 In-memory format runs can create a CONFIG with illegal cache size
WT-8350 Fix test/format problems where the wrong type of value was used to turn a configuration value off
WT-8354 Historic format builds used in compatibility runs need their previous syntax
WT-8355 Coverity analysis defect 121096: Uninitialized pointer read
WT-8357 Add more Evergreen expansions into the performance test output
WT-8358 Pass ops and args fields to wtperf via command line
WT-8359 Separate smoke tests from stress tests in Evergreen
WT-8363 Validate stats output by performance tests
WT-8364 Fix CMake bug when building cppsuite with TCMalloc
WT-8368 Fix verbose message output produced by '__wt_verbose_multi'
WT-8394 Revert Validate stats output by performance tests
234 files changed, 11265 insertions, 4917 deletions
diff --git a/src/third_party/wiredtiger/SConstruct b/src/third_party/wiredtiger/SConstruct index 006b0bfce08..6d971e22e98 100644 --- a/src/third_party/wiredtiger/SConstruct +++ b/src/third_party/wiredtiger/SConstruct @@ -32,6 +32,9 @@ AddOption("--enable-diagnostic", dest="diagnostic", action="store_true", default AddOption("--enable-lz4", dest="lz4", type="string", nargs=1, action="store", help="Use LZ4 compression") +AddOption("--enable-memkind", dest="memkind", type="string", nargs=1, action="store", + help="Enable support for Intel memkind library, needed for NVRAM block cache.") + AddOption("--enable-python", dest="lang-python", type="string", nargs=1, action="store", help="Build Python extension, specify location of swig.exe binary") @@ -113,6 +116,7 @@ useZlib = GetOption("zlib") useSnappy = GetOption("snappy") useLz4 = GetOption("lz4") useTcmalloc = GetOption("tcmalloc") +useMemkind = GetOption("memkind") wtlibs = [] conf = Configure(env) @@ -161,6 +165,16 @@ if useTcmalloc: print('tcmalloc.h must be installed!') Exit(1) +if useMemkind: + conf.env.Append(CPPPATH=[useMemkind + "/include"]) + conf.env.Append(LIBPATH=[useMemkind + "/lib"]) + if conf.CheckCHeader('memkind.h'): + conf.env.Append(CPPDEFINES=['HAVE_MEMKIND']) + wtlibs.append("memkind") + else: + print('memkind.h must be installed!') + Exit(1) + env = conf.Finish() # Configure build environment variables diff --git a/src/third_party/wiredtiger/bench/workgen/Makefile.am b/src/third_party/wiredtiger/bench/workgen/Makefile.am index 61512d65319..e635a746d6b 100644 --- a/src/third_party/wiredtiger/bench/workgen/Makefile.am +++ b/src/third_party/wiredtiger/bench/workgen/Makefile.am @@ -1,5 +1,7 @@ +# FIX-ME-WT-8263: Remove the next line to keep the flags defined in configure.ac.in +AM_CXXFLAGS = AM_CPPFLAGS = -I$(top_builddir) -AM_CPPFLAGS += -I$(top_srcdir)/src/include +AM_CPPFLAGS +=-I$(top_srcdir)/src/include AM_CPPFLAGS +=-I$(top_srcdir)/test/utility PYSRC = $(top_srcdir)/bench/workgen diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-50r50u-large.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-50r50u-large.wtperf new file mode 100644 index 00000000000..a829c7aafa0 --- /dev/null +++ b/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-50r50u-large.wtperf @@ -0,0 +1,26 @@ +# wtperf options file: simulate MongoDB. +# The configuration for the connection and table are from mongoDB. +# We use multiple tables to simulate collections and indexes. +# This test assumes that its correlating populate already completed and exists. +# +# Set cache to half of memory of AWS perf instance. Enable logging and +# checkpoints. Collect wiredtiger stats for ftdc. +conn_config="cache_size=16G,checkpoint=(wait=60,log_size=2GB),session_max=20000,log=(enabled),eviction=(threads_max=8)" +create=false +compression="snappy" +sess_config="isolation=snapshot" +table_config="type=file" +table_count=2 +# close_conn as false allows this test to close/finish faster, but if running +# as the set, the next test will need to run recovery. +close_conn=false +key_sz=40 +value_sz=120 +max_latency=2000 +pareto=20 +report_interval=10 +run_time=7200 +sample_interval=10 +sample_rate=1 +threads=((count=10,reads=1),(count=10,updates=1)) +warmup=120 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-populate-large.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-populate-large.wtperf new file mode 100644 index 00000000000..ffb621276ec --- /dev/null +++ b/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-populate-large.wtperf @@ -0,0 +1,26 @@ +# wtperf options file: simulate MongoDB. +# The configuration for the connection and table are from mongoDB. +# We use multiple tables to simulate collections and indexes. +# This test assumes that its correlating populate already completed and exists. +# +# Set cache to half of memory. This value is computed from the AWS instance +# where Jenkins perf runs which has 32G. Enable logging and checkpoints. +# Collect wiredtiger stats for ftdc. +# +# This generates about 80 Gb of uncompressed data. But it should compress +# well and be small on disk. +conn_config="cache_size=16G,checkpoint=(wait=60,log_size=2GB),session_max=20000,log=(enabled),eviction=(threads_max=8)" +compact=true +compression="snappy" +sess_config="isolation=snapshot" +table_config="internal_page_max=16K,type=file,leaf_page_max=16K,memory_page_max=10M,split_pct=90" +table_count=2 +icount=4000000000 +key_sz=40 +value_sz=120 +max_latency=2000 +populate_threads=20 +report_interval=10 +random_value=true +sample_interval=10 +sample_rate=1 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/checkpoint-stress-large.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/checkpoint-stress-large.wtperf new file mode 100644 index 00000000000..5d31fca97c8 --- /dev/null +++ b/src/third_party/wiredtiger/bench/wtperf/runners/checkpoint-stress-large.wtperf @@ -0,0 +1,27 @@ +# A stress configuration to create long running checkpoints while doing a lot +# of updates. +conn_config="cache_size=16GB,eviction=(threads_max=8),log=(enabled=false)" +table_config="leaf_page_max=32k,internal_page_max=16k,allocation_size=4k,split_pct=90,type=file" +# Enough data to fill the cache. 150 million 1k records results in two ~11GB +# tables +close_conn=false +icount=1500000000 +create=true +compression="snappy" +checkpoint_interval=60 +checkpoint_threads=1 +populate_threads=1 +report_interval=10 +# Run for a longer duration to ensure checkpoints are completing. +run_time=600 +# Sampling isn't being currently used by any automated test. Keeping it to be +# used in the future +sample_interval=10 +sample_rate=1 +# MongoDB always has multiple tables, and checkpoints behave differently when +# there is more than a single table. +table_count=2 +threads=((count=6,updates=1)) +value_sz=1000 +# Wait for the throughput to stabilize +warmup=120 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree-large-32GB.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree-large-32GB.wtperf new file mode 100644 index 00000000000..070e9903423 --- /dev/null +++ b/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree-large-32GB.wtperf @@ -0,0 +1,11 @@ +# wtperf options file: evict btree configuration +conn_config="cache_size=32G,eviction=(threads_max=8)" +table_config="type=file" +icount=1000000000 +report_interval=5 +run_time=120 +populate_threads=1 +threads=((count=16,reads=1)) +# Add throughput/latency monitoring +max_latency=2000 +sample_interval=5 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree-large.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree-large.wtperf new file mode 100644 index 00000000000..2db9e480b1a --- /dev/null +++ b/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree-large.wtperf @@ -0,0 +1,11 @@ +# wtperf options file: evict btree configuration +conn_config="cache_size=50M,eviction=(threads_max=8)" +table_config="type=file" +icount=1000000000 +report_interval=5 +run_time=120 +populate_threads=1 +threads=((count=16,reads=1)) +# Add throughput/latency monitoring +max_latency=2000 +sample_interval=5 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree-stress-multi-large.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree-stress-multi-large.wtperf new file mode 100644 index 00000000000..c964a463079 --- /dev/null +++ b/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree-stress-multi-large.wtperf @@ -0,0 +1,13 @@ +conn_config="cache_size=1G,eviction=(threads_max=8),session_max=2000" +table_config="type=file" +table_count=100 +close_conn=false +icount=1000000000 +report_interval=5 +run_time=600 +populate_threads=1 +threads=((count=100,updates=1,reads=4,ops_per_txn=30)) +# Warn if a latency over a quarter second is seen +max_latency=250 +sample_interval=5 +sample_rate=1 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/large-lsm-large.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/large-lsm-large.wtperf new file mode 100644 index 00000000000..c8a54198747 --- /dev/null +++ b/src/third_party/wiredtiger/bench/wtperf/runners/large-lsm-large.wtperf @@ -0,0 +1,11 @@ +# wtperf options file: large lsm configuration +conn_config="cache_size=20G,mmap=false,lsm_manager=(worker_thread_max=8)" +table_config="internal_page_max=128K,lsm=(bloom_config=(leaf_page_max=8MB),bloom_bit_count=28,bloom_hash_count=19,bloom_oldest=true,chunk_max=5GB,chunk_size=100MB),type=lsm,leaf_page_max=16K,os_cache_dirty_max=16MB" +compact=true +compression="snappy" +icount=4000000000 +report_interval=5 +run_time=1200 +populate_threads=1 +sample_interval=10 +threads=((count=16,reads=1)) diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/medium-btree-large-32GB.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/medium-btree-large-32GB.wtperf new file mode 100644 index 00000000000..ec8dc96cba4 --- /dev/null +++ b/src/third_party/wiredtiger/bench/wtperf/runners/medium-btree-large-32GB.wtperf @@ -0,0 +1,11 @@ +# wtperf options file: medium btree configuration +conn_config="cache_size=32G" +table_config="type=file" +icount=1000000000 +report_interval=5 +run_time=120 +populate_threads=1 +threads=((count=16,reads=1)) +# Add throughput/latency monitoring +max_latency=2000 +sample_interval=5 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/medium-btree-large.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/medium-btree-large.wtperf new file mode 100644 index 00000000000..2c1e30b3cee --- /dev/null +++ b/src/third_party/wiredtiger/bench/wtperf/runners/medium-btree-large.wtperf @@ -0,0 +1,11 @@ +# wtperf options file: medium btree configuration +conn_config="cache_size=1G" +table_config="type=file" +icount=1000000000 +report_interval=5 +run_time=120 +populate_threads=1 +threads=((count=16,reads=1)) +# Add throughput/latency monitoring +max_latency=2000 +sample_interval=5 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/multi-btree-zipfian-populate-large.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/multi-btree-zipfian-populate-large.wtperf new file mode 100644 index 00000000000..8ed4eeabe91 --- /dev/null +++ b/src/third_party/wiredtiger/bench/wtperf/runners/multi-btree-zipfian-populate-large.wtperf @@ -0,0 +1,19 @@ +# Create a set of tables with uneven distribution of data +conn_config="cache_size=1G,eviction=(threads_max=8),file_manager=(close_idle_time=100000),checkpoint=(wait=60,log_size=2GB),session_max=1000" +table_config="type=file" +table_count=100 +icount=0 +random_range=1000000000 +pareto=10 +range_partition=true +report_interval=5 + +run_ops=1000000000 +populate_threads=0 +icount=0 +threads=((count=20,inserts=1)) + +# Warn if a latency over 1 second is seen +max_latency=1000 +sample_interval=5 +sample_rate=1 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/overflow-130k-large.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/overflow-130k-large.wtperf new file mode 100644 index 00000000000..ec755029a91 --- /dev/null +++ b/src/third_party/wiredtiger/bench/wtperf/runners/overflow-130k-large.wtperf @@ -0,0 +1,18 @@ +# +# Run with very large overflow items and btree. +# +conn_config="cache_size=21G,checkpoint_sync=false,mmap=false" +compact=true +compression="snappy" +sess_config="isolation=snapshot" +table_config="internal_page_max=128K,type=file,leaf_page_max=16K,os_cache_dirty_max=16MB,leaf_value_max=32K" +icount=1500000 +key_sz=40 +value_sz=130000 +max_latency=2000 +populate_threads=1 +report_interval=5 +random_value=true +run_time=300 +threads=((count=10,read=1),(count=10,update=1)) +warmup=30 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/update-btree.json b/src/third_party/wiredtiger/bench/wtperf/runners/update-btree.json new file mode 100644 index 00000000000..bc0dc9abdeb --- /dev/null +++ b/src/third_party/wiredtiger/bench/wtperf/runners/update-btree.json @@ -0,0 +1,14 @@ +[ + { + "arguments": ["-o threads=((count=2,inserts=1,throttle=20000),(count=2,reads=1),(count=2,updates=1,throttle=20000))"], + "operations": [ "read", "load" ] + }, + { + "arguments": ["-o threads=((count=2,inserts=1,throttle=20000),(count=2,reads=1,throttle=20000),(count=2,updates=1))"], + "operations": [ "update" ] + }, + { + "arguments": ["-o threads=((count=2,inserts=1),(count=2,reads=1,throttle=20000),(count=2,updates=1,throttle=20000))"], + "operations": [ "insert" ] + } +] diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/update-checkpoint-btree-large.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/update-checkpoint-btree-large.wtperf new file mode 100644 index 00000000000..2a45b80b981 --- /dev/null +++ b/src/third_party/wiredtiger/bench/wtperf/runners/update-checkpoint-btree-large.wtperf @@ -0,0 +1,15 @@ +# wtperf options file: btree with inserts/updates and checkpointing, in memory +# Note: The cache needs to be sized to approximate the amount of inserts +# that will happen during the given run_time. +conn_config="cache_size=25G" +table_config="type=file" +icount=500000000 +report_interval=5 +checkpoint_threads=1 +checkpoint_interval=10 +run_time=300 +populate_threads=1 +threads=((count=1,inserts=1),(count=2,reads=1),(count=2,updates=1)) +# Add throughput/latency monitoring +max_latency=2000 +sample_interval=5 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/update-checkpoint.json b/src/third_party/wiredtiger/bench/wtperf/runners/update-checkpoint.json new file mode 100644 index 00000000000..9cb54f2cd6a --- /dev/null +++ b/src/third_party/wiredtiger/bench/wtperf/runners/update-checkpoint.json @@ -0,0 +1,15 @@ +[ + { + "arguments": ["-o threads=((count=1,inserts=1,throttle=20000),(count=2,reads=1),(count=2,updates=1,throttle=20000))"], + "operations": [ "read", "load" ] + }, + { + "arguments": ["-o threads=((count=1,inserts=1,throttle=20000),(count=2,reads=1,throttle=20000),(count=2,updates=1))"], + "operations": [ "update" ] + }, + { + "arguments": ["-o threads=((count=1,inserts=1),(count=2,reads=1,throttle=20000),(count=2,updates=1,throttle=20000))"], + "operations": [ "insert" ] + } +] + diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/update-delta-mix1-large.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/update-delta-mix1-large.wtperf new file mode 100644 index 00000000000..05e72b3bd6c --- /dev/null +++ b/src/third_party/wiredtiger/bench/wtperf/runners/update-delta-mix1-large.wtperf @@ -0,0 +1,18 @@ +# wtperf options file: Mixed workload where we grow some values and shrink +# others. Mixed load leaning toward growing the dataset. +# +conn_config="cache_size=2GB,checkpoint=(wait=30)" +table_config="type=file,leaf_page_max=32k,leaf_value_max=128k,split_pct=90" +# The values are starting small, insert a lot so our database grows larger than +# cache quickly. +icount=6000000 +report_interval=5 +run_time=300 +populate_threads=1 +# +# Run more grow workload threads than shrink threads. +# +threads=((count=4,update=1,update_delta=100),(count=2,update=1,update_delta=-150)) +value_sz=20000 +value_sz_min=1000 +value_sz_max=65536 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/update-grow-stress-large.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/update-grow-stress-large.wtperf new file mode 100644 index 00000000000..5c695759dcb --- /dev/null +++ b/src/third_party/wiredtiger/bench/wtperf/runners/update-grow-stress-large.wtperf @@ -0,0 +1,15 @@ +# wtperf options file: Grow the size of documents while there is cache +# pressure and appends are happening as well. +conn_config="cache_size=2GB,checkpoint=(wait=30)" +table_config="type=file,leaf_page_max=32k,leaf_value_max=128k,split_pct=90" +# The values are starting small, insert a lot so our database grows larger than +# cache quickly. +icount=10000000 +report_interval=5 +run_time=240 +populate_threads=1 +# Continue inserting new records. +threads=((count=1,inserts=1,throttle=1000),(count=4,update=1,update_delta=100)) +# Start with small values and let them grow slowly to large values. +value_sz=10000 +value_sz_max=65536 diff --git a/src/third_party/wiredtiger/bench/wtperf/wtperf.c b/src/third_party/wiredtiger/bench/wtperf/wtperf.c index 6e16da0a0d7..39a83417e0d 100644 --- a/src/third_party/wiredtiger/bench/wtperf/wtperf.c +++ b/src/third_party/wiredtiger/bench/wtperf/wtperf.c @@ -296,7 +296,7 @@ do_range_reads(WTPERF_THREAD *thread, WT_CURSOR *cursor, int64_t read_range) } /* pre_load_data -- - * Pull everything into cache before starting the workload phase. + * Pull everything into cache before starting the workload phase. */ static void pre_load_data(WTPERF *wtperf) @@ -1088,7 +1088,7 @@ monitor(void *arg) "insert maximum latency(uS)," "modify average latency(uS)," "modify min latency(uS)," - "modify maximum latency(uS)" + "modify maximum latency(uS)," "read average latency(uS)," "read minimum latency(uS)," "read maximum latency(uS)," diff --git a/src/third_party/wiredtiger/bench/wtperf/wtperf.h b/src/third_party/wiredtiger/bench/wtperf/wtperf.h index 14b5d55a2ed..293e67826a4 100644 --- a/src/third_party/wiredtiger/bench/wtperf/wtperf.h +++ b/src/third_party/wiredtiger/bench/wtperf/wtperf.h @@ -40,19 +40,32 @@ typedef struct __wtperf WTPERF; typedef struct __wtperf_thread WTPERF_THREAD; typedef struct __truncate_queue_entry TRUNCATE_QUEUE_ENTRY; +#ifndef LZ4_PATH +#define LZ4_PATH "lz4/.libs/libwiredtiger_lz4.so" +#endif +#ifndef SNAPPY_PATH +#define SNAPPY_PATH "snappy/.libs/libwiredtiger_snappy.so" +#endif +#ifndef ZLIB_PATH +#define ZLIB_PATH "zlib/.libs/libwiredtiger_zlib.so" +#endif +#ifndef ZSTD_PATH +#define ZSTD_PATH "zstd/.libs/libwiredtiger_zstd.so" +#endif + #define EXT_PFX ",extensions=(" #define EXT_SFX ")" #define EXTPATH "../../ext/compressors/" /* Extensions path */ #define BLKCMP_PFX "block_compressor=" #define LZ4_BLK BLKCMP_PFX "lz4" -#define LZ4_EXT EXT_PFX EXTPATH "lz4/.libs/libwiredtiger_lz4.so" EXT_SFX +#define LZ4_EXT EXT_PFX EXTPATH LZ4_PATH EXT_SFX #define SNAPPY_BLK BLKCMP_PFX "snappy" -#define SNAPPY_EXT EXT_PFX EXTPATH "snappy/.libs/libwiredtiger_snappy.so" EXT_SFX +#define SNAPPY_EXT EXT_PFX EXTPATH SNAPPY_PATH EXT_SFX #define ZLIB_BLK BLKCMP_PFX "zlib" -#define ZLIB_EXT EXT_PFX EXTPATH "zlib/.libs/libwiredtiger_zlib.so" EXT_SFX +#define ZLIB_EXT EXT_PFX EXTPATH ZLIB_PATH EXT_SFX #define ZSTD_BLK BLKCMP_PFX "zstd" -#define ZSTD_EXT EXT_PFX EXTPATH "zstd/.libs/libwiredtiger_zstd.so" EXT_SFX +#define ZSTD_EXT EXT_PFX EXTPATH ZSTD_PATH EXT_SFX #define MAX_MODIFY_PCT 10 #define MAX_MODIFY_NUM 16 diff --git a/src/third_party/wiredtiger/bench/wtperf/wtperf_run_py/perf_stat.py b/src/third_party/wiredtiger/bench/wtperf/wtperf_run_py/perf_stat.py index de62d328373..53e5f5af034 100644 --- a/src/third_party/wiredtiger/bench/wtperf/wtperf_run_py/perf_stat.py +++ b/src/third_party/wiredtiger/bench/wtperf/wtperf_run_py/perf_stat.py @@ -1,45 +1,149 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +import glob +import json +import re + + class PerfStat: def __init__(self, short_label: str, - pattern: str, - input_offset: int, output_label: str, + input_offset: int = 0, output_precision: int = 0, + pattern: str = None, + stat_file: str = 'test.stat', conversion_function=int): self.short_label: str = short_label - self.pattern: str = pattern - self.input_offset: int = input_offset self.output_label: str = output_label + self.input_offset: int = input_offset self.output_precision: int = output_precision + self.pattern: str = pattern + self.stat_file = stat_file self.conversion_function = conversion_function self.values = [] - def add_value(self, value): - converted_value = self.conversion_function(value) - self.values.append(converted_value) - - def get_num_values(self): - return len(self.values) - - def get_average(self): - num_values = len(self.values) - total = sum(self.values) - average = self.conversion_function(total / num_values) - return average - - def get_skipminmax_average(self): - num_values = len(self.values) - assert num_values >= 3 - minimum = min(self.values) - maximum = max(self.values) - total = sum(self.values) - total_skipminmax = total - maximum - minimum - num_values_skipminmax = num_values - 2 - skipminmax_average = self.conversion_function(total_skipminmax / num_values_skipminmax) - return skipminmax_average - - def get_core_average(self): + def add_values(self, values: list): + for val in values: + converted_value = self.conversion_function(val) + self.values.append(converted_value) + + def find_stat(self, test_stat_path: str): + matches = [] + for line in open(test_stat_path): + match = re.search(self.pattern, line) + if match: + matches.append(float(line.split()[self.input_offset])) + return matches + + def average(self, vals): + return self.conversion_function(sum(vals) / len(vals)) + + def get_value(self): + """Return the average of all gathered values""" if len(self.values) >= 3: - return self.get_skipminmax_average() + drop_min_and_max = sorted(self.values)[1:-1] + return self.average(drop_min_and_max) else: - return self.get_average() + return self.average(self.values) + + def get_value_list(self, brief: bool): + as_dict = { + 'name': self.output_label, + 'value': self.get_value() + } + if not brief: + as_dict['values'] = self.values + return [as_dict] + + def are_values_all_zero(self): + result = True + for value in self.values: + if value != 0: + result = False + return result + + +class PerfStatMin(PerfStat): + def get_value(self): + """Return the averaged minimum of all gathered values""" + min_3_vals = sorted(self.values)[:3] + return self.average(min_3_vals) + + +class PerfStatMax(PerfStat): + def get_value(self): + """Return the averaged maximum of all gathered values""" + max_3_vals = sorted(self.values)[-3:] + return self.average(max_3_vals) + + +class PerfStatCount(PerfStat): + def find_stat(self, test_stat_path: str): + """Return the total number of times a pattern matched""" + total = 0 + test_stat_path = glob.glob(test_stat_path)[0] + for line in open(test_stat_path): + match = re.search(self.pattern, line) + if match: + total += 1 + return [total] + + +class PerfStatLatency(PerfStat): + def __init__(self, short_label: str, stat_file: str, output_label: str, num_max: int): + super().__init__(short_label=short_label, + stat_file=stat_file, + output_label=output_label) + self.num_max = num_max + + def find_stat(self, test_stat_path: str): + values = [] + for line in open(test_stat_path): + as_dict = json.loads(line) + values.append(as_dict["wtperf"]["read"]["max latency"]) + values.append(as_dict["wtperf"]["update"]["max latency"]) + return values + + def get_value(self, nth_max: int): + """Return the nth maximum number from all the gathered values""" + return sorted(self.values)[-nth_max] + + def get_value_list(self, brief: bool): + as_list = [] + for i in range(1, self.num_max + 1): + as_dict = { + 'name': self.output_label + str(i), + 'value': self.get_value(i) + } + if not brief: + as_dict['values'] = self.values + as_list.append(as_dict) + return as_list diff --git a/src/third_party/wiredtiger/bench/wtperf/wtperf_run_py/perf_stat_collection.py b/src/third_party/wiredtiger/bench/wtperf/wtperf_run_py/perf_stat_collection.py index b26c66a020b..d62441f69a0 100644 --- a/src/third_party/wiredtiger/bench/wtperf/wtperf_run_py/perf_stat_collection.py +++ b/src/third_party/wiredtiger/bench/wtperf/wtperf_run_py/perf_stat_collection.py @@ -1,13 +1,40 @@ -import re +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import os from perf_stat import PerfStat +from typing import List -def find_stat(test_stat_path: str, pattern: str, position_of_value: int): - for line in open(test_stat_path): - match = re.match(pattern, line) - if match: - return float(line.split()[position_of_value]) - return 0 +def create_test_stat_path(test_home_path: str, test_stats_file: str): + return os.path.join(test_home_path, test_stats_file) class PerfStatCollection: @@ -17,23 +44,17 @@ class PerfStatCollection: def add_stat(self, perf_stat: PerfStat): self.perf_stats[perf_stat.short_label] = perf_stat - def find_stats(self, test_stat_path: str): + def find_stats(self, test_home: str, operations: List[str]): for stat in self.perf_stats.values(): - value = find_stat(test_stat_path=test_stat_path, - pattern=stat.pattern, - position_of_value=stat.input_offset) - stat.add_value(value=value) + if not operations or stat.short_label in operations: + test_stat_path = create_test_stat_path(test_home, stat.stat_file) + values = stat.find_stat(test_stat_path=test_stat_path) + stat.add_values(values=values) - def to_value_list(self): - as_list = [] + def to_value_list(self, brief: bool): + stats_list = [] for stat in self.perf_stats.values(): - as_list.append({ - 'name': stat.output_label, - 'value': stat.get_core_average(), - 'values': stat.values - }) - return as_list - - def to_dict(self): - as_dict = {'metrics': self.to_value_list()} - return as_dict + if not stat.are_values_all_zero(): + stat_list = stat.get_value_list(brief = brief) + stats_list.extend(stat_list) + return stats_list diff --git a/src/third_party/wiredtiger/bench/wtperf/wtperf_run_py/validate_expected_stats.py b/src/third_party/wiredtiger/bench/wtperf/wtperf_run_py/validate_expected_stats.py new file mode 100644 index 00000000000..ce92029efe2 --- /dev/null +++ b/src/third_party/wiredtiger/bench/wtperf/wtperf_run_py/validate_expected_stats.py @@ -0,0 +1,76 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import argparse +import json +import os +import re + + +def main(): + """ + Validate the expected output from the last run of wtperf_run.py + Take in a list of stats and their values as json and verify them against the contents of evergreen_out.json. + Example usage: + ./validate_perf_stats.py '{"Pages seen by eviction": 200, "Insert count": 1153326}' './evergreen_out.json' + """ + + parser = argparse.ArgumentParser() + parser.add_argument('expected_stats', type=str, help='The expected stats and their values') + parser.add_argument('stat_file', type=str, help='The evergreen stat file produced by the perf test') + args = parser.parse_args() + + stat_file_regex = re.compile(r"evergreen_out[\w-]*\.json") + if not re.match(stat_file_regex, os.path.basename(args.stat_file)): + print(f"ERROR: '{args.stat_file}' should be the path to an evergreen_out*.json file") + exit(1) + + expected_stats = json.loads(args.expected_stats) + + stat_file = json.load(open(args.stat_file, 'r')) + test_output = {stat['name']: stat['value'] for stat in stat_file[0]["metrics"]} + + errors = [] + for stat in expected_stats: + if stat not in test_output: + errors.append(f"'{stat}'\t not present in evergreen_out.json") + elif expected_stats[stat] != test_output[stat]: + errors.append(f"'{stat}'\t value mismatch. Expected: {expected_stats[stat]}, Actual: {test_output[stat]}") + + if errors: + print("ERROR: Expected values not found:") + print('\n'.join(errors)) + exit(1) + else: + print("Expected stats and values found") + + +if __name__ == '__main__': + main() diff --git a/src/third_party/wiredtiger/bench/wtperf/wtperf_run_py/wtperf_config.py b/src/third_party/wiredtiger/bench/wtperf/wtperf_run_py/wtperf_config.py index d2b70dff2dc..5f6861a0d5d 100644 --- a/src/third_party/wiredtiger/bench/wtperf/wtperf_run_py/wtperf_config.py +++ b/src/third_party/wiredtiger/bench/wtperf/wtperf_run_py/wtperf_config.py @@ -1,23 +1,70 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + class WTPerfConfig: def __init__(self, wtperf_path: str, home_dir: str, test: str, + batch_file: str = None, + arguments=None, + operations=None, environment: str = None, run_max: int = 1, - verbose: bool = False): + verbose: bool = False, + git_root: str = None, + json_info=None): + if json_info is None: + json_info = {} self.wtperf_path: str = wtperf_path self.home_dir: str = home_dir self.test: str = test + self.batch_file = batch_file + self.arguments = arguments + self.operations = operations self.environment: str = environment self.run_max: int = run_max self.verbose: bool = verbose + self.git_root: str = git_root + self.json_info: dict = json_info def to_value_dict(self): as_dict = {'wt_perf_path': self.wtperf_path, 'test': self.test, + 'batch_file': self.batch_file, + 'arguments': self.arguments, + 'operations': self.operations, 'home_dir': self.home_dir, 'environment': self.environment, 'run_max': self.run_max, - 'verbose': self.verbose} + 'verbose': self.verbose, + 'git_root': self.git_root, + 'json_info': self.json_info} return as_dict diff --git a/src/third_party/wiredtiger/bench/wtperf/wtperf_run_py/wtperf_run.py b/src/third_party/wiredtiger/bench/wtperf/wtperf_run_py/wtperf_run.py index fb1b1f18d6c..6683f92d18c 100644 --- a/src/third_party/wiredtiger/bench/wtperf/wtperf_run_py/wtperf_run.py +++ b/src/third_party/wiredtiger/bench/wtperf/wtperf_run_py/wtperf_run.py @@ -1,38 +1,84 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + import argparse -import json import os.path -import re -import subprocess -import sys import platform import psutil - -from wtperf_config import WTPerfConfig -from perf_stat import PerfStat +import subprocess +import sys +import json +from perf_stat import PerfStat, PerfStatCount, PerfStatLatency, PerfStatMax, PerfStatMin from perf_stat_collection import PerfStatCollection +from pygit2 import discover_repository, Repository +from pygit2 import GIT_SORT_NONE +from typing import List +from wtperf_config import WTPerfConfig -# the 'test.stat' file is where wt-perf.c writes out it's statistics -# (within the directory specified by the 'home' parameter) -test_stats_file = 'test.stat' +def create_test_home_path(home: str, test_run: int, operations: List[str] = None): + home_path = "{}_{}".format(home, test_run) + if operations: + # Use the first operation name as part of the home path + home_path += "_{}".format(operations[0]) + return home_path -def create_test_home_path(home: str, test_run: int): - return '{}_{}'.format(home, test_run) +def get_git_info(git_working_tree_dir): + repository_path = discover_repository(git_working_tree_dir) + assert repository_path is not None -def create_test_stat_path(test_home_path: str): - return os.path.join(test_home_path, test_stats_file) + repo = Repository(repository_path) + commits = list(repo.walk(repo.head.target, GIT_SORT_NONE)) + head_commit = commits[0] + diff = repo.diff() + git_info = { + 'head_commit': { + 'hash': head_commit.hex, + 'message': head_commit.message, + 'author': head_commit.author.name + }, + 'branch': { + 'name': repo.head.shorthand + }, + 'stats': { + 'files_changed': diff.stats.files_changed, + }, + 'num_commits': len(commits) + } -def find_stat(test_stat_path: str, pattern: str, position_of_value: int): - for line in open(test_stat_path): - match = re.match(pattern, line) - if match: - return line.split()[position_of_value] - return 0 + return git_info -def construct_wtperf_command_line(wtperf: str, env: str, test: str, home: str): +def construct_wtperf_command_line(wtperf: str, env: str, test: str, home: str, arguments: List[str]): command_line = [] if env is not None: command_line.append(env) @@ -40,41 +86,73 @@ def construct_wtperf_command_line(wtperf: str, env: str, test: str, home: str): if test is not None: command_line.append('-O') command_line.append(test) + if arguments is not None: + command_line.extend(arguments) if home is not None: command_line.append('-h') command_line.append(home) return command_line -def run_test(config: WTPerfConfig, test_run: int): - test_home = create_test_home_path(home=config.home_dir, test_run=test_run) +def brief_perf_stats(config: WTPerfConfig, perf_stats: PerfStatCollection): + as_list = [{ + "info": { + "test_name": os.path.basename(config.test) + }, + "metrics": perf_stats.to_value_list(brief=True) + }] + return as_list + + +def detailed_perf_stats(config: WTPerfConfig, perf_stats: PerfStatCollection): + total_memory_gb = psutil.virtual_memory().total / (1024 * 1024 * 1024) + as_dict = { + 'Test Name': os.path.basename(config.test), + 'config': config.to_value_dict(), + 'metrics': perf_stats.to_value_list(brief=False), + 'system': { + 'cpu_physical_cores': psutil.cpu_count(logical=False), + 'cpu_logical_cores': psutil.cpu_count(), + 'total_physical_memory_gb': total_memory_gb, + 'platform': platform.platform() + } + } + + if config.git_root: + as_dict['git'] = get_git_info(config.git_root) + + return as_dict + + +def run_test_wrapper(config: WTPerfConfig, operations: List[str] = None, arguments: List[str] = None): + for test_run in range(config.run_max): + print("Starting test {}".format(test_run)) + run_test(config=config, test_run=test_run, operations=operations, arguments=arguments) + print("Completed test {}".format(test_run)) + + +def run_test(config: WTPerfConfig, test_run: int, operations: List[str] = None, arguments: List[str] = None): + test_home = create_test_home_path(home=config.home_dir, test_run=test_run, operations=operations) command_line = construct_wtperf_command_line( wtperf=config.wtperf_path, env=config.environment, + arguments=arguments, test=config.test, home=test_home) - # print('Command Line for test: {}'.format(command_line)) - subprocess.run(command_line) + try: + subprocess.run(command_line, check=True, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, + universal_newlines=True) + except subprocess.CalledProcessError as cpe: + print("Error: {}".format(cpe.output)) + exit(1) -def process_results(config: WTPerfConfig, perf_stats: PerfStatCollection): +def process_results(config: WTPerfConfig, perf_stats: PerfStatCollection, operations: List[str] = None): for test_run in range(config.run_max): - test_home = create_test_home_path(home=config.home_dir, test_run=test_run) - test_stats_path = create_test_stat_path(test_home) + test_home = create_test_home_path(home=config.home_dir, test_run=test_run, operations=operations) if config.verbose: - print('Reading test stats file: {}'.format(test_stats_path)) - perf_stats.find_stats(test_stat_path=test_stats_path) - - total_memory_gb = psutil.virtual_memory().total / (1024 * 1024 * 1024) - as_dict = {'config': config.to_value_dict(), - 'metrics': perf_stats.to_value_list(), - 'system': { - 'cpu_physical_cores': psutil.cpu_count(logical=False), - 'cpu_logical_cores': psutil.cpu_count(), - 'total_physical_memory_gb': total_memory_gb, - 'platform': platform.platform()} - } - return as_dict + print('Reading stats from {} directory.'.format(test_home)) + perf_stats.find_stats(test_home=test_home, operations=operations) def setup_perf_stats(): @@ -82,29 +160,48 @@ def setup_perf_stats(): perf_stats.add_stat(PerfStat(short_label="load", pattern='Load time:', input_offset=2, - output_label='Load time:', + output_label='Load time', output_precision=2, conversion_function=float)) perf_stats.add_stat(PerfStat(short_label="insert", pattern=r'Executed \d+ insert operations', input_offset=1, - output_label='Insert count:')) + output_label='Insert count')) perf_stats.add_stat(PerfStat(short_label="modify", pattern=r'Executed \d+ modify operations', input_offset=1, - output_label='Modify count:')) + output_label='Modify count')) perf_stats.add_stat(PerfStat(short_label="read", pattern=r'Executed \d+ read operations', input_offset=1, - output_label='Read count:')) + output_label='Read count')) perf_stats.add_stat(PerfStat(short_label="truncate", pattern=r'Executed \d+ truncate operations', input_offset=1, - output_label='Truncate count:')) + output_label='Truncate count')) perf_stats.add_stat(PerfStat(short_label="update", pattern=r'Executed \d+ update operations', input_offset=1, - output_label='Update count:')) + output_label='Update count')) + perf_stats.add_stat(PerfStatMax(short_label="max_update_throughput", + pattern=r'updates,', + input_offset=8, + output_label='Max update throughput')) + perf_stats.add_stat(PerfStatMin(short_label="min_update_throughput", + pattern=r'updates,', + input_offset=8, + output_label='Min update throughput')) + perf_stats.add_stat(PerfStatCount(short_label="warnings", + pattern='WARN', + output_label='Warnings')) + perf_stats.add_stat(PerfStatLatency(short_label="max_latencies", + stat_file='monitor.json', + output_label='Latency Max', + num_max = 5)) + perf_stats.add_stat(PerfStatCount(short_label="eviction_page_seen", + stat_file='WiredTigerStat*', + pattern='[0-9].wt cache: pages seen by eviction', + output_label='Pages seen by eviction')) return perf_stats @@ -114,12 +211,18 @@ def main(): parser.add_argument('-e', '--env', help='any environment variables that need to be set for running wtperf') parser.add_argument('-t', '--test', help='path of the wtperf test to execute') parser.add_argument('-o', '--outfile', help='path of the file to write test output to') + parser.add_argument('-b', '--brief_output', action="store_true", help='brief (not detailed) test output') parser.add_argument('-m', '--runmax', type=int, default=1, help='maximum number of times to run the test') parser.add_argument('-ho', '--home', help='path of the "home" directory that wtperf will use') parser.add_argument('-re', '--reuse', action="store_true", help='reuse and reanalyse results from previous tests rather than running tests again') + parser.add_argument('-g', '--git_root', help='path of the Git working directory') + parser.add_argument('-i', '--json_info', help='additional test information in a json format string') + parser.add_argument('-bf', '--batch_file', help='Run all specified configurations for a single test') + parser.add_argument('-args', '--arguments', help='Additional arguments to pass into wtperf') + parser.add_argument('-ops', '--operations', help='List of operations to report metrics for') parser.add_argument('-v', '--verbose', action="store_true", help='be verbose') args = parser.parse_args() @@ -127,13 +230,19 @@ def main(): print('WTPerfPy') print('========') print("Configuration:") - print(" WtPerf path: {}".format(args.wtperf)) - print(" Environment: {}".format(args.env)) - print(" Test path: {}".format(args.test)) - print(" Home base: {}".format(args.home)) - print(" Outfile: {}".format(args.outfile)) - print(" Runmax: {}".format(args.runmax)) - print(" Reuse results: {}".format(args.reuse)) + print(" WtPerf path: {}".format(args.wtperf)) + print(" Environment: {}".format(args.env)) + print(" Test path: {}".format(args.test)) + print(" Home base: {}".format(args.home)) + print(" Batch file: {}".format(args.batch_file)) + print(" Arguments: {}".format(args.arguments)) + print(" Operations: {}".format(args.operations)) + print(" Git root: {}".format(args.git_root)) + print(" Outfile: {}".format(args.outfile)) + print(" Runmax: {}".format(args.runmax)) + print(" JSON info {}".format(args.json_info)) + print(" Reuse results: {}".format(args.reuse)) + print(" Brief output: {}".format(args.brief_output)) if args.wtperf is None: sys.exit('The path to the wtperf executable is required') @@ -141,33 +250,76 @@ def main(): sys.exit('The path to the test file is required') if args.home is None: sys.exit('The path to the "home" directory is required') + if args.batch_file and not os.path.isfile(args.batch_file): + sys.exit("batch_file: {} not found!".format(args.batch_file)) + if args.batch_file and (args.arguments or args.operations): + sys.exit("A batch file (-bf) should not be defined at the same time as -ops or -args") + + json_info = json.loads(args.json_info) if args.json_info else {} + arguments = json.loads(args.arguments) if args.arguments else None + operations = json.loads(args.operations) if args.operations else None config = WTPerfConfig(wtperf_path=args.wtperf, home_dir=args.home, test=args.test, + batch_file=args.batch_file, + arguments=arguments, + operations=operations, environment=args.env, run_max=args.runmax, - verbose=args.verbose) + verbose=args.verbose, + git_root=args.git_root, + json_info=json_info) perf_stats: PerfStatCollection = setup_perf_stats() - # Run tests (if we're not reusing results) + if config.batch_file: + if args.verbose: + print("Reading batch file {}".format(config.batch_file)) + with open(config.batch_file, "r") as file: + batch_file_contents = json.load(file) + + # Run test if not args.reuse: - for test_run in range(args.runmax): - print("Starting test {}".format(test_run)) - run_test(config=config, test_run=test_run) - print("Completed test {}".format(test_run)) + if config.batch_file: + for content in batch_file_contents: + if args.verbose: + print("Argument: {}, Operation: {}".format(content["arguments"], content["operations"])) + run_test_wrapper(config=config, operations=content["operations"], arguments=content["arguments"]) + else: + run_test_wrapper(config=config, arguments=arguments, operations=operations) + + if not args.verbose and not args.outfile: + sys.exit("Enable verbosity (or provide a file path) to dump the stats. " + "Try 'python3 wtperf_run.py --help' for more information.") + + # Process result + if config.batch_file: + for content in batch_file_contents: + process_results(config, perf_stats, operations=content["operations"]) + else: + process_results(config, perf_stats, operations=operations) - # Process results - perf_dict = process_results(config, perf_stats) - perf_json = json.dumps(perf_dict, indent=4, sort_keys=True) + # Output result + if args.brief_output: + if args.verbose: + print("Brief stats output (Evergreen compatible format):") + perf_results = brief_perf_stats(config, perf_stats) + else: + if args.verbose: + print("Detailed stats output (Atlas compatible format):") + perf_results = detailed_perf_stats(config, perf_stats) if args.verbose: - print("JSON: {}".format(perf_json)) + perf_json = json.dumps(perf_results, indent=4, sort_keys=True) + print("{}".format(perf_json)) if args.outfile: + dir_name = os.path.dirname(args.outfile) + if dir_name: + os.makedirs(dir_name, exist_ok=True) with open(args.outfile, 'w') as outfile: - json.dump(perf_dict, outfile, indent=4, sort_keys=True) + json.dump(perf_results, outfile, indent=4, sort_keys=True) if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/build_posix/aclocal/options.m4 b/src/third_party/wiredtiger/build_posix/aclocal/options.m4 index 651b777fee1..0fe512c0c8c 100644 --- a/src/third_party/wiredtiger/build_posix/aclocal/options.m4 +++ b/src/third_party/wiredtiger/build_posix/aclocal/options.m4 @@ -186,6 +186,23 @@ if test "$wt_cv_enable_tcmalloc" = "yes"; then fi AM_CONDITIONAL([TCMalloc], [test "$wt_cv_enable_tcmalloc" = "yes"]) +AC_MSG_CHECKING(if --enable-memkind option specified) +AC_ARG_ENABLE(memkind, + [AS_HELP_STRING([--enable-memkind], + [Memkind library enables using memory on Optane NVRAM. Enable if planning to use NVRAM block cache.])], r=$enableval, r=no) +case "$r" in +no) wt_cv_enable_memkind=no;; +*) wt_cv_enable_memkind=yes;; +esac +AC_MSG_RESULT($wt_cv_enable_memkind) +if test "$wt_cv_enable_memkind" = "yes"; then + AC_CHECK_HEADER(memkind.h,, + [AC_MSG_ERROR([--enable-memkind requires memkind.h])]) + AC_CHECK_LIB(memkind, memkind_get_version,, + [AC_MSG_ERROR([--enable-memkind requires memkind library])]) +fi +AM_CONDITIONAL([Memkind], [test "$wt_cv_enable_memkind" = "yes"]) + AH_TEMPLATE(SPINLOCK_TYPE, [Spinlock type from mutex.h.]) AC_MSG_CHECKING(if --with-spinlock option specified) AC_ARG_WITH(spinlock, diff --git a/src/third_party/wiredtiger/build_posix/aclocal/strict.m4 b/src/third_party/wiredtiger/build_posix/aclocal/strict.m4 index 545ca02a451..4159eea0e4b 100644 --- a/src/third_party/wiredtiger/build_posix/aclocal/strict.m4 +++ b/src/third_party/wiredtiger/build_posix/aclocal/strict.m4 @@ -3,31 +3,21 @@ # GCC warnings. AC_DEFUN([AM_GCC_WARNINGS], [ - w="$w -Wall -Wextra -Werror" - w="$w -Waggregate-return" - w="$w -Wbad-function-cast" + # List of common C/CXX flags. w="$w -Wcast-align" - w="$w -Wdeclaration-after-statement" w="$w -Wdouble-promotion" + w="$w -Werror" w="$w -Wfloat-equal" w="$w -Wformat-nonliteral" w="$w -Wformat-security" w="$w -Wformat=2" w="$w -Winit-self" - w="$w -Wjump-misses-init" w="$w -Wmissing-declarations" w="$w -Wmissing-field-initializers" - w="$w -Wmissing-prototypes" - w="$w -Wnested-externs" - w="$w -Wold-style-definition" w="$w -Wpacked" w="$w -Wpointer-arith" - w="$w -Wpointer-sign" w="$w -Wredundant-decls" - w="$w -Wshadow" - w="$w -Wsign-conversion" - w="$w -Wstrict-prototypes" w="$w -Wswitch-enum" w="$w -Wundef" w="$w -Wuninitialized" @@ -36,9 +26,8 @@ AC_DEFUN([AM_GCC_WARNINGS], [ w="$w -Wwrite-strings" # Non-fatal informational warnings. - # We don't turn on the unsafe-loop-optimizations warning after gcc7, - # it's too noisy to tolerate. Regardless, don't fail even when it's - # configured. + # The unsafe-loop-optimizations warning is only enabled for specific gcc versions. + # Regardless, don't fail when it's configured. w="$w -Wno-error=unsafe-loop-optimizations" # GCC 4.7 @@ -50,13 +39,10 @@ AC_DEFUN([AM_GCC_WARNINGS], [ gcc8=0 case "$1" in [*4.7.[0-9]*]) # gcc4.7 - w="$w -Wno-c11-extensions" - w="$w -Wunsafe-loop-optimizations";; + w="$w -Wno-c11-extensions";; [*5.[0-9].[0-9]*]) # gcc5.X - w="$w -Wunsafe-loop-optimizations" gcc5=1;; [*6.[0-9].[0-9]*]) # gcc6.X - w="$w -Wunsafe-loop-optimizations" gcc5=1 gcc6=1;; [*7.[0-9].[0-9]*]) # gcc7.X @@ -72,8 +58,6 @@ AC_DEFUN([AM_GCC_WARNINGS], [ if test $gcc5 -eq 1; then w="$w -Wformat-signedness" - w="$w -Wjump-misses-init" - w="$w -Wredundant-decls" w="$w -Wunused-macros" w="$w -Wvariadic-macros" fi @@ -94,13 +78,46 @@ AC_DEFUN([AM_GCC_WARNINGS], [ w="$w -Wmultistatement-macros" fi - wt_cv_strict_warnings="$w" + w_c="$w" + w_cxx="$w" + + # FIX-ME-WT-8247: Add those flags to the common ones if we want them for the compilation of the c++ files too. + w_c="$w_c -Waggregate-return" + w_c="$w_c -Wall" + w_c="$w_c -Wextra" + w_c="$w_c -Wshadow" + w_c="$w_c -Wsign-conversion" + + # Specific C flags. + w_c="$w_c -Wbad-function-cast" + w_c="$w_c -Wdeclaration-after-statement" + w_c="$w_c -Wjump-misses-init" + w_c="$w_c -Wmissing-prototypes" + w_c="$w_c -Wnested-externs" + w_c="$w_c -Wold-style-definition" + w_c="$w_c -Wpointer-sign" + w_c="$w_c -Wstrict-prototypes" + + # We only turn on the unsafe-loop-optimizations warning before gcc7, + # it's too noisy to tolerate otherwise. + case "$1" in + [*4.7.[0-9]*]) # gcc4.7 + w_c="$w_c -Wunsafe-loop-optimizations";; + [*5.[0-9].[0-9]*]) # gcc5.X + w_c="$w_c -Wunsafe-loop-optimizations";; + [*6.[0-9].[0-9]*]) # gcc6.X + w_c="$w_c -Wunsafe-loop-optimizations";; + esac + + wt_c_strict_warnings="$w_c" + wt_cxx_strict_warnings="$w_cxx" ]) # Clang warnings. AC_DEFUN([AM_CLANG_WARNINGS], [ - w="-Weverything -Werror" + # List of common C/CXX flags. + w="$w -Werror" w="$w -Wno-cast-align" w="$w -Wno-documentation-unknown-command" w="$w -Wno-format-nonliteral" @@ -146,5 +163,13 @@ AC_DEFUN([AM_CLANG_WARNINGS], [ # Ignore unrecognized options. w="$w -Wno-unknown-warning-option" - wt_cv_strict_warnings="$w" + w_c="$w" + w_cxx="$w" + + # Specific C flags. + # The -Weverything flag is not recommended, we only keep it for C files as it was already enabled. + w_c="$w_c -Weverything" + + wt_c_strict_warnings="$w_c" + wt_cxx_strict_warnings="$w_cxx" ]) diff --git a/src/third_party/wiredtiger/build_posix/configure.ac.in b/src/third_party/wiredtiger/build_posix/configure.ac.in index 6681e0e5650..c4b63ef92e5 100644 --- a/src/third_party/wiredtiger/build_posix/configure.ac.in +++ b/src/third_party/wiredtiger/build_posix/configure.ac.in @@ -139,7 +139,8 @@ if test "$wt_cv_enable_strict" = "yes"; then [--enable-strict does not support "$wt_cv_cc_version".]);; esac - AM_CFLAGS="$AM_CFLAGS $wt_cv_strict_warnings" + AM_CFLAGS="$AM_CFLAGS $wt_c_strict_warnings" + AM_CXXFLAGS="$AM_CXXFLAGS $wt_cxx_strict_warnings" fi # Python API @@ -237,7 +238,9 @@ fi AC_DEFINE_UNQUOTED(WT_BUFFER_ALIGNMENT_DEFAULT, $BUFFER_ALIGNMENT, [Default alignment of buffers used for I/O]) +# Export the C and CXX flags. AC_SUBST(AM_CFLAGS) +AC_SUBST(AM_CXXFLAGS) # Warn that diagnostic builds should not be used in production if test "$wt_cv_enable_diagnostic" = "yes"; then diff --git a/src/third_party/wiredtiger/build_win/wiredtiger_config.h b/src/third_party/wiredtiger/build_win/wiredtiger_config.h index a68d64a58bf..bfee99faf9b 100644 --- a/src/third_party/wiredtiger/build_win/wiredtiger_config.h +++ b/src/third_party/wiredtiger/build_win/wiredtiger_config.h @@ -52,6 +52,9 @@ /* Define to 1 if you have the `lz4' library (-llz4). */ /* #undef HAVE_LIBLZ4 */ +/* Define to 1 if you have the `memkind' library (-lmemkind). */ +/* #undef HAVE_LIBMEMKIND */ + /* Define to 1 if you have the `pthread' library (-lpthread). */ /* #undef HAVE_LIBPTHREAD */ diff --git a/src/third_party/wiredtiger/cmake/configs/base.cmake b/src/third_party/wiredtiger/cmake/configs/base.cmake index 098859f2734..1ee24fea4e0 100644 --- a/src/third_party/wiredtiger/cmake/configs/base.cmake +++ b/src/third_party/wiredtiger/cmake/configs/base.cmake @@ -38,7 +38,7 @@ config_bool( config_string( WT_BUFFER_ALIGNMENT_DEFAULT - "WiredTiger buffer boundary aligment" + "WiredTiger buffer boundary alignment" DEFAULT 0 ) @@ -61,6 +61,19 @@ config_bool( ) config_bool( + ENABLE_SHARED + "Compile as a shared library" + DEFAULT ON +) + +config_bool( + WITH_PIC + "Generate position-independent code. Note PIC will always \ + be used on shared targets, irrespective of the value of this configuration." + DEFAULT OFF +) + +config_bool( ENABLE_STRICT "Compile with strict compiler warnings enabled" DEFAULT OFF diff --git a/src/third_party/wiredtiger/cmake/configs/x86/windows/config.cmake b/src/third_party/wiredtiger/cmake/configs/x86/windows/config.cmake index 842708a1f9d..0cce82d1b8e 100644 --- a/src/third_party/wiredtiger/cmake/configs/x86/windows/config.cmake +++ b/src/third_party/wiredtiger/cmake/configs/x86/windows/config.cmake @@ -13,6 +13,8 @@ set(SPINLOCK_TYPE "msvc" CACHE STRING "" FORCE) # We force a static compilation to generate a ".lib" file. We can then # additionally generate a dll file using a *DEF file. set(ENABLE_STATIC ON CACHE BOOL "" FORCE) +set(ENABLE_SHARED OFF CACHE BOOL "" FORCE) +set(WITH_PIC ON CACHE BOOL "" FORCE) # Compile as C code . add_compile_options(/TC) diff --git a/src/third_party/wiredtiger/cmake/define_libwiredtiger.cmake b/src/third_party/wiredtiger/cmake/define_libwiredtiger.cmake new file mode 100644 index 00000000000..d477aaa0820 --- /dev/null +++ b/src/third_party/wiredtiger/cmake/define_libwiredtiger.cmake @@ -0,0 +1,103 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information +# + +# define_wiredtiger_library(target type) +# A helper that defines a wiredtiger library target. This defining a set of common targets and properties we +# want to be associated to any given 'libwiredtiger' target. Having this as a macro allows us to de-duplicate common +# definitions when creating multiple versions of libwiredtiger i.e. static and shared builds. Note: this +# macro assumes you only call it once per libwiredtiger flavour (e.g. only one 'static' definition), use carefully. +# target - Name of the libwiredtiger target. +# type - Library type of wiredtiger target e.g. STATIC, SHARED or MODULE. +# SOURCES - Sources to be compiled under the wiredtiger library. Requires that at least one source file/object is defined. +# PUBLIC_INCLUDES - Public interface includes of the wiredtiger library. +# PRIVATE_INCLUDES - Private interface includes of the wiredtiger library. +macro(define_wiredtiger_library target type) + cmake_parse_arguments( + "DEFINE_WT" + "" + "" + "SOURCES;PUBLIC_INCLUDES;PRIVATE_INCLUDES" + ${ARGN} + ) + if (NOT "${DEFINE_WT_UNPARSED_ARGUMENTS}" STREQUAL "") + message(FATAL_ERROR "Unknown arguments to define_wiredtiger_library: ${DEFINE_WT_UNPARSED_ARGUMENTS}") + endif() + if ("${DEFINE_WT_SOURCES}" STREQUAL "") + message(FATAL_ERROR "No sources given to define_wiredtiger_library") + endif() + + # Define the wiredtiger library target. + add_library(${target} ${type} ${DEFINE_WT_SOURCES}) + # Append any include directories to the library target. + if(DEFINE_WT_PUBLIC_INCLUDES) + target_include_directories(${target} PUBLIC ${DEFINE_WT_PUBLIC_INCLUDES}) + endif() + if(DEFINE_WT_PRIVATE_INCLUDES) + target_include_directories(${target} PRIVATE ${DEFINE_WT_PRIVATE_INCLUDES}) + endif() + # Append any provided C flags. + if(COMPILER_DIAGNOSTIC_C_FLAGS) + target_compile_options(${target} PRIVATE ${COMPILER_DIAGNOSTIC_C_FLAGS}) + endif() + + # We want to set the following target properties: + # OUTPUT_NAME - Generate a library with the name "libwiredtiger[.so|.a". Note this assumes each invocation + # of this macro is specifying a unique libwiredtiger target type (e.g 'SHARED', 'STATIC'), multiple declarations + # of a 'SHARED' wiredtiger library would conflict. + # NO_SYSTEM_FROM_IMPORTED - don't treat include interface directories consumed on an imported target as system + # directories. + set_target_properties(${target} PROPERTIES + OUTPUT_NAME "wiredtiger" + NO_SYSTEM_FROM_IMPORTED TRUE + ) + + # Ensure we link any available library dependencies to our wiredtiger target. + if(HAVE_LIBPTHREAD) + target_link_libraries(${target} PUBLIC ${HAVE_LIBPTHREAD}) + if(HAVE_LIBPTHREAD_INCLUDES) + target_include_directories(${target} PUBLIC ${HAVE_LIBPTHREAD_INCLUDES}) + endif() + endif() + if(HAVE_LIBRT) + target_link_libraries(${target} PUBLIC ${HAVE_LIBRT}) + if(HAVE_LIBRT_INCLUDES) + target_include_directories(${target} PUBLIC ${HAVE_LIBRT_INCLUDES}) + endif() + endif() + if(HAVE_LIBDL) + target_link_libraries(${target} PUBLIC ${HAVE_LIBDL}) + if(HAVE_LIBDL_INCLUDES) + target_include_directories(${target} PUBLIC ${HAVE_LIBDL_INCLUDES}) + endif() + endif() + if(ENABLE_TCMALLOC) + target_link_libraries(${target} PRIVATE wt::tcmalloc) + endif() + + # We want to capture any transitive dependencies associated with the builtin library + # target and ensure we are explicitly linking the 3rd party libraries. + if(HAVE_BUILTIN_EXTENSION_LZ4) + target_link_libraries(${target} PRIVATE wt::lz4) + endif() + + if(HAVE_BUILTIN_EXTENSION_SNAPPY) + target_link_libraries(${target} PRIVATE wt::snappy) + endif() + + if(HAVE_BUILTIN_EXTENSION_SODIUM) + target_link_libraries(${target} PRIVATE wt::sodium) + endif() + + if(HAVE_BUILTIN_EXTENSION_ZLIB) + target_link_libraries(${target} PRIVATE wt::zlib) + endif() + + if(HAVE_BUILTIN_EXTENSION_ZSTD) + target_link_libraries(${target} PRIVATE wt::zstd) + endif() +endmacro() diff --git a/src/third_party/wiredtiger/cmake/helpers.cmake b/src/third_party/wiredtiger/cmake/helpers.cmake index f3019627b8e..77a7961e499 100644 --- a/src/third_party/wiredtiger/cmake/helpers.cmake +++ b/src/third_party/wiredtiger/cmake/helpers.cmake @@ -376,7 +376,7 @@ function(config_include config_name description) set(${config_name} OFF CACHE INTERNAL "" FORCE) set(${config_name}_DISABLED ON CACHE INTERNAL "" FORCE) endif() - # Set an internal cahce variable with the CPP include statement. We can use this when building out our config header. + # Set an internal cache variable with the CPP include statement. We can use this when building out our config header. if (${${config_name}}) set(${config_name}_DECL "#include <${CONFIG_INCLUDE_FILE}>" CACHE INTERNAL "") endif() @@ -422,13 +422,14 @@ function(config_lib config_name description) find_library(has_lib_${config_name} ${CONFIG_LIB_LIB}) set(CMAKE_REQUIRED_FLAGS) set(has_lib "0") + set(has_include "") if(has_lib_${config_name}) set(has_lib ${has_lib_${config_name}}) if (CONFIG_LIB_HEADER) find_path(include_path_${config_name} ${CONFIG_LIB_HEADER}) if (include_path_${config_name}) message("-- Looking for library ${CONFIG_LIB_LIB}: found ${has_lib_${config_name}}, include path ${include_path_${config_name}}") - include_directories(${include_path_${config_name}}) + set(has_include ${include_path_${config_name}}) else() message("-- Looking for library ${CONFIG_LIB_LIB}: found ${has_lib_${config_name}}") endif() @@ -444,8 +445,10 @@ function(config_lib config_name description) if(${config_name}_DISABLED) unset(${config_name}_DISABLED CACHE) set(${config_name} ${has_lib} CACHE STRING "${description}" FORCE) + set(${config_name}_INCLUDES ${has_include} CACHE STRING "Additional include paths for ${config_name}" FORCE) else() set(${config_name} ${has_lib} CACHE STRING "${description}") + set(${config_name}_INCLUDES ${has_include} CACHE STRING "Additional include paths for ${config_name}") endif() # 'check_library_exists' sets our given temp variable into the cache. Clear this so it doesn't persist between # configuration runs. @@ -554,7 +557,7 @@ endfunction() # assert_type_size(type size) # Wrapper function around 'test_type_size' that additionally asserts whether the given types meets an expected size. # Throws a fatal error if the type is not found or doesn't equal the expected size. -# type - eame of the type to test. +# type - name of the type to test. # size - expected size of the type. # EXTRA_INCLUDES - extra/optional include files to access the given type e.g. a custom typedef in an include header. function(assert_type_size type size) @@ -582,7 +585,7 @@ function(assert_type_size type size) message(FATAL_ERROR "Type assertion failed: ${type} does not exists") endif() - if((NOT ${size} EQUAL 0) AND (NOT ${output_type_size} EQUAL ${size})) + if((NOT ${size} EQUAL 0) AND (NOT ${output_type_size} EQUAL ${size})) # Type does not meet size assertion. message(FATAL_ERROR "Type assertion failed: ${type} does not equal size ${size}") endif() diff --git a/src/third_party/wiredtiger/cmake/install/install.cmake b/src/third_party/wiredtiger/cmake/install/install.cmake index ec9b30234d6..e6ac82d2b10 100644 --- a/src/third_party/wiredtiger/cmake/install/install.cmake +++ b/src/third_party/wiredtiger/cmake/install/install.cmake @@ -10,21 +10,25 @@ include(GNUInstallDirs) # Library installs -# Define the public headers for wiredtiger library to be used when installing the target. -set_property( - TARGET wiredtiger - PROPERTY PUBLIC_HEADER - ${CMAKE_BINARY_DIR}/include/wiredtiger.h - ${CMAKE_SOURCE_DIR}/src/include/wiredtiger_ext.h +# Define the wiredtiger public headers we want to export when running the install target. +install( + FILES ${CMAKE_BINARY_DIR}/include/wiredtiger.h ${CMAKE_SOURCE_DIR}/src/include/wiredtiger_ext.h + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} ) -# Set the version property of the wiredtiger library so we can export a versioned install. -set_target_properties(wiredtiger PROPERTIES VERSION "${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH}") -# Install the wiredtiger library target. -install(TARGETS wiredtiger +# Define the wiredtiger library targets we will install. +set(wt_targets) +if(ENABLE_SHARED) + list(APPEND wt_targets wiredtiger_shared) +endif() +if(ENABLE_STATIC) + list(APPEND wt_targets wiredtiger_static) +endif() + +# Install the wiredtiger library targets. +install(TARGETS ${wt_targets} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} - PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} ) # Create our wiredtiger pkgconfig (for POSIX builds). diff --git a/src/third_party/wiredtiger/cmake/strict/cl_strict.cmake b/src/third_party/wiredtiger/cmake/strict/cl_strict.cmake index 625bf9e88d5..12607dd64e9 100644 --- a/src/third_party/wiredtiger/cmake/strict/cl_strict.cmake +++ b/src/third_party/wiredtiger/cmake/strict/cl_strict.cmake @@ -6,16 +6,13 @@ # See the file LICENSE for redistribution information. # -# Warning level 3. -list(APPEND win_c_flags "/WX") -# Ignore warning about mismatched const qualifiers. -list(APPEND win_c_flags "/wd4090") -# Ignore deprecated functions. -list(APPEND win_c_flags "/wd4996") -# Complain about unreferenced format parameter. -list(APPEND win_c_flags "/we4100") -# Enable security check. -list(APPEND win_c_flags "/GS") +include(cmake/strict/strict_flags_helpers.cmake) -# Set our base compiler flags that can be used by the rest of our build. -set(COMPILER_DIAGNOSTIC_FLAGS "${COMPILER_DIAGNOSTIC_FLAGS};${win_c_flags}" CACHE INTERNAL "" FORCE) +# Get common CL flags. +set(cl_flags) +get_cl_base_flags(cl_flags C) + +# Specific C flags: + +# Set our common compiler flags that can be used by the rest of our build. +set(COMPILER_DIAGNOSTIC_C_FLAGS ${cl_flags}) diff --git a/src/third_party/wiredtiger/cmake/strict/clang_strict.cmake b/src/third_party/wiredtiger/cmake/strict/clang_strict.cmake index f87c988fe02..676f9483405 100644 --- a/src/third_party/wiredtiger/cmake/strict/clang_strict.cmake +++ b/src/third_party/wiredtiger/cmake/strict/clang_strict.cmake @@ -6,60 +6,14 @@ # See the file LICENSE for redistribution information. # -list(APPEND clang_base_c_flags "-Weverything") -list(APPEND clang_base_c_flags "-Werror") -list(APPEND clang_base_c_flags "-Wno-cast-align") -list(APPEND clang_base_c_flags "-Wno-documentation-unknown-command") -list(APPEND clang_base_c_flags "-Wno-format-nonliteral") -list(APPEND clang_base_c_flags "-Wno-packed") -list(APPEND clang_base_c_flags "-Wno-padded") -list(APPEND clang_base_c_flags "-Wno-reserved-id-macro") -list(APPEND clang_base_c_flags "-Wno-zero-length-array") +include(cmake/strict/strict_flags_helpers.cmake) -# We should turn on cast-qual, but not as a fatal error: see WT-2690. -# For now, turn it off. -list(APPEND clang_base_c_flags "-Wno-cast-qual") +# Get common CLANG flags. +set(clang_flags) +get_clang_base_flags(clang_flags C) -# Turn off clang thread-safety-analysis, it doesn't like some of the -# code patterns in WiredTiger. -list(APPEND clang_base_c_flags "-Wno-thread-safety-analysis") +# Specific C flags: +list(APPEND clang_flags "-Weverything") -# On Centos 7.3.1611, system header files aren't compatible with -# -Wdisabled-macro-expansion. -list(APPEND clang_base_c_flags "-Wno-disabled-macro-expansion") - -# We occasionally use an extra semicolon to indicate an empty loop or -# conditional body. -list(APPEND clang_base_c_flags "-Wno-extra-semi-stmt") - -# Ignore unrecognized options. -list(APPEND clang_base_c_flags "-Wno-unknown-warning-option") - -if(WT_DARWIN AND (CMAKE_C_COMPILER_VERSION VERSION_EQUAL 4.1)) - # Apple clang has its own numbering system, and older OS X - # releases need some special love. Turn off some flags for - # Apple's clang 4.1: - # Apple clang version 4.1 - # ((tags/Apple/clang-421.11.66) (based on LLVM 3.1svn) - list(APPEND clang_base_c_flags "-Wno-attributes") - list(APPEND clang_base_c_flags "-Wno-pedantic") - list(APPEND clang_base_c_flags "-Wno-unused-command-line-argument") -endif() - -if(CMAKE_C_COMPILER_VERSION VERSION_GREATER_EQUAL 10) - # Clang 10+ has added additional on-by-default diagnostics that isn't - # compatible with some of the code patterns in WiredTiger. - list(APPEND clang_base_c_flags "-Wno-implicit-fallthrough") - list(APPEND clang_base_c_flags "-Wno-implicit-int-float-conversion") -endif() - -if(WT_DARWIN AND NOT CMAKE_CROSSCOMPILING) - # If we are not cross-compiling, we can safely disable this diagnostic. - # Its incompatible with strict diagnostics when including external - # libraries that are not in the default linker path - # e.g. linking zlib/snappy/... from /usr/local/. - list(APPEND clang_base_c_flags "-Wno-poison-system-directories") -endif() - -# Set our base clang flags to ensure it propogates to the rest of our build. -set(COMPILER_DIAGNOSTIC_FLAGS "${COMPILER_DIAGNOSTIC_FLAGS};${clang_base_c_flags}" CACHE INTERNAL "" FORCE) +# Set our common compiler flags that can be used by the rest of our build. +set(COMPILER_DIAGNOSTIC_C_FLAGS ${clang_flags}) diff --git a/src/third_party/wiredtiger/cmake/strict/clangxx_strict.cmake b/src/third_party/wiredtiger/cmake/strict/clangxx_strict.cmake new file mode 100644 index 00000000000..3533c55cdb1 --- /dev/null +++ b/src/third_party/wiredtiger/cmake/strict/clangxx_strict.cmake @@ -0,0 +1,18 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information. +# + +include(cmake/strict/strict_flags_helpers.cmake) + +# Get common CLANG flags. +set(clangxx_flags) +get_clang_base_flags(clangxx_flags CXX) + +# Specific CXX flags: + +# Set our common compiler flags that can be used by the rest of our build. +set(COMPILER_DIAGNOSTIC_CXX_FLAGS ${clangxx_flags}) diff --git a/src/third_party/wiredtiger/cmake/strict/clxx_strict.cmake b/src/third_party/wiredtiger/cmake/strict/clxx_strict.cmake new file mode 100644 index 00000000000..af799922c7c --- /dev/null +++ b/src/third_party/wiredtiger/cmake/strict/clxx_strict.cmake @@ -0,0 +1,18 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information. +# + +include(cmake/strict/strict_flags_helpers.cmake) + +# Get common CL flags. +set(clxx_flags) +get_cl_base_flags(clxx_flags CXX) + +# Specific CXX flags: + +# Set our common compiler flags that can be used by the rest of our build. +set(COMPILER_DIAGNOSTIC_CXX_FLAGS ${clxx_flags}) diff --git a/src/third_party/wiredtiger/cmake/strict/gcc_strict.cmake b/src/third_party/wiredtiger/cmake/strict/gcc_strict.cmake index 420a4457676..5a894fcd595 100644 --- a/src/third_party/wiredtiger/cmake/strict/gcc_strict.cmake +++ b/src/third_party/wiredtiger/cmake/strict/gcc_strict.cmake @@ -6,74 +6,38 @@ # See the file LICENSE for redistribution information. # -list(APPEND gcc_base_c_flags "-Wall") -list(APPEND gcc_base_c_flags "-Wextra") -list(APPEND gcc_base_c_flags "-Werror") -list(APPEND gcc_base_c_flags "-Waggregate-return") -list(APPEND gcc_base_c_flags "-Wbad-function-cast") -list(APPEND gcc_base_c_flags "-Wcast-align") -list(APPEND gcc_base_c_flags "-Wdeclaration-after-statement") -list(APPEND gcc_base_c_flags "-Wdouble-promotion") -list(APPEND gcc_base_c_flags "-Wfloat-equal") -list(APPEND gcc_base_c_flags "-Wformat-nonliteral") -list(APPEND gcc_base_c_flags "-Wformat-security") -list(APPEND gcc_base_c_flags "-Wformat=2") -list(APPEND gcc_base_c_flags "-Winit-self") -list(APPEND gcc_base_c_flags "-Wjump-misses-init") -list(APPEND gcc_base_c_flags "-Wmissing-declarations") -list(APPEND gcc_base_c_flags "-Wmissing-field-initializers") -list(APPEND gcc_base_c_flags "-Wmissing-prototypes") -list(APPEND gcc_base_c_flags "-Wnested-externs") -list(APPEND gcc_base_c_flags "-Wold-style-definition") -list(APPEND gcc_base_c_flags "-Wpacked") -list(APPEND gcc_base_c_flags "-Wpointer-arith") -list(APPEND gcc_base_c_flags "-Wpointer-sign") -list(APPEND gcc_base_c_flags "-Wredundant-decls") -list(APPEND gcc_base_c_flags "-Wshadow") -list(APPEND gcc_base_c_flags "-Wsign-conversion") -list(APPEND gcc_base_c_flags "-Wstrict-prototypes") -list(APPEND gcc_base_c_flags "-Wswitch-enum") -list(APPEND gcc_base_c_flags "-Wundef") -list(APPEND gcc_base_c_flags "-Wuninitialized") -list(APPEND gcc_base_c_flags "-Wunreachable-code") -list(APPEND gcc_base_c_flags "-Wunused") -list(APPEND gcc_base_c_flags "-Wwrite-strings") +include(cmake/strict/strict_flags_helpers.cmake) -# Non-fatal informational warnings. -# We don't turn on the unsafe-loop-optimizations warning after gcc7, -# it's too noisy to tolerate. Regardless, don't fail even when it's -# configured. -list(APPEND gcc_base_c_flags "-Wno-error=unsafe-loop-optimizations") +# Get common GNU flags. +set(gcc_flags) +get_gnu_base_flags(gcc_flags C) + +# FIX-ME-WT-8247: Add those flags to the common GNU flags if we want them for the compilation of the +# c++ files too. +list(APPEND gcc_flags "-Waggregate-return") +list(APPEND gcc_flags "-Wall") +list(APPEND gcc_flags "-Wextra") +list(APPEND gcc_flags "-Wshadow") +list(APPEND gcc_flags "-Wsign-conversion") + +# Specific C flags: +list(APPEND gcc_flags "-Wbad-function-cast") +list(APPEND gcc_flags "-Wdeclaration-after-statement") +list(APPEND gcc_flags "-Wjump-misses-init") +list(APPEND gcc_flags "-Wmissing-prototypes") +list(APPEND gcc_flags "-Wnested-externs") +list(APPEND gcc_flags "-Wold-style-definition") +list(APPEND gcc_flags "-Wpointer-sign") +list(APPEND gcc_flags "-Wstrict-prototypes") + +# We only turn on the unsafe-loop-optimizations warning before gcc7, +# it's too noisy to tolerate otherwise. if(${CMAKE_C_COMPILER_VERSION} VERSION_EQUAL 4.7) - list(APPEND gcc_base_c_flags "-Wno-c11-extensions") - list(APPEND gcc_base_c_flags "-Wunsafe-loop-optimizations") + list(APPEND gcc_flags "-Wunsafe-loop-optimizations") elseif(${CMAKE_C_COMPILER_VERSION} VERSION_EQUAL 5) - list(APPEND gcc_base_c_flags "-Wunsafe-loop-optimizations") -endif() - -if(${CMAKE_C_COMPILER_VERSION} VERSION_GREATER_EQUAL 5) - list(APPEND gcc_base_c_flags "-Wformat-signedness") - list(APPEND gcc_base_c_flags "-Wjump-misses-init") - list(APPEND gcc_base_c_flags "-Wredundant-decls") - list(APPEND gcc_base_c_flags "-Wunused-macros") - list(APPEND gcc_base_c_flags "-Wvariadic-macros") -endif() -if(${CMAKE_C_COMPILER_VERSION} VERSION_GREATER_EQUAL 6) - list(APPEND gcc_base_c_flags "-Wduplicated-cond") - list(APPEND gcc_base_c_flags "-Wlogical-op") - list(APPEND gcc_base_c_flags "-Wunused-const-variable=2") -endif() -if(${CMAKE_C_COMPILER_VERSION} VERSION_GREATER_EQUAL 7) - list(APPEND gcc_base_c_flags "-Walloca") - list(APPEND gcc_base_c_flags "-Walloc-zero") - list(APPEND gcc_base_c_flags "-Wduplicated-branches") - list(APPEND gcc_base_c_flags "-Wformat-overflow=2") - list(APPEND gcc_base_c_flags "-Wformat-truncation=2") - list(APPEND gcc_base_c_flags "-Wrestrict") -endif() -if(${CMAKE_C_COMPILER_VERSION} VERSION_GREATER_EQUAL 8) - list(APPEND gcc_base_c_flags "-Wmultistatement-macros") + list(APPEND gcc_flags "-Wunsafe-loop-optimizations") +elseif(${CMAKE_C_COMPILER_VERSION} VERSION_EQUAL 6) + list(APPEND gcc_flags "-Wunsafe-loop-optimizations") endif() -# Set our base gcc flags to ensure it propogates to the rest of our build. -set(COMPILER_DIAGNOSTIC_FLAGS "${COMPILER_DIAGNOSTIC_FLAGS};${gcc_base_c_flags}" CACHE INTERNAL "" FORCE) +set(COMPILER_DIAGNOSTIC_C_FLAGS ${gcc_flags}) diff --git a/src/third_party/wiredtiger/cmake/strict/gxx_strict.cmake b/src/third_party/wiredtiger/cmake/strict/gxx_strict.cmake new file mode 100644 index 00000000000..81d205a3b88 --- /dev/null +++ b/src/third_party/wiredtiger/cmake/strict/gxx_strict.cmake @@ -0,0 +1,17 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information. +# + +include(cmake/strict/strict_flags_helpers.cmake) + +# Get common GNU flags. +set(gxx_flags) +get_gnu_base_flags(gxx_flags CXX) + +# Specific CXX flags: + +set(COMPILER_DIAGNOSTIC_CXX_FLAGS ${gxx_flags}) diff --git a/src/third_party/wiredtiger/cmake/strict/strict_flags_helpers.cmake b/src/third_party/wiredtiger/cmake/strict/strict_flags_helpers.cmake new file mode 100644 index 00000000000..8b400f4cc7d --- /dev/null +++ b/src/third_party/wiredtiger/cmake/strict/strict_flags_helpers.cmake @@ -0,0 +1,227 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information +# + +# get_gnu_base_flags(flags) +# Helper function that generates a set of common GNU flags for a given language. +# flags - list of flags. +function(get_gnu_base_flags flags) + cmake_parse_arguments( + PARSE_ARGV + 1 + "GNU_FLAGS" + "CXX;C" + "" + "" + ) + + if (NOT "${GNU_FLAGS_UNPARSED_ARGUMENTS}" STREQUAL "") + message(FATAL_ERROR "Unknown arguments to get_gnu_base_flags: ${GNU_FLAGS_UNPARSED_ARGUMENTS}") + endif() + + set(cmake_compiler_version) + + if(${GNU_FLAGS_C} AND ${GNU_FLAGS_CXX}) + message(FATAL_ERROR "Only one language is accepted") + elseif(GNU_FLAGS_C) + set(cmake_compiler_version ${CMAKE_C_COMPILER_VERSION}) + elseif(GNU_FLAGS_CXX) + set(cmake_compiler_version ${CMAKE_CXX_COMPILER_VERSION}) + else() + message(FATAL_ERROR "No language passed") + endif() + + set(gnu_flags) + + list(APPEND gnu_flags "-Wcast-align") + list(APPEND gnu_flags "-Wdouble-promotion") + list(APPEND gnu_flags "-Werror") + list(APPEND gnu_flags "-Wfloat-equal") + list(APPEND gnu_flags "-Wformat-nonliteral") + list(APPEND gnu_flags "-Wformat-security") + list(APPEND gnu_flags "-Wformat=2") + list(APPEND gnu_flags "-Winit-self") + list(APPEND gnu_flags "-Wmissing-declarations") + list(APPEND gnu_flags "-Wmissing-field-initializers") + list(APPEND gnu_flags "-Wpacked") + list(APPEND gnu_flags "-Wpointer-arith") + list(APPEND gnu_flags "-Wredundant-decls") + list(APPEND gnu_flags "-Wswitch-enum") + list(APPEND gnu_flags "-Wundef") + list(APPEND gnu_flags "-Wuninitialized") + list(APPEND gnu_flags "-Wunreachable-code") + list(APPEND gnu_flags "-Wunused") + list(APPEND gnu_flags "-Wwrite-strings") + + # Non-fatal informational warnings. + # The unsafe-loop-optimizations warning is only enabled for specific gcc versions. + # Regardless, don't fail when it's configured. + list(APPEND gnu_flags "-Wno-error=unsafe-loop-optimizations") + + if(${cmake_compiler_version} VERSION_EQUAL 4.7) + list(APPEND gnu_flags "-Wno-c11-extensions") + endif() + if(${cmake_compiler_version} VERSION_GREATER_EQUAL 5) + list(APPEND gnu_flags "-Wformat-signedness") + list(APPEND gnu_flags "-Wunused-macros") + list(APPEND gnu_flags "-Wvariadic-macros") + endif() + if(${cmake_compiler_version} VERSION_GREATER_EQUAL 6) + list(APPEND gnu_flags "-Wduplicated-cond") + list(APPEND gnu_flags "-Wlogical-op") + list(APPEND gnu_flags "-Wunused-const-variable=2") + endif() + if(${cmake_compiler_version} VERSION_GREATER_EQUAL 7) + list(APPEND gnu_flags "-Walloca") + list(APPEND gnu_flags "-Walloc-zero") + list(APPEND gnu_flags "-Wduplicated-branches") + list(APPEND gnu_flags "-Wformat-overflow=2") + list(APPEND gnu_flags "-Wformat-truncation=2") + list(APPEND gnu_flags "-Wrestrict") + endif() + if(${cmake_compiler_version} VERSION_GREATER_EQUAL 8) + list(APPEND gnu_flags "-Wmultistatement-macros") + endif() + + set(${flags} ${gnu_flags} PARENT_SCOPE) + +endfunction() + +# get_clang_base_flags(flags) +# Helper function that generates a set of common CLANG flags for a given language. +# flags - list of flags. +function(get_clang_base_flags flags) + cmake_parse_arguments( + PARSE_ARGV + 1 + "CLANG_FLAGS" + "CXX;C" + "" + "" + ) + + if (NOT "${CLANG_FLAGS_UNPARSED_ARGUMENTS}" STREQUAL "") + message(FATAL_ERROR "Unknown arguments to get_clang_base_flags: ${CLANG_FLAGS_UNPARSED_ARGUMENTS}") + endif() + + set(cmake_compiler_version) + + if(${CLANG_FLAGS_C} AND ${CLANG_FLAGS_CXX}) + message(FATAL_ERROR "Only one language is accepted") + elseif(CLANG_FLAGS_C) + set(cmake_compiler_version ${CMAKE_C_COMPILER_VERSION}) + elseif(CLANG_FLAGS_CXX) + set(cmake_compiler_version ${CMAKE_CXX_COMPILER_VERSION}) + else() + message(FATAL_ERROR "No language passed") + endif() + + set(clang_flags) + + list(APPEND clang_flags "-Werror") + list(APPEND clang_flags "-Wno-cast-align") + list(APPEND clang_flags "-Wno-documentation-unknown-command") + list(APPEND clang_flags "-Wno-format-nonliteral") + list(APPEND clang_flags "-Wno-packed") + list(APPEND clang_flags "-Wno-padded") + list(APPEND clang_flags "-Wno-reserved-id-macro") + list(APPEND clang_flags "-Wno-zero-length-array") + + # We should turn on cast-qual, but not as a fatal error: see WT-2690. + # For now, turn it off. + list(APPEND clang_flags "-Wno-cast-qual") + + # Turn off clang thread-safety-analysis, it doesn't like some of the + # code patterns in WiredTiger. + list(APPEND clang_flags "-Wno-thread-safety-analysis") + + # On Centos 7.3.1611, system header files aren't compatible with + # -Wdisabled-macro-expansion. + list(APPEND clang_flags "-Wno-disabled-macro-expansion") + + # We occasionally use an extra semicolon to indicate an empty loop or + # conditional body. + list(APPEND clang_flags "-Wno-extra-semi-stmt") + + # Ignore unrecognized options. + list(APPEND clang_flags "-Wno-unknown-warning-option") + + if(WT_DARWIN AND NOT CMAKE_CROSSCOMPILING) + # If we are not cross-compiling, we can safely disable this diagnostic. + # Its incompatible with strict diagnostics when including external + # libraries that are not in the default linker path + # e.g. linking zlib/snappy/... from /usr/local/. + list(APPEND clang_flags "-Wno-poison-system-directories") + endif() + + if(WT_DARWIN AND (${cmake_compiler_version} VERSION_EQUAL 4.1)) + # Apple clang has its own numbering system, and older OS X + # releases need some special love. Turn off some flags for + # Apple's clang 4.1: + # Apple clang version 4.1 + # ((tags/Apple/clang-421.11.66) (based on LLVM 3.1svn) + list(APPEND clang_flags "-Wno-attributes") + list(APPEND clang_flags "-Wno-pedantic") + list(APPEND clang_flags "-Wno-unused-command-line-argument") + endif() + + if(${cmake_compiler_version} VERSION_GREATER_EQUAL 10) + # Clang 10+ has added additional on-by-default diagnostics that isn't + # compatible with some of the code patterns in WiredTiger. + list(APPEND clang_flags "-Wno-implicit-fallthrough") + list(APPEND clang_flags "-Wno-implicit-int-float-conversion") + endif() + + set(${flags} ${clang_flags} PARENT_SCOPE) + +endfunction() + +# get_cl_base_flags(flags) +# Helper function that generates a set of common CL flags for a given language. +# flags - list of flags. +function(get_cl_base_flags flags) + cmake_parse_arguments( + PARSE_ARGV + 1 + "CL_FLAGS" + "CXX;C" + "" + "" + ) + + if (NOT "${CL_FLAGS_UNPARSED_ARGUMENTS}" STREQUAL "") + message(FATAL_ERROR "Unknown arguments to get_cl_base_flags: ${CL_FLAGS_UNPARSED_ARGUMENTS}") + endif() + + set(cmake_compiler_version) + + if(${CL_FLAGS_C} AND ${CL_FLAGS_CXX}) + message(FATAL_ERROR "Only one language is accepted") + elseif(CL_FLAGS_C) + set(cmake_compiler_version ${CMAKE_C_COMPILER_VERSION}) + elseif(CL_FLAGS_CXX) + set(cmake_compiler_version ${CMAKE_CXX_COMPILER_VERSION}) + else() + message(FATAL_ERROR "No language passed") + endif() + + set(cl_flags) + + # Warning level 3. + list(APPEND cl_flags "/WX") + # Ignore warning about mismatched const qualifiers. + list(APPEND cl_flags "/wd4090") + # Ignore deprecated functions. + list(APPEND cl_flags "/wd4996") + # Complain about unreferenced format parameter. + list(APPEND cl_flags "/we4100") + # Enable security check. + list(APPEND cl_flags "/GS") + + set(${flags} ${cl_flags} PARENT_SCOPE) + +endfunction() diff --git a/src/third_party/wiredtiger/cmake/third_party/gperftools.cmake b/src/third_party/wiredtiger/cmake/third_party/gperftools.cmake new file mode 100644 index 00000000000..1062050f9b1 --- /dev/null +++ b/src/third_party/wiredtiger/cmake/third_party/gperftools.cmake @@ -0,0 +1,50 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +if(NOT HAVE_LIBTCMALLOC) + # We don't need to construct a tcmalloc library target. + return() +endif() + +if(TARGET wt::tcmalloc) + # Avoid redefining the imported library. + return() +endif() + +# Construct an imported tcmalloc target the project can use. We use the double colons (::) as +# a convention to tell CMake that the target name is associated with an IMPORTED target (which +# allows CMake to issue a diagnostic message if the library wasn't found). +add_library(wt::tcmalloc SHARED IMPORTED GLOBAL) +set_target_properties(wt::tcmalloc PROPERTIES + IMPORTED_LOCATION ${HAVE_LIBTCMALLOC} +) +if(HAVE_LIBTCMALLOC_INCLUDES) + set_target_properties(wt::tcmalloc PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES ${HAVE_LIBTCMALLOC_INCLUDES} + ) +endif() diff --git a/src/third_party/wiredtiger/cmake/third_party/lz4.cmake b/src/third_party/wiredtiger/cmake/third_party/lz4.cmake new file mode 100644 index 00000000000..55653a0c8ed --- /dev/null +++ b/src/third_party/wiredtiger/cmake/third_party/lz4.cmake @@ -0,0 +1,50 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +if(NOT HAVE_LIBLZ4) + # We don't need to construct a lz4 library target. + return() +endif() + +if(TARGET wt::lz4) + # Avoid redefining the imported library, given this file can be used as an include. + return() +endif() + +# Define the imported lz4 library target that can be subsequently linked across the build system. +# We use the double colons (::) as a convention to tell CMake that the target name is associated +# with an IMPORTED target (which allows CMake to issue a diagnostic message if the library wasn't found). +add_library(wt::lz4 SHARED IMPORTED GLOBAL) +set_target_properties(wt::lz4 PROPERTIES + IMPORTED_LOCATION ${HAVE_LIBLZ4} +) +if (HAVE_LIBLZ4_INCLUDES) + set_target_properties(wt::lz4 PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES ${HAVE_LIBLZ4_INCLUDES} + ) +endif() diff --git a/src/third_party/wiredtiger/cmake/third_party/snappy.cmake b/src/third_party/wiredtiger/cmake/third_party/snappy.cmake new file mode 100644 index 00000000000..a7c3bc8540b --- /dev/null +++ b/src/third_party/wiredtiger/cmake/third_party/snappy.cmake @@ -0,0 +1,50 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +if(NOT HAVE_LIBSNAPPY) + # We don't need to construct a snappy library target. + return() +endif() + +if(TARGET wt::snappy) + # Avoid redefining the imported library. + return() +endif() + +# Define the imported snappy library target that can be subsequently linked across the build system. +# We use the double colons (::) as a convention to tell CMake that the target name is associated +# with an IMPORTED target (which allows CMake to issue a diagnostic message if the library wasn't found). +add_library(wt::snappy SHARED IMPORTED GLOBAL) +set_target_properties(wt::snappy PROPERTIES + IMPORTED_LOCATION ${HAVE_LIBSNAPPY} +) +if (HAVE_LIBSNAPPY_INCLUDES) + set_target_properties(wt::snappy PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES ${HAVE_LIBSNAPPY_INCLUDES} + ) +endif() diff --git a/src/third_party/wiredtiger/cmake/third_party/sodium.cmake b/src/third_party/wiredtiger/cmake/third_party/sodium.cmake new file mode 100644 index 00000000000..4def57766e7 --- /dev/null +++ b/src/third_party/wiredtiger/cmake/third_party/sodium.cmake @@ -0,0 +1,50 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +if(NOT HAVE_LIBSODIUM) + # We don't need to construct a sodium library target. + return() +endif() + +if(TARGET wt::sodium) + # Avoid redefining the imported library. + return() +endif() + +# Define the imported sodium library target that can be subsequently linked across the build system. +# We use the double colons (::) as a convention to tell CMake that the target name is associated +# with an IMPORTED target (which allows CMake to issue a diagnostic message if the library wasn't found). +add_library(wt::sodium SHARED IMPORTED GLOBAL) +set_target_properties(wt::sodium PROPERTIES + IMPORTED_LOCATION ${HAVE_LIBSODIUM} +) +if (HAVE_LIBSODIUM_INCLUDES) + set_target_properties(wt::sodium PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES ${HAVE_LIBSODIUM_INCLUDES} + ) +endif() diff --git a/src/third_party/wiredtiger/cmake/third_party/zlib.cmake b/src/third_party/wiredtiger/cmake/third_party/zlib.cmake new file mode 100644 index 00000000000..b3b89fd8498 --- /dev/null +++ b/src/third_party/wiredtiger/cmake/third_party/zlib.cmake @@ -0,0 +1,50 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +if(NOT HAVE_LIBZ) + # We don't need to construct a zlib library target. + return() +endif() + +if(TARGET wt::zlib) + # Avoid redefining the imported library. + return() +endif() + +# Define the imported zlib library target that can be subsequently linked across the build system. +# We use the double colons (::) as a convention to tell CMake that the target name is associated +# with an IMPORTED target (which allows CMake to issue a diagnostic message if the library wasn't found). +add_library(wt::zlib SHARED IMPORTED GLOBAL) +set_target_properties(wt::zlib PROPERTIES + IMPORTED_LOCATION ${HAVE_LIBZ} +) +if (HAVE_LIBZ_INCLUDES) + set_target_properties(wt::zlib PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES ${HAVE_LIBZ_INCLUDES} + ) +endif() diff --git a/src/third_party/wiredtiger/cmake/third_party/zstd.cmake b/src/third_party/wiredtiger/cmake/third_party/zstd.cmake new file mode 100644 index 00000000000..3f25c158e7d --- /dev/null +++ b/src/third_party/wiredtiger/cmake/third_party/zstd.cmake @@ -0,0 +1,50 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +if(NOT HAVE_LIBZSTD) + # We don't need to construct a zstd library target. + return() +endif() + +if(TARGET wt::zstd) + # Avoid redefining the imported library. + return() +endif() + +# Define the imported zstd library target that can be subsequently linked across the build system. +# We use the double colons (::) as a convention to tell CMake that the target name is associated +# with an IMPORTED target (which allows CMake to issue a diagnostic message if the library wasn't found). +add_library(wt::zstd SHARED IMPORTED GLOBAL) +set_target_properties(wt::zstd PROPERTIES + IMPORTED_LOCATION ${HAVE_LIBZSTD} +) +if (HAVE_LIBZSTD_INCLUDES) + set_target_properties(wt::zstd PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES ${HAVE_LIBZSTD_INCLUDES} + ) +endif() diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index 3d202b5f500..133b789e8cb 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -49,10 +49,10 @@ common_runtime_config = [ enable enhanced checking. ''', type='category', subconfig= [ Config('commit_timestamp', 'none', r''' - This option is no longer supported, retained for backward compatibility.''', + This option is no longer supported, retained for backward compatibility''', choices=['always', 'key_consistent', 'never', 'none']), Config('durable_timestamp', 'none', r''' - This option is no longer supported, retained for backward compatibility.''', + This option is no longer supported, retained for backward compatibility''', choices=['always', 'key_consistent', 'never', 'none']), Config('write_timestamp', 'off', r''' verify that commit timestamps are used per the configured @@ -339,7 +339,7 @@ file_config = format_meta + file_runtime_config + tiered_config + [ the file format''', choices=['btree']), Config('huffman_key', 'none', r''' - This option is no longer supported, retained for backward compatibility.'''), + This option is no longer supported, retained for backward compatibility'''), Config('huffman_value', 'none', r''' configure Huffman encoding for values. Permitted values are \c "none", \c "english", \c "utf8<file>" or \c "utf16<file>". @@ -363,17 +363,13 @@ file_config = format_meta + file_runtime_config + tiered_config + [ block compression is done''', min='512B', max='512MB'), Config('internal_item_max', '0', r''' - This option is no longer supported, retained for backward compatibility.''', + This option is no longer supported, retained for backward compatibility''', min=0), Config('internal_key_max', '0', r''' - the largest key stored in an internal node, in bytes. If set, keys - larger than the specified size are stored as overflow items (which - may require additional I/O to access). The default and the maximum - allowed value are both one-tenth the size of a newly split internal - page''', + This option is no longer supported, retained for backward compatibility''', min='0'), Config('key_gap', '10', r''' - This option is no longer supported, retained for backward compatibility.''', + This option is no longer supported, retained for backward compatibility''', min='0'), Config('leaf_key_max', '0', r''' the largest key stored in a leaf node, in bytes. If set, keys @@ -398,7 +394,7 @@ file_config = format_meta + file_runtime_config + tiered_config + [ a newly split leaf page''', min='0'), Config('leaf_item_max', '0', r''' - This option is no longer supported, retained for backward compatibility.''', + This option is no longer supported, retained for backward compatibility''', min=0), Config('memory_page_image_max', '0', r''' the maximum in-memory page image represented by a single storage block. @@ -466,6 +462,8 @@ lsm_meta = file_config + lsm_config + [ tiered_meta = file_meta + tiered_config + [ Config('last', '0', r''' the last allocated object ID'''), + Config('oldest', '1', r''' + the oldest allocated object ID'''), Config('tiers', '', r''' list of data sources to combine into a tiered storage structure''', type='list'), ] @@ -510,6 +508,49 @@ table_meta = format_meta + table_only_config # Connection runtime config, shared by conn.reconfigure and wiredtiger_open connection_runtime_config = [ + Config('block_cache', '', r''' + block cache configuration options''', + type='category', subconfig=[ + Config('cache_on_checkpoint', 'true', r''' + cache blocks written by a checkpoint''', + type='boolean'), + Config('cache_on_writes', 'true', r''' + cache blocks as they are written (other than checkpoint blocks)''', + type='boolean'), + Config('enabled', 'false', r''' + enable block cache''', + type='boolean'), + Config('blkcache_eviction_aggression', '1800', r''' + seconds an unused block remains in the cache before it is evicted''', + min='1', max='7200'), + Config('full_target', '95', r''' + the fraction of the block cache that must be full before eviction + will remove unused blocks''', + min='30', max='100'), + Config('size', '0', r''' + maximum memory to allocate for the block cache''', + min='0', max='10TB'), + Config('hashsize', '0', r''' + number of buckets in the hashtable that keeps track of blocks''', + min='512', max='256K'), + Config('max_percent_overhead', '10', r''' + maximum tolerated overhead expressed as the number of blocks added + and removed as percent of blocks looked up; cache population + and eviction will be suppressed if the overhead exceeds the + supplied threshold''', + min='1', max='500'), + Config('nvram_path', '', r''' + the absolute path to the file system mounted on the NVRAM device'''), + Config('percent_file_in_dram', '50', r''' + bypass cache for a file if the set percentage of the file fits in system DRAM + (as specified by block_cache.system_ram)''', + min='0', max='100'), + Config('system_ram', '0', r''' + the bytes of system DRAM available for caching filesystem blocks''', + min='0', max='1024GB'), + Config('type', '', r''' + cache location: DRAM or NVRAM'''), + ]), Config('cache_size', '100MB', r''' maximum heap memory to allocate for the cache. A database should configure either \c cache_size or \c shared_cache but not both''', @@ -640,41 +681,47 @@ connection_runtime_config = [ perform eviction in worker threads when the cache contains at least this much dirty content. It is a percentage of the cache size if the value is within the range of 1 to 100 or an absolute size when greater - than 100. The value is not allowed to exceed the \c cache_size.''', + than 100. The value is not allowed to exceed the \c cache_size and has + to be lower than its counterpart \c eviction_dirty_trigger''', min=1, max='10TB'), Config('eviction_dirty_trigger', '20', r''' trigger application threads to perform eviction when the cache contains at least this much dirty content. It is a percentage of the cache size if the value is within the range of 1 to 100 or an absolute size when - greater than 100. The value is not allowed to exceed the \c cache_size. + greater than 100. The value is not allowed to exceed the \c cache_size + and has to be greater than its counterpart \c eviction_dirty_target. This setting only alters behavior if it is lower than eviction_trigger ''', min=1, max='10TB'), Config('eviction_target', '80', r''' perform eviction in worker threads when the cache contains at least this much content. It is a percentage of the cache size if the value is within the range of 10 to 100 or an absolute size when greater than 100. - The value is not allowed to exceed the \c cache_size''', + The value is not allowed to exceed the \c cache_size and has to be lower + than its counterpart \c eviction_trigger''', min=10, max='10TB'), Config('eviction_trigger', '95', r''' trigger application threads to perform eviction when the cache contains at least this much content. It is a percentage of the cache size if the value is within the range of 10 to 100 or an absolute size when greater - than 100. The value is not allowed to exceed the \c cache_size''', + than 100. The value is not allowed to exceed the \c cache_size and has + to be greater than its counterpart \c eviction_target''', min=10, max='10TB'), Config('eviction_updates_target', '0', r''' perform eviction in worker threads when the cache contains at least this many bytes of updates. It is a percentage of the cache size if the value is within the range of 0 to 100 or an absolute size when greater than 100. Calculated as half of \c eviction_dirty_target by default. - The value is not allowed to exceed the \c cache_size''', + The value is not allowed to exceed the \c cache_size and has to be lower + than its counterpart \c eviction_updates_trigger''', min=0, max='10TB'), Config('eviction_updates_trigger', '0', r''' trigger application threads to perform eviction when the cache contains at least this many bytes of updates. It is a percentage of the cache size if the value is within the range of 1 to 100 or an absolute size when greater than 100\. Calculated as half of \c eviction_dirty_trigger by default. - The value is not allowed to exceed the \c cache_size. This setting only - alters behavior if it is lower than \c eviction_trigger''', + The value is not allowed to exceed the \c cache_size and has to be greater than + its counterpart \c eviction_updates_target. This setting only alters behavior + if it is lower than \c eviction_trigger''', min=0, max='10TB'), Config('file_manager', '', r''' control how file handles are managed''', @@ -807,12 +854,15 @@ connection_runtime_config = [ 'history_store_search', 'history_store_sweep_race', 'prepare_checkpoint_delay', 'split_1', 'split_2', 'split_3', 'split_4', 'split_5', 'split_6', 'split_7']), Config('verbose', '[]', r''' - enable messages for various events. Options are given as a - list, such as <code>"verbose=[evictserver,read]"</code>''', + enable messages for various subsystems and operations. Options are given as a list, + where each message type can optionally define an associated verbosity level, such as + <code>"verbose=[evictserver,read:1,rts:0]"</code>. Verbosity levels that can be provided + include <code>0</code> (INFO) and <code>1</code> (DEBUG).''', type='list', choices=[ 'api', 'backup', 'block', + 'block_cache', 'checkpoint', 'checkpoint_cleanup', 'checkpoint_progress', @@ -824,9 +874,9 @@ connection_runtime_config = [ 'evictserver', 'fileops', 'handleops', - 'log', 'history_store', 'history_store_activity', + 'log', 'lsm', 'lsm_manager', 'metadata', @@ -1040,6 +1090,12 @@ session_config = [ closed. This value is inherited from ::wiredtiger_open \c cache_cursors''', type='boolean'), + Config('cache_max_wait_ms', '0', r''' + the maximum number of milliseconds an application thread will wait + for space to be available in cache before giving up. + Default value will be the global setting of the + connection config''', + min=0), Config('ignore_cache_size', 'false', r''' when set, operations performed by this session ignore the cache size and are not blocked when the cache is full. Note that use of this @@ -1674,11 +1730,10 @@ methods = { applicable only for prepared transactions. Indicates if the prepare timestamp and the commit timestamp of this transaction can be rounded up. If the prepare timestamp is less than the oldest - timestamp, the prepare timestamp will be rounded to the oldest + timestamp, the prepare timestamp will be rounded to the oldest timestamp. If the commit timestamp is less than the prepare timestamp, the commit timestamp will be rounded up to the prepare - timestamp. Allows setting the prepared timestamp smaller than or equal - to the latest active read timestamp''', type='boolean'), + timestamp''', type='boolean'), Config('read', 'false', r''' if the read timestamp is less than the oldest timestamp, the read timestamp will be rounded up to the oldest timestamp''', diff --git a/src/third_party/wiredtiger/dist/filelist b/src/third_party/wiredtiger/dist/filelist index d2a2d93cff3..a37d0e116e0 100644 --- a/src/third_party/wiredtiger/dist/filelist +++ b/src/third_party/wiredtiger/dist/filelist @@ -2,6 +2,7 @@ # List of source files for WiredTiger library. src/block/block_addr.c +src/block/block_cache.c src/block/block_ckpt.c src/block/block_ckpt_scan.c src/block/block_compact.c diff --git a/src/third_party/wiredtiger/dist/s_clang-format.list b/src/third_party/wiredtiger/dist/s_clang-format.list index 51c807aa2a4..f50d772a061 100644 --- a/src/third_party/wiredtiger/dist/s_clang-format.list +++ b/src/third_party/wiredtiger/dist/s_clang-format.list @@ -17,3 +17,4 @@ src/os_posix/os_getopt.c src/support/hash_city.c src/support/hash_fnv.c src/support/stat.c +test/format/config_def.c diff --git a/src/third_party/wiredtiger/dist/s_clang-scan.diff b/src/third_party/wiredtiger/dist/s_clang-scan.diff index 6cc57d04cd5..6090177bb41 100644 --- a/src/third_party/wiredtiger/dist/s_clang-scan.diff +++ b/src/third_party/wiredtiger/dist/s_clang-scan.diff @@ -114,7 +114,3 @@ src/txn/txn_ckpt.c:775:19: warning: Although the value stored to 'logging' is us full = idle = logging = tracking = use_timestamp = false; ^ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 1 warning generated. -src/txn/txn_rollback_to_stable.c:1386:43: warning: Although the value stored to 'has_txn_updates_gt_than_ckpt_snap' is used in the enclosing expression, the value is never actually read from 'has_txn_updates_gt_than_ckpt_snap' - durable_ts_found = prepared_updates = has_txn_updates_gt_than_ckpt_snap = false; - ^ ~~~~~ -1 warning generated. diff --git a/src/third_party/wiredtiger/dist/s_define.list b/src/third_party/wiredtiger/dist/s_define.list index f3607a74127..8c0eb073007 100644 --- a/src/third_party/wiredtiger/dist/s_define.list +++ b/src/third_party/wiredtiger/dist/s_define.list @@ -20,6 +20,7 @@ WT_ATOMIC_CAS WT_ATOMIC_CAS_FUNC WT_ATOMIC_FUNC WT_BACKUP_INVALID +WT_BLKCACHE_ID_SIZE WT_BLOCK_DESC_SIZE WT_BLOCK_EXTLIST_VERSION_ORIG WT_BLOCK_HEADER_SIZE @@ -92,6 +93,7 @@ WT_TRET_ERROR_OK WT_TXN_UPDATE WT_UPDATE_SIZE WT_USE_OPENAT +WT_VERBOSE_DEFAULT WT_WITH_LOCK_NOWAIT WT_WITH_LOCK_WAIT __F @@ -99,3 +101,6 @@ __WIREDTIGER_EXT_H_ __WIREDTIGER_H_ __WT_INTERNAL_H __wt_bswap16 +__wt_verbose_error +__wt_verbose_level +__wt_verbose_level_multi diff --git a/src/third_party/wiredtiger/dist/s_docs b/src/third_party/wiredtiger/dist/s_docs index 428e7c29375..03a1b6b47a5 100755 --- a/src/third_party/wiredtiger/dist/s_docs +++ b/src/third_party/wiredtiger/dist/s_docs @@ -63,19 +63,6 @@ glossarychk() structurechk() { - # @page names should match the source file name - (cd ../src/docs && - grep @page *.dox | - sed 's/\([^:]*\)\.dox:.*@page \([^ ]*\) .*/\1 \2/g' | - sed 's/-/_/g' | awk '{ if ($1 != $2) { print $1 " != " $2; } }') > $t - test -s $t && { - echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" - echo "@page references don't match source file names" - sed -e 's/^/ /' < $t - echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" - e=1 - } - # sections are a global name space for doxygen, and must be uniquely # named or you can get the wrong results. For example, if you have # "@section foo ABC" and "@section foo DEF", they will both appear as diff --git a/src/third_party/wiredtiger/dist/s_function b/src/third_party/wiredtiger/dist/s_function index d9014cf5394..a6e0a5efc25 100755 --- a/src/third_party/wiredtiger/dist/s_function +++ b/src/third_party/wiredtiger/dist/s_function @@ -108,4 +108,13 @@ for f in `find bench examples ext src test -name '*.c' -o -name '*._inline.h'`; } done +# __wt_verbose with a bitwise OR category parameter. +for f in `find src -name '*.c' -o -name '*._inline.h'`; do + file_parse $f | python dist/s_function_verbose.py > $t + test -s $t && { + echo "$f: Invalid use of verbose category parameter (bitwise OR)." + sed 's/^/function @ line:/' < $t + } +done + exit 0 diff --git a/src/third_party/wiredtiger/dist/s_function_verbose.py b/src/third_party/wiredtiger/dist/s_function_verbose.py new file mode 100644 index 00000000000..a9a28807c21 --- /dev/null +++ b/src/third_party/wiredtiger/dist/s_function_verbose.py @@ -0,0 +1,23 @@ +# When calling '__wt_verbose', the second parameter can only be a single verbose +# category definition i.e. can't treat the parameter as a flag/mask value. +# Iterate over uses of '__wt_verbose' and detect any invalid uses where multiple +# verbose categories are bitwise OR'd. +import re, sys + +verbose_regex = re.compile('([0-9]+):\s*__wt_verbose\(.*?,(.*?)[\"\']') +bitwise_or_regex = re.compile('^.*(?<!\|)\|(?!\|).*$') +for line in sys.stdin: + # Find all uses of __wt_verbose in a given line, capturing the line number + # and 2nd paramter as groups. + m = verbose_regex.findall(line) + if len(m) != 0: + for verb_match in m: + if len(verb_match) != 2: + continue + line = verb_match[0] + verb_parameter = verb_match[1] + # Test if the verbose category parameter uses a bitwise OR. + bit_m = bitwise_or_regex.search(verb_parameter) + if bit_m != None: + sys.stdout.write(line) + sys.stdout.write('\n') diff --git a/src/third_party/wiredtiger/dist/s_longlines b/src/third_party/wiredtiger/dist/s_longlines index 16fa1252131..9d38a481fbc 100755 --- a/src/third_party/wiredtiger/dist/s_longlines +++ b/src/third_party/wiredtiger/dist/s_longlines @@ -12,6 +12,7 @@ l=`(cd .. && -e '/checksum\/zseries/d' \ -e '/config\/config_def\.c/d' \ -e '/dist\/stat_data\.py/d' \ + -e '/format\/config_def\.c/d' \ -e '/include\/extern\.h/d' \ -e '/include\/extern_posix\.h/d' \ -e '/include\/extern_win\.h/d' \ diff --git a/src/third_party/wiredtiger/dist/s_stat b/src/third_party/wiredtiger/dist/s_stat index 10eba129153..747b5a96bca 100755 --- a/src/third_party/wiredtiger/dist/s_stat +++ b/src/third_party/wiredtiger/dist/s_stat @@ -23,18 +23,19 @@ search=`sed \ # There are some fields that are used, but we can't detect it. cat << UNUSED_STAT_FIELDS btree_clean_checkpoint_timer +compress_hist_ratio_max lock_checkpoint_count lock_checkpoint_wait_application lock_checkpoint_wait_internal -lock_durable_timestamp_read_count -lock_durable_timestamp_wait_application -lock_durable_timestamp_wait_internal -lock_durable_timestamp_write_count lock_dhandle_read_count lock_dhandle_wait lock_dhandle_wait_application lock_dhandle_wait_internal lock_dhandle_write_count +lock_durable_timestamp_read_count +lock_durable_timestamp_wait_application +lock_durable_timestamp_wait_internal +lock_durable_timestamp_write_count lock_metadata_count lock_metadata_wait_application lock_metadata_wait_internal diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index b355d29c901..a2f43166fc2 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -35,6 +35,7 @@ BBBBBBBBBBBBBBBBBB BDB BDB's BIGENDIAN +BLKCACHE BOOL BSR BTREE @@ -143,6 +144,7 @@ FALLOC FALLTHROUGH FH FIXME +FLCS FLD FLSv FLv @@ -225,6 +227,7 @@ Kounavis LANGID LAS LF +LIBMEMKIND LLLLLL LLLLLLL LLVM @@ -296,6 +299,7 @@ NOVALUE NOWAIT NUL NUM +NVRAM NZD NetBSD NoAddr @@ -310,6 +314,7 @@ OVFL ObWgfvgw Obama Opcode +Optane Outfmt PARAM PFX @@ -408,6 +413,7 @@ TMP TODO TORTIOUS TSO +TW TXN TXNC Teardown @@ -441,6 +447,7 @@ Unordered Uryyb VALGRIND VARCHAR +VLCS VLDB VMSG VPM @@ -502,6 +509,7 @@ abcdef abcdefghijklmnopqrstuvwxyz addl addr +addrs agc alfred alloc @@ -544,6 +552,7 @@ bitpos bitstring bitwise blk +blkcache blkmod bm bnd @@ -733,6 +742,7 @@ directio dirlist disjunction disjunctions +disproportionally dlclose dlh dll @@ -825,6 +835,7 @@ filesystems fillms firstfit fixup +floatcmp floatnum fmt fmterr @@ -880,6 +891,8 @@ gt handleops handlep hardcoded +hashsize +hashtable hashval havesize hdr @@ -996,6 +1009,7 @@ lexicographically lf lfence libdatasource +libmemkind libs libsodium libtool @@ -1060,6 +1074,7 @@ memcmp memcpy memdup memget +memkind memmove memset memsize @@ -1128,6 +1143,7 @@ notsup notused novalue nowait +ns nsec nset nsnap @@ -1137,6 +1153,7 @@ nuls num numSymbols numbare +nvram objs offpage ofh @@ -1176,6 +1193,7 @@ pcpu perf pfx pluggable +pmem poc popen poptable @@ -1244,6 +1262,7 @@ readv realloc rebalance rebalancing +recency recno recnos reconfig @@ -1424,6 +1443,7 @@ unbuffered uncacheable uncompressed uncompressing +unconfigured uncustomized undef underflowed diff --git a/src/third_party/wiredtiger/dist/s_void b/src/third_party/wiredtiger/dist/s_void index 249c89fbbab..7ea8ce985dd 100755 --- a/src/third_party/wiredtiger/dist/s_void +++ b/src/third_party/wiredtiger/dist/s_void @@ -34,6 +34,7 @@ func_ok() -e '/int __bm_stat$/d' \ -e '/int __checkpoint_presync$/d' \ -e '/int __compact_uri_analyze$/d' \ + -e '/int __compact_walk_page_skip$/d' \ -e '/int __config_parser_close$/d' \ -e '/int __curlog_reset$/d' \ -e '/int __cursor_fix_implicit$/d' \ diff --git a/src/third_party/wiredtiger/dist/stat_data.py b/src/third_party/wiredtiger/dist/stat_data.py index 72cbbb3d470..62160e3ee7d 100644 --- a/src/third_party/wiredtiger/dist/stat_data.py +++ b/src/third_party/wiredtiger/dist/stat_data.py @@ -185,6 +185,27 @@ conn_stats = [ BlockStat('block_byte_write_checkpoint', 'bytes written for checkpoint', 'size'), BlockStat('block_byte_write_mmap', 'bytes written via memory map API', 'size'), BlockStat('block_byte_write_syscall', 'bytes written via system call API', 'size'), + BlockStat('block_cache_blocks', 'block cache total blocks'), + BlockStat('block_cache_blocks_evicted', 'block cache evicted blocks'), + BlockStat('block_cache_blocks_removed', 'block cache removed blocks'), + BlockStat('block_cache_blocks_update', 'block cache cached blocks updated'), + BlockStat('block_cache_blocks_insert_read', 'block cache total blocks inserted on read path'), + BlockStat('block_cache_blocks_insert_write', 'block cache total blocks inserted on write path'), + BlockStat('block_cache_bypass_chkpt', 'block cache number of put bypasses on checkpoint I/O'), + BlockStat('block_cache_bypass_get', 'block cache number of bypasses on get'), + BlockStat('block_cache_bypass_filesize', 'block cache file size causing bypass'), + BlockStat('block_cache_bypass_overhead_put', 'block cache number of bypasses due to overhead on put'), + BlockStat('block_cache_bypass_put', 'block cache number of bypasses on put because file is too small'), + BlockStat('block_cache_bypass_writealloc', 'block cache number of bypasses because no-write-allocate setting was on'), + BlockStat('block_cache_bytes', 'block cache total bytes'), + BlockStat('block_cache_bytes_update', 'block cache cached bytes updated'), + BlockStat('block_cache_bytes_insert_read', 'block cache total bytes inserted on read path'), + BlockStat('block_cache_bytes_insert_write', 'block cache total bytes inserted on write path'), + BlockStat('block_cache_data_refs', 'block cache lookups'), + BlockStat('block_cache_eviction_passes', 'block cache number of eviction passes'), + BlockStat('block_cache_hits', 'block cache number of hits including existence checks'), + BlockStat('block_cache_misses', 'block cache number of misses including existence checks'), + BlockStat('block_cache_not_evicted_overhead', 'block cache number of blocks not evicted due to overhead'), BlockStat('block_map_read', 'mapped blocks read'), BlockStat('block_preload', 'blocks pre-loaded'), BlockStat('block_read', 'blocks read'), @@ -211,7 +232,6 @@ conn_stats = [ CacheStat('cache_eviction_fail_active_children_on_an_internal_page', 'pages selected for eviction unable to be evicted because of active children on an internal page'), CacheStat('cache_eviction_fail_checkpoint_out_of_order_ts', 'pages selected for eviction unable to be evicted because of race between checkpoint and out of order timestamps handling'), CacheStat('cache_eviction_fail_in_reconciliation', 'pages selected for eviction unable to be evicted because of failure in reconciliation'), - CacheStat('cache_eviction_fail_parent_has_overflow_items', 'pages selected for eviction unable to be evicted as the parent page has overflow items'), CacheStat('cache_eviction_force', 'forced eviction - pages selected count'), CacheStat('cache_eviction_force_long_update_list', 'forced eviction - pages selected because of a large number of updates to a single item'), CacheStat('cache_eviction_force_clean', 'forced eviction - pages evicted that were clean count'), @@ -467,7 +487,6 @@ conn_stats = [ # Reconciliation statistics ########################################## RecStat('rec_maximum_seconds', 'maximum seconds spent in a reconciliation call', 'no_clear,no_scale,size'), - RecStat('rec_overflow_key_internal', 'internal-page overflow keys'), RecStat('rec_overflow_key_leaf', 'leaf-page overflow keys'), RecStat('rec_pages_with_prepare', 'page reconciliation calls that resulted in values with prepared transaction metadata'), RecStat('rec_pages_with_ts', 'page reconciliation calls that resulted in values with timestamps'), @@ -636,13 +655,11 @@ dsrc_stats = [ BtreeStat('btree_column_variable', 'column-store variable-size leaf pages', 'no_scale,tree_walk'), BtreeStat('btree_compact_pages_reviewed', 'btree compact pages reviewed', 'no_clear,no_scale'), BtreeStat('btree_compact_pages_skipped', 'btree compact pages skipped', 'no_clear,no_scale'), - BtreeStat('btree_compact_pages_write_selected', 'btree compact pages selected to be rewritten', 'no_clear,no_scale'), - BtreeStat('btree_compact_pages_rewritten', 'pages rewritten by compaction'), - BtreeStat('btree_compact_skipped', 'btree skipped by compaction as process would not reduce size'), + BtreeStat('btree_compact_pages_rewritten', 'btree compact pages rewritten', 'no_clear,no_scale'), + BtreeStat('btree_compact_skipped', 'btree skipped by compaction as process would not reduce size', 'no_clear,no_scale'), BtreeStat('btree_entries', 'number of key/value pairs', 'no_scale,tree_walk'), BtreeStat('btree_fixed_len', 'fixed-record size', 'max_aggregate,no_scale,size'), BtreeStat('btree_maximum_depth', 'maximum tree depth', 'max_aggregate,no_scale'), - BtreeStat('btree_maxintlkey', 'maximum internal page key size', 'max_aggregate,no_scale,size'), BtreeStat('btree_maxintlpage', 'maximum internal page size', 'max_aggregate,no_scale,size'), BtreeStat('btree_maxleafkey', 'maximum leaf page key size', 'max_aggregate,no_scale,size'), BtreeStat('btree_maxleafpage', 'maximum leaf page size', 'max_aggregate,no_scale,size'), @@ -686,9 +703,16 @@ dsrc_stats = [ ########################################## # Compression statistics ########################################## - CompressStat('compress_read', 'compressed pages read'), + CompressStat('compress_hist_ratio_2', 'number of blocks with compress ratio smaller than 2'), + CompressStat('compress_hist_ratio_4', 'number of blocks with compress ratio smaller than 4'), + CompressStat('compress_hist_ratio_8', 'number of blocks with compress ratio smaller than 8'), + CompressStat('compress_hist_ratio_16', 'number of blocks with compress ratio smaller than 16'), + CompressStat('compress_hist_ratio_32', 'number of blocks with compress ratio smaller than 32'), + CompressStat('compress_hist_ratio_64', 'number of blocks with compress ratio smaller than 64'), + CompressStat('compress_hist_ratio_max', 'number of blocks with compress ratio greater than 64'), CompressStat('compress_precomp_intl_max_page_size', 'compressed page maximum internal page size prior to compression', 'no_clear,no_scale,size'), CompressStat('compress_precomp_leaf_max_page_size', 'compressed page maximum leaf page size prior to compression ', 'no_clear,no_scale,size'), + CompressStat('compress_read', 'compressed pages read'), CompressStat('compress_write', 'compressed pages written'), CompressStat('compress_write_fail', 'page written failed to compress'), CompressStat('compress_write_too_small', 'page written was too small to compress'), @@ -741,7 +765,6 @@ dsrc_stats = [ RecStat('rec_multiblock_internal', 'internal page multi-block writes'), RecStat('rec_multiblock_leaf', 'leaf page multi-block writes'), RecStat('rec_multiblock_max', 'maximum blocks required for a page', 'max_aggregate,no_scale'), - RecStat('rec_overflow_key_internal', 'internal-page overflow keys'), RecStat('rec_overflow_key_leaf', 'leaf-page overflow keys'), RecStat('rec_overflow_value', 'overflow values written'), RecStat('rec_page_match', 'page checksum matches'), diff --git a/src/third_party/wiredtiger/examples/c/ex_all.c b/src/third_party/wiredtiger/examples/c/ex_all.c index c769ffff4b4..99c9d36d19a 100644 --- a/src/third_party/wiredtiger/examples/c/ex_all.c +++ b/src/third_party/wiredtiger/examples/c/ex_all.c @@ -1228,8 +1228,8 @@ main(int argc, char *argv[]) /* * Configure eviction to begin at 90% full, and run until the cache is only 75% dirty. */ - error_check( - wiredtiger_open(home, NULL, "create,eviction_trigger=90,eviction_dirty_target=75", &conn)); + error_check(wiredtiger_open(home, NULL, + "create,eviction_trigger=90,eviction_dirty_target=75,eviction_dirty_trigger=90", &conn)); /*! [Eviction configuration] */ error_check(conn->close(conn, NULL)); diff --git a/src/third_party/wiredtiger/ext/storage_sources/local_store/local_store.c b/src/third_party/wiredtiger/ext/storage_sources/local_store/local_store.c index ee5720a1844..fc1984b2547 100644 --- a/src/third_party/wiredtiger/ext/storage_sources/local_store/local_store.c +++ b/src/third_party/wiredtiger/ext/storage_sources/local_store/local_store.c @@ -123,6 +123,7 @@ typedef struct local_file_handle { */ static int local_bucket_path(WT_FILE_SYSTEM *, const char *, char **); static int local_cache_path(WT_FILE_SYSTEM *, const char *, char **); +static int local_home_path(WT_FILE_SYSTEM *, const char *, char **); static int local_configure(LOCAL_STORAGE *, WT_CONFIG_ARG *); static int local_configure_int(LOCAL_STORAGE *, WT_CONFIG_ARG *, const char *, uint32_t *); static int local_delay(LOCAL_STORAGE *); @@ -356,6 +357,16 @@ local_cache_path(WT_FILE_SYSTEM *file_system, const char *name, char **pathp) } /* + * local_home_path -- + * Construct the source pathname from the file system and local name. + */ +static int +local_home_path(WT_FILE_SYSTEM *file_system, const char *name, char **pathp) +{ + return (local_path(file_system, ((LOCAL_FILE_SYSTEM *)file_system)->home_dir, name, pathp)); +} + +/* * local_path -- * Construct a pathname from the file system and local name. */ @@ -667,15 +678,18 @@ local_flush(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, WT_FILE_SYST { LOCAL_STORAGE *local; int ret; - char *dest_path; + char *dest_path, *src_path; (void)config; /* unused */ - dest_path = NULL; + dest_path = src_path = NULL; local = (LOCAL_STORAGE *)storage_source; ret = 0; if (file_system == NULL || source == NULL || object == NULL) - return local_err(local, session, EINVAL, "ss_flush_finish: required arguments missing"); + return local_err(local, session, EINVAL, "ss_flush: required arguments missing"); + + if ((ret = local_home_path(file_system, source, &src_path)) != 0) + goto err; if ((ret = local_bucket_path(file_system, object, &dest_path)) != 0) goto err; @@ -683,13 +697,15 @@ local_flush(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, WT_FILE_SYST if ((ret = local_delay(local)) != 0) goto err; - if ((ret = local_file_copy(local, session, source, dest_path, WT_FS_OPEN_FILE_TYPE_DATA)) != 0) + if ((ret = local_file_copy(local, session, src_path, dest_path, WT_FS_OPEN_FILE_TYPE_DATA)) != + 0) goto err; local->object_writes++; err: free(dest_path); + free(src_path); return (ret); } @@ -703,16 +719,19 @@ local_flush_finish(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, { LOCAL_STORAGE *local; int ret; - char *dest_path; + char *dest_path, *src_path; (void)config; /* unused */ - dest_path = NULL; + dest_path = src_path = NULL; local = (LOCAL_STORAGE *)storage_source; ret = 0; if (file_system == NULL || source == NULL || object == NULL) return local_err(local, session, EINVAL, "ss_flush_finish: required arguments missing"); + if ((ret = local_home_path(file_system, source, &src_path)) != 0) + goto err; + if ((ret = local_cache_path(file_system, object, &dest_path)) != 0) goto err; @@ -721,7 +740,7 @@ local_flush_finish(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, * Link the object with the original local object. The could be replaced by a file copy if * portability is an issue. */ - if ((ret = link(source, dest_path)) != 0) { + if ((ret = link(src_path, dest_path)) != 0) { ret = local_err( local, session, errno, "ss_flush_finish link %s to %s failed", source, dest_path); goto err; @@ -731,6 +750,7 @@ local_flush_finish(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, ret = local_err(local, session, errno, "%s: ss_flush_finish chmod failed", dest_path); err: free(dest_path); + free(src_path); return (ret); } diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 777ecff8c4c..80d01e4d3f4 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-5.0", - "commit": "8fc3ec7e84a127ed181d64968e24ac2d86b1c7af" + "commit": "6f517e60fbc9c8e33032e75d683f53ae62ca531c" } diff --git a/src/third_party/wiredtiger/lang/python/setup_pip.py b/src/third_party/wiredtiger/lang/python/setup_pip.py index ede4f472548..2a1d63f8d7f 100755 --- a/src/third_party/wiredtiger/lang/python/setup_pip.py +++ b/src/third_party/wiredtiger/lang/python/setup_pip.py @@ -217,7 +217,7 @@ builtin_libraries = [b[1] for b in builtins] # Here's the configure/make operations we perform before the python extension # is linked. configure_cmds = [ - 'cmake -B cmake_pip_build -G Ninja -DENABLE_STATIC=1 -DCMAKE_C_FLAGS="${CFLAGS:-}" -DENABLE_PYTHON=1 ' + \ + 'cmake -B cmake_pip_build -G Ninja -DENABLE_STATIC=1 -DENABLE_SHARED=0 -DWITH_PIC=1 -DCMAKE_C_FLAGS="${CFLAGS:-}" -DENABLE_PYTHON=1 ' + \ ' '.join(map(lambda name: '-DHAVE_BUILTIN_EXTENSION_' + name.upper() + '=1', builtin_names)), ] @@ -280,12 +280,7 @@ wt_ext = Extension('_wiredtiger', extra_compile_args = cflags + cppflags, extra_link_args = ldflags, libraries = builtin_libraries, - # FIXME-WT-7905: Remove manual linking of static extension libraries. - # Unfortunately CMake's use of the builtin doesn't currently support linking in the extension - # objects into a static libwiredtiger archive. As a workaround, we need to manually link - # the ext/compressor libraries. - extra_objects = [ os.path.join(build_dir, 'libwiredtiger.a') ] + \ - list(map(lambda name: os.path.join(build_dir, 'ext', 'compressors', name) + '/libwiredtiger_' + name + '.a', builtin_names)), + extra_objects = [ os.path.join(build_dir, 'libwiredtiger.a') ], include_dirs = inc_paths, library_dirs = lib_paths, ) diff --git a/src/third_party/wiredtiger/src/block/block_cache.c b/src/third_party/wiredtiger/src/block/block_cache.c new file mode 100644 index 00000000000..b0e060b6d19 --- /dev/null +++ b/src/third_party/wiredtiger/src/block/block_cache.c @@ -0,0 +1,845 @@ +/*- + * Copyright (c) 2014-2020 MongoDB, Inc. + * Copyright (c) 2008-2014 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __blkcache_aggregate_metadata -- + * Sum the metadata for all buckets and write it into the cache wide variables. + */ +static void +__blkcache_aggregate_metadata(WT_BLKCACHE *blkcache) +{ + uint64_t bytes_used, num_data_blocks; + u_int i; + + bytes_used = num_data_blocks = 0; + + for (i = 0; i < blkcache->hash_size; i++) { + bytes_used += blkcache->bucket_metadata[i].bucket_bytes_used; + num_data_blocks += blkcache->bucket_metadata[i].bucket_num_data_blocks; + } + + blkcache->bytes_used = bytes_used; + blkcache->num_data_blocks = num_data_blocks; +} + +/* + * __blkcache_alloc -- + * Allocate a block of memory in the cache. + */ +static int +__blkcache_alloc(WT_SESSION_IMPL *session, size_t size, void **retp) +{ + WT_BLKCACHE *blkcache; + + blkcache = &S2C(session)->blkcache; + + if (blkcache->type == BLKCACHE_DRAM) + return (__wt_malloc(session, size, retp)); + else if (blkcache->type == BLKCACHE_NVRAM) { +#ifdef HAVE_LIBMEMKIND + *retp = memkind_malloc(blkcache->pmem_kind, size); + if (*retp == NULL) + return (WT_BLKCACHE_FULL); +#else + WT_RET_MSG(session, EINVAL, "NVRAM block cache type requires libmemkind"); +#endif + } + return (0); +} + +/* + * __blkcache_free -- + * Free a chunk of memory. + */ +static void +__blkcache_free(WT_SESSION_IMPL *session, void *ptr) +{ + WT_BLKCACHE *blkcache; + + blkcache = &S2C(session)->blkcache; + + if (blkcache->type == BLKCACHE_DRAM) + __wt_free(session, ptr); + else if (blkcache->type == BLKCACHE_NVRAM) { +#ifdef HAVE_LIBMEMKIND + memkind_free(blkcache->pmem_kind, ptr); +#else + __wt_err(session, EINVAL, "NVRAM block cache type requires libmemkind"); +#endif + } +} + +/* + * __blkcache_update_ref_histogram -- + * Update the histogram of block accesses when the block is freed or on exit. + */ +static void +__blkcache_update_ref_histogram(WT_SESSION_IMPL *session, WT_BLKCACHE_ITEM *blkcache_item, int type) +{ + WT_BLKCACHE *blkcache; + u_int bucket; + + blkcache = &S2C(session)->blkcache; + + bucket = blkcache_item->num_references / BLKCACHE_HIST_BOUNDARY; + if (bucket > BLKCACHE_HIST_BUCKETS - 1) + bucket = BLKCACHE_HIST_BUCKETS - 1; + + blkcache->cache_references[bucket]++; + + if (type == BLKCACHE_RM_FREE) + blkcache->cache_references_removed_blocks[bucket]++; + else if (type == BLKCACHE_RM_EVICTION) + blkcache->cache_references_evicted_blocks[bucket]++; +} + +/* + * __blkcache_print_reference_hist -- + * Print a histogram showing how a type of block given in the header is reused. + */ +static void +__blkcache_print_reference_hist(WT_SESSION_IMPL *session, const char *header, uint32_t *hist) +{ + int j; + + __wt_verbose(session, WT_VERB_BLKCACHE, "%s:", header); + __wt_verbose(session, WT_VERB_BLKCACHE, "%s", "Reuses \t Number of blocks"); + __wt_verbose(session, WT_VERB_BLKCACHE, "%s", "-----------------------------"); + for (j = 0; j < BLKCACHE_HIST_BUCKETS; j++) { + __wt_verbose(session, WT_VERB_BLKCACHE, "[%d - %d] \t %u", j * BLKCACHE_HIST_BOUNDARY, + (j + 1) * BLKCACHE_HIST_BOUNDARY, hist[j]); + } +} + +/* + * __blkcache_high_overhead -- + * Estimate the overhead of using the cache. The overhead comes from block insertions and + * removals, which produce writes. Writes disproportionally slow down the reads on Optane NVRAM. + */ +static inline bool +__blkcache_high_overhead(WT_SESSION_IMPL *session) +{ + WT_BLKCACHE *blkcache; + uint64_t ops; + + blkcache = &S2C(session)->blkcache; + + ops = blkcache->inserts + blkcache->removals; + return (blkcache->lookups > ops && ((ops * 100) / blkcache->lookups) > blkcache->overhead_pct); +} + +/* + * __blkcache_should_evict -- + * Decide if the block should be evicted. + */ +static bool +__blkcache_should_evict(WT_SESSION_IMPL *session, WT_BLKCACHE_ITEM *blkcache_item, int *reason) +{ + WT_BLKCACHE *blkcache; + + blkcache = &S2C(session)->blkcache; + *reason = BLKCACHE_EVICT_OTHER; + + /* + * Keep track of the smallest number of references for blocks that have not been recently + * accessed. + */ + if ((blkcache_item->freq_rec_counter < blkcache->evict_aggressive) && + (blkcache_item->num_references < blkcache->min_num_references)) + blkcache->min_num_references = blkcache_item->num_references; + + /* Don't evict if there is plenty of free space */ + if (blkcache->bytes_used < blkcache->full_target) + return (false); + + /* + * Don't evict if there is high overhead due to blocks being inserted/removed. Churn kills + * performance and evicting when churn is high will exacerbate the overhead. + */ + if (__blkcache_high_overhead(session)) { + WT_STAT_CONN_INCR(session, block_cache_not_evicted_overhead); + return (false); + } + + /* + * Evict if the block has not been accessed for the amount of time corresponding to the evict + * aggressive setting. Within the category of blocks that fit this criterion, choose those with + * the lowest number of accesses first. + */ + if (blkcache_item->freq_rec_counter < blkcache->evict_aggressive && + blkcache_item->num_references < (blkcache->min_num_references + BLKCACHE_MINREF_INCREMENT)) + return (true); + else { + *reason = BLKCACHE_NOT_EVICTION_CANDIDATE; + return (false); + } +} + +/* + * __blkcache_eviction_thread -- + * Periodically sweep the cache and evict unused blocks. + */ +static WT_THREAD_RET +__blkcache_eviction_thread(void *arg) +{ + WT_BLKCACHE *blkcache; + WT_BLKCACHE_ITEM *blkcache_item, *blkcache_item_tmp; + WT_SESSION_IMPL *session; + int i, reason; + bool no_eviction_candidates; + + session = (WT_SESSION_IMPL *)arg; + blkcache = &S2C(session)->blkcache; + + __wt_verbose(session, WT_VERB_BLKCACHE, + "Block cache eviction thread starting... " + "Aggressive target = %d, full target = %" PRIu64 ":", + blkcache->evict_aggressive, blkcache->full_target); + + while (!blkcache->blkcache_exiting) { + + /* + * Sweep the cache every second to ensure time-based decay of frequency/recency counters of + * resident blocks. + */ + __wt_sleep(1, 0); + + /* Check if the cache is being destroyed */ + if (blkcache->blkcache_exiting) + return (0); + + /* Aggregate per bucket metadata into cache wide variables. */ + __blkcache_aggregate_metadata(blkcache); + + /* + * Walk the cache, gathering statistics and evicting blocks that are within our target. We + * sweep the cache every second, decrementing the frequency/recency counter of each block. + * Blocks whose counter goes below the threshold will get evicted. The threshold is set + * according to how soon we expect the blocks to become irrelevant. For example, if the + * threshold is set to 1800 seconds (=30 minutes), blocks that were used once but then + * weren't referenced for 30 minutes will be evicted. Blocks that were referenced a lot in + * the past but weren't referenced in the past 30 minutes will stay in the cache a bit + * longer, until their frequency/recency counter drops below the threshold. + * + * As we cycle through the blocks, we keep track of the minimum number of references + * observed for blocks whose frequency/recency counter has gone below the threshold. We will + * evict blocks with the smallest counter before evicting those with a larger one. + */ + no_eviction_candidates = true; + for (i = 0; i < (int)blkcache->hash_size; i++) { + __wt_spin_lock(session, &blkcache->hash_locks[i]); + TAILQ_FOREACH_SAFE(blkcache_item, &blkcache->hash[i], hashq, blkcache_item_tmp) + { + if (__blkcache_should_evict(session, blkcache_item, &reason)) { + TAILQ_REMOVE(&blkcache->hash[i], blkcache_item, hashq); + __blkcache_free(session, blkcache_item->data); + __blkcache_update_ref_histogram(session, blkcache_item, BLKCACHE_RM_EVICTION); + blkcache->bucket_metadata[i].bucket_num_data_blocks--; + blkcache->bucket_metadata[i].bucket_bytes_used -= blkcache_item->id.size; + + /* + * Update the number of removals because it is used to estimate the overhead, + * and we want the overhead contributed by eviction to be part of that + * calculation. + */ + blkcache->removals++; + + WT_STAT_CONN_INCR(session, block_cache_blocks_evicted); + WT_STAT_CONN_DECRV(session, block_cache_bytes, blkcache_item->id.size); + WT_STAT_CONN_DECR(session, block_cache_blocks); + __wt_free(session, blkcache_item); + } else { + blkcache_item->freq_rec_counter--; + if (reason != BLKCACHE_NOT_EVICTION_CANDIDATE) + no_eviction_candidates = false; + } + } + __wt_spin_unlock(session, &blkcache->hash_locks[i]); + if (blkcache->blkcache_exiting) + return (0); + } + if (no_eviction_candidates) { + blkcache->min_num_references += BLKCACHE_MINREF_INCREMENT; + } + + WT_STAT_CONN_INCR(session, block_cache_eviction_passes); + } + return (0); +} + +/* + * __blkcache_estimate_filesize -- + * Estimate the size of files used by this workload. + */ +static size_t +__blkcache_estimate_filesize(WT_SESSION_IMPL *session) +{ + WT_BLKCACHE *blkcache; + WT_BLOCK *block; + WT_CONNECTION_IMPL *conn; + size_t size; + uint64_t bucket; + + conn = S2C(session); + blkcache = &conn->blkcache; + + /* This is a deliberate race condition */ + if (blkcache->refs_since_filesize_estimated++ < BLKCACHE_FILESIZE_EST_FREQ) + return (blkcache->estimated_file_size); + + blkcache->refs_since_filesize_estimated = 0; + + size = 0; + __wt_spin_lock(session, &conn->block_lock); + for (bucket = 0; bucket < conn->hash_size; bucket++) { + TAILQ_FOREACH (block, &conn->blockhash[bucket], hashq) { + size += (size_t)block->size; + } + } + blkcache->estimated_file_size = size; + __wt_spin_unlock(session, &conn->block_lock); + + WT_STAT_CONN_SET(session, block_cache_bypass_filesize, blkcache->estimated_file_size); + + return (blkcache->estimated_file_size); +} + +/* + * __wt_blkcache_get_or_check -- + * Get a block from the cache or check if one exists. + */ +int +__wt_blkcache_get_or_check( + WT_SESSION_IMPL *session, wt_off_t offset, size_t size, uint32_t checksum, void *data_ptr) +{ + WT_BLKCACHE *blkcache; + WT_BLKCACHE_ID id; + WT_BLKCACHE_ITEM *blkcache_item; + uint64_t bucket, hash; + + blkcache = &S2C(session)->blkcache; + + if (blkcache->type == BLKCACHE_UNCONFIGURED) + return (-1); + + WT_STAT_CONN_INCR(session, block_cache_data_refs); + + /* + * We race to avoid using synchronization. We only care about an approximate value, so we accept + * inaccuracy for the sake of avoiding synchronization on the critical path. + */ + blkcache->lookups++; + + /* + * If more than the configured fraction of all file objects is likely to fit in the OS buffer + * cache, don't use this cache. + */ + if ((__blkcache_estimate_filesize(session) * blkcache->percent_file_in_os_cache) / 100 < + blkcache->system_ram) { + WT_STAT_CONN_INCR(session, block_cache_bypass_get); + return (WT_BLKCACHE_BYPASS); + } + + /* + * We will use the item id structure as a byte array for (1) computing a hash, and (2) for + * comparing with the items in the hash table. + */ + id.checksum = checksum; + id.offset = offset; + id.size = (uint32_t)size; + id.fid = S2BT(session)->id; + hash = __wt_hash_city64(&id, sizeof(id)); + + bucket = hash % blkcache->hash_size; + __wt_spin_lock(session, &blkcache->hash_locks[bucket]); + TAILQ_FOREACH (blkcache_item, &blkcache->hash[bucket], hashq) { + if (blkcache_item->id.checksum == id.checksum && blkcache_item->id.offset == id.offset && + blkcache_item->id.size == id.size && blkcache_item->id.fid == id.fid) { + if (data_ptr != NULL) + memcpy(data_ptr, blkcache_item->data, size); + + blkcache_item->num_references++; + if (blkcache_item->freq_rec_counter < 0) + blkcache_item->freq_rec_counter = 0; + blkcache_item->freq_rec_counter++; + + __wt_spin_unlock(session, &blkcache->hash_locks[bucket]); + WT_STAT_CONN_INCR(session, block_cache_hits); + __wt_verbose(session, WT_VERB_BLKCACHE, + "block found in cache: offset=%" PRIuMAX ", size=%" WT_SIZET_FMT ", checksum=%" PRIu32 + ", hash=%" PRIu64, + (uintmax_t)offset, size, checksum, hash); + return (0); + } + } + + /* Block not found */ + __wt_verbose(session, WT_VERB_BLKCACHE, + "block not found in cache: offset=%" PRIuMAX ", size=%" WT_SIZET_FMT ", checksum=%" PRIu32 + ", hash=%" PRIu64, + (uintmax_t)offset, size, checksum, hash); + + __wt_spin_unlock(session, &blkcache->hash_locks[bucket]); + WT_STAT_CONN_INCR(session, block_cache_misses); + return (-1); +} + +/* + * __wt_blkcache_put -- + * Put a block into the cache. + */ +int +__wt_blkcache_put(WT_SESSION_IMPL *session, wt_off_t offset, size_t size, uint32_t checksum, + void *data, bool checkpoint_io, bool write) +{ + WT_BLKCACHE *blkcache; + WT_BLKCACHE_ID id; + WT_BLKCACHE_ITEM *blkcache_item; + WT_DECL_RET; + uint64_t bucket, hash; + void *data_ptr; + + blkcache = &S2C(session)->blkcache; + data_ptr = NULL; + + if (blkcache->type == BLKCACHE_UNCONFIGURED) + return (-1); + + /* Bypass on write if the no-write-allocate setting is on */ + if (write && blkcache->cache_on_writes == false) { + WT_STAT_CONN_INCR(session, block_cache_bypass_writealloc); + return (-1); + } + + /* Are we within cache size limits? */ + if (blkcache->bytes_used >= blkcache->max_bytes) + return (WT_BLKCACHE_FULL); + + /* + * If more than the configured fraction of the file is likely to fit into the OS buffer cache, + * don't use this cache. + */ + if ((__blkcache_estimate_filesize(session) * blkcache->percent_file_in_os_cache) / 100 < + blkcache->system_ram) { + WT_STAT_CONN_INCR(session, block_cache_bypass_put); + return (WT_BLKCACHE_BYPASS); + } + + /* + * Do not write allocate if this block is written as part of checkpoint. Hot blocks get written + * and over-written a lot as part of checkpoint, so we don't want to cache them, because (a) + * they are in the DRAM cache anyway, and (b) they are likely to be overwritten anyway. + * + * Writes that are not part of checkpoint I/O are done in the service of eviction. Those are the + * blocks that the DRAM cache would like to keep but can't, and we definitely want to keep them. + */ + if ((blkcache->cache_on_checkpoint == false) && checkpoint_io) { + WT_STAT_CONN_INCR(session, block_cache_bypass_chkpt); + return (WT_BLKCACHE_BYPASS); + } + + /* Bypass on high overhead */ + if (__blkcache_high_overhead(session) == true) { + WT_STAT_CONN_INCR(session, block_cache_bypass_overhead_put); + return (WT_BLKCACHE_BYPASS); + } + /* + * Allocate space in the cache outside of the critical section. In the unlikely event that we + * fail to allocate metadata, or if the item exists and the caller did not check for that prior + * to calling this function, we will free the space. + */ + WT_RET(__blkcache_alloc(session, size, &data_ptr)); + + /* + * We will use the item id structure as a byte array for (1) computing a hash, and (2) for + * comparing with the items in the hash table. + */ + id.checksum = checksum; + id.offset = offset; + id.size = (uint32_t)size; + id.fid = S2BT(session)->id; + hash = __wt_hash_city64(&id, sizeof(id)); + + bucket = hash % blkcache->hash_size; + __wt_spin_lock(session, &blkcache->hash_locks[bucket]); + TAILQ_FOREACH (blkcache_item, &blkcache->hash[bucket], hashq) { + if (blkcache_item->id.checksum == id.checksum && blkcache_item->id.offset == id.offset && + blkcache_item->id.size == id.size && blkcache_item->id.fid == id.fid) + goto item_exists; + } + + WT_ERR(__wt_calloc_one(session, &blkcache_item)); + blkcache_item->id = id; + blkcache_item->data = data_ptr; + + /* + * Set the recency timestamp on newly inserted blocks to the maximum value to reduce the chance + * of them being evicted before they are reused. + */ + blkcache_item->freq_rec_counter = 1; + + if (data != NULL && blkcache_item->data != NULL) /* This makes static analyzers happier. */ + memcpy(blkcache_item->data, data, size); + TAILQ_INSERT_HEAD(&blkcache->hash[bucket], blkcache_item, hashq); + + blkcache->bucket_metadata[bucket].bucket_num_data_blocks++; + blkcache->bucket_metadata[bucket].bucket_bytes_used += size; + blkcache->inserts++; + + __wt_spin_unlock(session, &blkcache->hash_locks[bucket]); + + WT_STAT_CONN_INCRV(session, block_cache_bytes, size); + WT_STAT_CONN_INCR(session, block_cache_blocks); + if (write == true) { + WT_STAT_CONN_INCRV(session, block_cache_bytes_insert_write, size); + WT_STAT_CONN_INCR(session, block_cache_blocks_insert_write); + } else { + WT_STAT_CONN_INCRV(session, block_cache_bytes_insert_read, size); + WT_STAT_CONN_INCR(session, block_cache_blocks_insert_read); + } + + __wt_verbose(session, WT_VERB_BLKCACHE, + "block inserted in cache: offset=%" PRIuMAX ", size=%" WT_SIZET_FMT ", checksum=%" PRIu32 + ", hash=%" PRIu64, + (uintmax_t)offset, size, checksum, hash); + return (0); + +item_exists: + if (write) { + memcpy(blkcache_item->data, data, size); + WT_STAT_CONN_INCRV(session, block_cache_bytes_update, size); + WT_STAT_CONN_INCR(session, block_cache_blocks_update); + } + + __wt_verbose(session, WT_VERB_BLKCACHE, + "block exists during put: offset=%" PRIuMAX ", size=%" WT_SIZET_FMT + ", " + "checksum=%" PRIu32 ", hash=%" PRIu64, + (uintmax_t)offset, size, checksum, hash); +err: + __blkcache_free(session, data_ptr); + __wt_spin_unlock(session, &blkcache->hash_locks[bucket]); + return (ret); +} + +/* + * __wt_blkcache_remove -- + * Remove a block from the cache. + */ +void +__wt_blkcache_remove(WT_SESSION_IMPL *session, wt_off_t offset, size_t size, uint32_t checksum) +{ + WT_BLKCACHE *blkcache; + WT_BLKCACHE_ID id; + WT_BLKCACHE_ITEM *blkcache_item; + uint64_t bucket, hash; + + blkcache = &S2C(session)->blkcache; + + if (blkcache->type == BLKCACHE_UNCONFIGURED) + return; + + /* + * We will use the item id structure as a byte array for (1) computing a hash, and (2) for + * comparing with the items in the hash table. + */ + id.checksum = checksum; + id.offset = offset; + id.size = (uint32_t)size; + id.fid = S2BT(session)->id; + hash = __wt_hash_city64(&id, sizeof(id)); + + bucket = hash % blkcache->hash_size; + __wt_spin_lock(session, &blkcache->hash_locks[bucket]); + TAILQ_FOREACH (blkcache_item, &blkcache->hash[bucket], hashq) { + if (blkcache_item->id.checksum == id.checksum && blkcache_item->id.offset == id.offset && + blkcache_item->id.size == id.size && blkcache_item->id.fid == id.fid) { + TAILQ_REMOVE(&blkcache->hash[bucket], blkcache_item, hashq); + blkcache->bucket_metadata[bucket].bucket_num_data_blocks--; + blkcache->bucket_metadata[bucket].bucket_bytes_used -= size; + __blkcache_update_ref_histogram(session, blkcache_item, BLKCACHE_RM_FREE); + __wt_spin_unlock(session, &blkcache->hash_locks[bucket]); + __blkcache_free(session, blkcache_item->data); + __wt_overwrite_and_free(session, blkcache_item); + WT_STAT_CONN_DECRV(session, block_cache_bytes, size); + WT_STAT_CONN_DECR(session, block_cache_blocks); + WT_STAT_CONN_INCR(session, block_cache_blocks_removed); + blkcache->removals++; + __wt_verbose(session, WT_VERB_BLKCACHE, + "block removed from cache: offset=%" PRIuMAX ", size=%" WT_SIZET_FMT + ", checksum=%" PRIu32 ", hash=%" PRIu64, + (uintmax_t)offset, size, checksum, hash); + return; + } + } + __wt_spin_unlock(session, &blkcache->hash_locks[bucket]); +} + +/* + * __blkcache_init -- + * Initialize the block cache. + */ +static int +__blkcache_init(WT_SESSION_IMPL *session, size_t cache_size, u_int hash_size, u_int type, + char *nvram_device_path, size_t system_ram, u_int percent_file_in_os_cache, bool cache_on_writes, + u_int overhead_pct, u_int evict_aggressive, uint64_t full_target, bool cache_on_checkpoint) +{ + WT_BLKCACHE *blkcache; + WT_DECL_RET; + uint64_t i; + + blkcache = &S2C(session)->blkcache; + blkcache->cache_on_checkpoint = cache_on_checkpoint; + blkcache->cache_on_writes = cache_on_writes; + blkcache->hash_size = hash_size; + blkcache->percent_file_in_os_cache = percent_file_in_os_cache; + blkcache->full_target = full_target; + blkcache->max_bytes = cache_size; + blkcache->overhead_pct = overhead_pct; + blkcache->system_ram = system_ram; + + if (type == BLKCACHE_NVRAM) { +#ifdef HAVE_LIBMEMKIND + if ((ret = memkind_create_pmem(nvram_device_path, 0, &blkcache->pmem_kind)) != 0) + WT_RET_MSG(session, ret, "block cache failed to initialize: memkind_create_pmem"); + + WT_RET(__wt_strndup( + session, nvram_device_path, strlen(nvram_device_path), &blkcache->nvram_device_path)); + __wt_free(session, nvram_device_path); +#else + (void)nvram_device_path; + WT_RET_MSG(session, EINVAL, "NVRAM block cache type requires libmemkind"); +#endif + } + + WT_RET(__wt_calloc_def(session, blkcache->hash_size, &blkcache->hash)); + WT_RET(__wt_calloc_def(session, blkcache->hash_size, &blkcache->hash_locks)); + WT_RET(__wt_calloc_def(session, blkcache->hash_size, &blkcache->bucket_metadata)); + + for (i = 0; i < blkcache->hash_size; i++) { + TAILQ_INIT(&blkcache->hash[i]); /* Block cache hash lists */ + WT_RET(__wt_spin_init(session, &blkcache->hash_locks[i], "block cache bucket locks")); + } + + /* Create the eviction thread */ + WT_RET(__wt_thread_create( + session, &blkcache->evict_thread_tid, __blkcache_eviction_thread, (void *)session)); + blkcache->evict_aggressive = -((int)evict_aggressive); + blkcache->min_num_references = 1000; /* initialize to a large value */ + + blkcache->type = type; + + __wt_verbose(session, WT_VERB_BLKCACHE, + "block cache initialized: type=%s, size=%" WT_SIZET_FMT " path=%s", + (type == BLKCACHE_NVRAM) ? "nvram" : (type == BLKCACHE_DRAM) ? "dram" : "unconfigured", + cache_size, (blkcache->nvram_device_path == NULL) ? "--" : blkcache->nvram_device_path); + + return (ret); +} + +/* + * __wt_block_cache_destroy -- + * Destroy the block cache and free all memory. + */ +void +__wt_block_cache_destroy(WT_SESSION_IMPL *session) +{ + WT_BLKCACHE *blkcache; + WT_BLKCACHE_ITEM *blkcache_item; + WT_DECL_RET; + uint64_t i; + + blkcache = &S2C(session)->blkcache; + + __wt_verbose(session, WT_VERB_BLKCACHE, + "block cache with %" PRIu64 " bytes used to be destroyed", blkcache->bytes_used); + + if (blkcache->type == BLKCACHE_UNCONFIGURED) + return; + + blkcache->blkcache_exiting = true; + WT_TRET(__wt_thread_join(session, &blkcache->evict_thread_tid)); + __wt_verbose(session, WT_VERB_BLKCACHE, "%s", "block cache eviction thread exited"); + + __blkcache_aggregate_metadata(blkcache); + + if (blkcache->bytes_used == 0) + goto done; + + for (i = 0; i < blkcache->hash_size; i++) { + __wt_spin_lock(session, &blkcache->hash_locks[i]); + while (!TAILQ_EMPTY(&blkcache->hash[i])) { + blkcache_item = TAILQ_FIRST(&blkcache->hash[i]); + TAILQ_REMOVE(&blkcache->hash[i], blkcache_item, hashq); + /* + * Some workloads crash on freeing NVRAM arenas. If that occurs the call to free can be + * removed and the library/OS will clean up for us once the process exits. + */ + __blkcache_free(session, blkcache_item->data); + __blkcache_update_ref_histogram(session, blkcache_item, BLKCACHE_RM_EXIT); + blkcache->bucket_metadata[i].bucket_num_data_blocks--; + blkcache->bucket_metadata[i].bucket_bytes_used -= blkcache_item->id.size; + __wt_free(session, blkcache_item); + } + __wt_spin_unlock(session, &blkcache->hash_locks[i]); + } + __blkcache_aggregate_metadata(blkcache); + WT_ASSERT(session, blkcache->bytes_used == 0 && blkcache->num_data_blocks == 0); + +done: + /* Print reference histograms */ + __blkcache_print_reference_hist(session, "All blocks", blkcache->cache_references); + __blkcache_print_reference_hist( + session, "Removed blocks", blkcache->cache_references_removed_blocks); + __blkcache_print_reference_hist( + session, "Evicted blocks", blkcache->cache_references_evicted_blocks); + +#ifdef HAVE_LIBMEMKIND + if (blkcache->type == BLKCACHE_NVRAM) { + memkind_destroy_kind(blkcache->pmem_kind); + __wt_free(session, blkcache->nvram_device_path); + } +#endif + __wt_free(session, blkcache->hash); + __wt_free(session, blkcache->hash_locks); + __wt_free(session, blkcache->bucket_metadata); + /* + * Zeroing the structure has the effect of setting the block cache type to unconfigured. + */ + memset((void *)blkcache, 0, sizeof(WT_BLKCACHE)); +} + +/* + * __blkcache_reconfig -- + * We currently disallow reconfiguration. If and when we do, this function will destroy the + * block cache, making it ready for clean initialization. + */ +static int +__blkcache_reconfig(WT_SESSION_IMPL *session, bool reconfig, size_t cache_size, size_t hash_size, + u_int type, char *nvram_device_path, size_t system_ram, u_int percent_file_in_os_cache, + bool cache_on_writes, u_int overhead_pct, u_int evict_aggressive, uint64_t full_target, + bool cache_on_checkpoint) +{ + WT_BLKCACHE *blkcache; + + blkcache = &S2C(session)->blkcache; + + if (!reconfig || blkcache->type == BLKCACHE_UNCONFIGURED) + return (0); + + if (blkcache->cache_on_checkpoint != cache_on_checkpoint || + blkcache->cache_on_writes != cache_on_writes || blkcache->hash_size != hash_size || + blkcache->percent_file_in_os_cache != percent_file_in_os_cache || + blkcache->full_target != full_target || blkcache->max_bytes != cache_size || + blkcache->overhead_pct != overhead_pct || blkcache->system_ram != system_ram || + blkcache->evict_aggressive != -((int)evict_aggressive) || blkcache->type != type || + (nvram_device_path != NULL && blkcache->nvram_device_path == NULL) || + (nvram_device_path == NULL && blkcache->nvram_device_path != NULL) || + (nvram_device_path != NULL && blkcache->nvram_device_path != NULL && + (strlen(nvram_device_path) != strlen(blkcache->nvram_device_path))) || + (nvram_device_path != NULL && + strncmp(nvram_device_path, blkcache->nvram_device_path, strlen(nvram_device_path)) != 0)) { + __wt_err(session, EINVAL, "block cache reconfiguration not supported"); + return (WT_ERROR); + } + return (0); +} + +/* + * __wt_block_cache_setup -- + * Set up the block cache. + */ +int +__wt_block_cache_setup(WT_SESSION_IMPL *session, const char *cfg[], bool reconfig) +{ + WT_BLKCACHE *blkcache; + WT_CONFIG_ITEM cval; + WT_DECL_RET; + uint64_t cache_size, full_target, system_ram; + u_int cache_type, evict_aggressive, hash_size, overhead_pct, percent_file_in_os_cache; + char *nvram_device_path; + bool cache_on_checkpoint, cache_on_writes; + + blkcache = &S2C(session)->blkcache; + cache_on_checkpoint = cache_on_writes = true; + nvram_device_path = (char *)""; + + if (blkcache->type != BLKCACHE_UNCONFIGURED && !reconfig) + WT_RET_MSG(session, EINVAL, "block cache setup requested for a configured cache"); + + /* When reconfiguring, check if there are any modifications that we care about. */ + if (blkcache->type != BLKCACHE_UNCONFIGURED && reconfig) { + if ((ret = __wt_config_gets(session, cfg + 1, "block_cache", &cval)) == WT_NOTFOUND) + return (0); + WT_RET(ret); + } + + WT_RET(__wt_config_gets(session, cfg, "block_cache.enabled", &cval)); + if (cval.val == 0) + return (0); + + WT_RET(__wt_config_gets(session, cfg, "block_cache.size", &cval)); + if ((cache_size = (uint64_t)cval.val) <= 0) + WT_RET_MSG(session, EINVAL, "block cache size must be greater than zero"); + + WT_RET(__wt_config_gets(session, cfg, "block_cache.hashsize", &cval)); + if ((hash_size = (u_int)cval.val) == 0) + hash_size = BLKCACHE_HASHSIZE_DEFAULT; + else if (hash_size < BLKCACHE_HASHSIZE_MIN || hash_size > BLKCACHE_HASHSIZE_MAX) + WT_RET_MSG(session, EINVAL, "block cache hash size must be between %d and %d entries", + BLKCACHE_HASHSIZE_MIN, BLKCACHE_HASHSIZE_MAX); + + WT_RET(__wt_config_gets(session, cfg, "block_cache.type", &cval)); + if (WT_STRING_MATCH("dram", cval.str, cval.len) || WT_STRING_MATCH("DRAM", cval.str, cval.len)) + cache_type = BLKCACHE_DRAM; + else if (WT_STRING_MATCH("nvram", cval.str, cval.len) || + WT_STRING_MATCH("NVRAM", cval.str, cval.len)) { +#ifdef HAVE_LIBMEMKIND + cache_type = BLKCACHE_NVRAM; + WT_RET(__wt_config_gets(session, cfg, "block_cache.nvram_path", &cval)); + WT_RET(__wt_strndup(session, cval.str, cval.len, &nvram_device_path)); + WT_ASSERT(session, __wt_absolute_path(nvram_device_path)); +#else + WT_RET_MSG(session, EINVAL, "NVRAM block cache requires libmemkind"); +#endif + } else + WT_RET_MSG(session, EINVAL, "Invalid block cache type"); + + WT_RET(__wt_config_gets(session, cfg, "block_cache.system_ram", &cval)); + system_ram = (uint64_t)cval.val; + + WT_RET(__wt_config_gets(session, cfg, "block_cache.percent_file_in_dram", &cval)); + percent_file_in_os_cache = (u_int)cval.val; + + WT_RET(__wt_config_gets(session, cfg, "block_cache.cache_on_checkpoint", &cval)); + if (cval.val == 0) + cache_on_checkpoint = false; + + WT_RET(__wt_config_gets(session, cfg, "block_cache.blkcache_eviction_aggression", &cval)); + evict_aggressive = (u_int)cval.val; + + WT_RET(__wt_config_gets(session, cfg, "block_cache.full_target", &cval)); + full_target = (uint64_t)((float)cache_size * (float)cval.val / (float)100); + + WT_RET(__wt_config_gets(session, cfg, "block_cache.cache_on_writes", &cval)); + if (cval.val == 0) + cache_on_writes = false; + + WT_RET(__wt_config_gets(session, cfg, "block_cache.max_percent_overhead", &cval)); + overhead_pct = (u_int)cval.val; + + WT_RET(__blkcache_reconfig(session, reconfig, cache_size, hash_size, cache_type, + nvram_device_path, system_ram, percent_file_in_os_cache, cache_on_writes, overhead_pct, + evict_aggressive, full_target, cache_on_checkpoint)); + + return (__blkcache_init(session, cache_size, hash_size, cache_type, nvram_device_path, + system_ram, percent_file_in_os_cache, cache_on_writes, overhead_pct, evict_aggressive, + full_target, cache_on_checkpoint)); +} diff --git a/src/third_party/wiredtiger/src/block/block_compact.c b/src/third_party/wiredtiger/src/block/block_compact.c index 51614c4aae7..58deed3cb89 100644 --- a/src/third_party/wiredtiger/src/block/block_compact.c +++ b/src/third_party/wiredtiger/src/block/block_compact.c @@ -8,7 +8,7 @@ #include "wt_internal.h" -static void __block_dump_avail(WT_SESSION_IMPL *, WT_BLOCK *, bool); +static void __block_dump_file_stat(WT_SESSION_IMPL *, WT_BLOCK *, bool); /* * __wt_block_compact_start -- @@ -24,9 +24,9 @@ __wt_block_compact_start(WT_SESSION_IMPL *session, WT_BLOCK *block) /* Reset the compaction state information. */ block->compact_pct_tenths = 0; + block->compact_pages_rewritten = 0; block->compact_pages_reviewed = 0; block->compact_pages_skipped = 0; - block->compact_pages_written = 0; return (0); } @@ -42,15 +42,58 @@ __wt_block_compact_end(WT_SESSION_IMPL *session, WT_BLOCK *block) __wt_block_configure_first_fit(block, false); /* Dump the results of the compaction pass. */ - if (WT_VERBOSE_ISSET(session, WT_VERB_COMPACT)) { + if (WT_VERBOSE_LEVEL_ISSET(session, WT_VERB_COMPACT, WT_VERBOSE_DEBUG)) { __wt_spin_lock(session, &block->live_lock); - __block_dump_avail(session, block, false); + __block_dump_file_stat(session, block, false); __wt_spin_unlock(session, &block->live_lock); } return (0); } /* + * __wt_block_compact_get_progress_stats -- + * Collect compact progress stats. + */ +void +__wt_block_compact_get_progress_stats(WT_SESSION_IMPL *session, WT_BM *bm, + uint64_t *pages_reviewedp, uint64_t *pages_skippedp, uint64_t *pages_rewrittenp) +{ + WT_BLOCK *block; + + WT_UNUSED(session); + block = bm->block; + *pages_reviewedp = block->compact_pages_reviewed; + *pages_skippedp = block->compact_pages_skipped; + *pages_rewrittenp = block->compact_pages_rewritten; +} + +/* + * __wt_block_compact_progress -- + * Output compact progress message. + */ +void +__wt_block_compact_progress(WT_SESSION_IMPL *session, WT_BLOCK *block, u_int *msg_countp) +{ + struct timespec cur_time; + uint64_t time_diff; + + if (!WT_VERBOSE_LEVEL_ISSET(session, WT_VERB_COMPACT_PROGRESS, WT_VERBOSE_DEBUG)) + return; + + __wt_epoch(session, &cur_time); + + /* Log one progress message every twenty seconds. */ + time_diff = WT_TIMEDIFF_SEC(cur_time, session->compact->begin); + if (time_diff / WT_PROGRESS_MSG_PERIOD > *msg_countp) { + ++*msg_countp; + __wt_verbose_debug(session, WT_VERB_COMPACT_PROGRESS, + " compacting %s for %" PRIu64 " seconds; reviewed %" PRIu64 " pages, skipped %" PRIu64 + " pages, rewritten %" PRIu64 "pages", + block->name, time_diff, block->compact_pages_reviewed, block->compact_pages_skipped, + block->compact_pages_rewritten); + } +} +/* * __wt_block_compact_skip -- * Return if compaction will shrink the file. */ @@ -68,14 +111,19 @@ __wt_block_compact_skip(WT_SESSION_IMPL *session, WT_BLOCK *block, bool *skipp) * we need some metrics to decide if it's worth doing. Ignore small files, and files where we * are unlikely to recover 10% of the file. */ - if (block->size <= WT_MEGABYTE) + if (block->size <= WT_MEGABYTE) { + __wt_verbose_info(session, WT_VERB_COMPACT, + "%s: skipping because the file size must be greater than 1MB: %" PRIuMAX "B.", + block->name, (uintmax_t)block->size); + return (0); + } __wt_spin_lock(session, &block->live_lock); /* Dump the current state of the file. */ - if (WT_VERBOSE_ISSET(session, WT_VERB_COMPACT)) - __block_dump_avail(session, block, true); + if (WT_VERBOSE_LEVEL_ISSET(session, WT_VERB_COMPACT, WT_VERBOSE_DEBUG)) + __block_dump_file_stat(session, block, true); /* Sum the available bytes in the initial 80% and 90% of the file. */ avail_eighty = avail_ninety = 0; @@ -108,13 +156,18 @@ __wt_block_compact_skip(WT_SESSION_IMPL *session, WT_BLOCK *block, bool *skipp) block->compact_pct_tenths = 1; } - __wt_verbose(session, WT_VERB_COMPACT, + __wt_verbose_debug(session, WT_VERB_COMPACT, + "%s: total reviewed %" PRIu64 " pages, total skipped %" PRIu64 " pages, total wrote %" PRIu64 + " pages", + block->name, block->compact_pages_reviewed, block->compact_pages_skipped, + block->compact_pages_rewritten); + __wt_verbose_debug(session, WT_VERB_COMPACT, "%s: %" PRIuMAX "MB (%" PRIuMAX ") available space in the first 80%% of the file", block->name, (uintmax_t)avail_eighty / WT_MEGABYTE, (uintmax_t)avail_eighty); - __wt_verbose(session, WT_VERB_COMPACT, + __wt_verbose_debug(session, WT_VERB_COMPACT, "%s: %" PRIuMAX "MB (%" PRIuMAX ") available space in the first 90%% of the file", block->name, (uintmax_t)avail_ninety / WT_MEGABYTE, (uintmax_t)avail_ninety); - __wt_verbose(session, WT_VERB_COMPACT, + __wt_verbose_debug(session, WT_VERB_COMPACT, "%s: require 10%% or %" PRIuMAX "MB (%" PRIuMAX ") in the first 90%% of the file to perform compaction, compaction %s", block->name, (uintmax_t)(block->size / 10) / WT_MEGABYTE, (uintmax_t)block->size / 10, @@ -126,27 +179,22 @@ __wt_block_compact_skip(WT_SESSION_IMPL *session, WT_BLOCK *block, bool *skipp) } /* - * __wt_block_compact_page_skip -- + * __compact_page_skip -- * Return if writing a particular page will shrink the file. */ -int -__wt_block_compact_page_skip( - WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *addr, size_t addr_size, bool *skipp) +static void +__compact_page_skip( + WT_SESSION_IMPL *session, WT_BLOCK *block, wt_off_t offset, uint32_t size, bool *skipp) { WT_EXT *ext; WT_EXTLIST *el; - wt_off_t limit, offset; - uint32_t checksum, objectid, size; + wt_off_t limit; *skipp = true; /* Return a default skip. */ - /* Crack the cookie. */ - WT_RET(__wt_block_addr_unpack( - session, block, addr, addr_size, &objectid, &offset, &size, &checksum)); - /* * If this block is in the chosen percentage of the file and there's a block on the available - * list that's appears before that percentage of the file, rewrite the block. Checking the + * list that appears before that percentage of the file, rewrite the block. Checking the * available list is necessary (otherwise writing the block would extend the file), but there's * an obvious race if the file is sufficiently busy. */ @@ -164,44 +212,166 @@ __wt_block_compact_page_skip( } } __wt_spin_unlock(session, &block->live_lock); +} + +/* + * __wt_block_compact_page_skip -- + * Return if writing a particular page will shrink the file. + */ +int +__wt_block_compact_page_skip( + WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *addr, size_t addr_size, bool *skipp) +{ + wt_off_t offset; + uint32_t size, checksum, objectid; + + WT_UNUSED(addr_size); + *skipp = true; /* Return a default skip. */ + offset = 0; + + /* Crack the cookie. */ + WT_RET(__wt_block_addr_unpack( + session, block, addr, addr_size, &objectid, &offset, &size, &checksum)); + + __compact_page_skip(session, block, offset, size, skipp); ++block->compact_pages_reviewed; if (*skipp) ++block->compact_pages_skipped; else - ++block->compact_pages_written; + ++block->compact_pages_rewritten; return (0); } /* - * __block_dump_avail -- - * Dump out the avail list so we can see what compaction will look like. + * __wt_block_compact_page_rewrite -- + * Rewrite a page if it will shrink the file. + */ +int +__wt_block_compact_page_rewrite( + WT_SESSION_IMPL *session, WT_BLOCK *block, uint8_t *addr, size_t *addr_sizep, bool *skipp) +{ + WT_DECL_ITEM(tmp); + WT_DECL_RET; + wt_off_t offset, new_offset; + uint32_t size, checksum, objectid; + uint8_t *endp; + bool discard_block; + + *skipp = true; /* Return a default skip. */ + new_offset = 0; /* -Werror=maybe-uninitialized */ + + discard_block = false; + + WT_ERR(__wt_block_addr_unpack( + session, block, addr, *addr_sizep, &objectid, &offset, &size, &checksum)); + + /* Check if the block is worth rewriting. */ + __compact_page_skip(session, block, offset, size, skipp); + + if (*skipp) + return (0); + + /* Read the block. */ + WT_ERR(__wt_scr_alloc(session, size, &tmp)); + WT_ERR(__wt_read(session, block->fh, offset, size, tmp->mem)); + + /* Allocate a replacement block. */ + WT_ERR(__wt_block_ext_prealloc(session, 5)); + __wt_spin_lock(session, &block->live_lock); + ret = __wt_block_alloc(session, block, &new_offset, (wt_off_t)size); + __wt_spin_unlock(session, &block->live_lock); + WT_ERR(ret); + discard_block = true; + + /* Write the block. */ + WT_ERR(__wt_write(session, block->fh, new_offset, size, tmp->mem)); + + /* Free the original block. */ + __wt_spin_lock(session, &block->live_lock); + ret = __wt_block_off_free(session, block, objectid, offset, (wt_off_t)size); + __wt_spin_unlock(session, &block->live_lock); + WT_ERR(ret); + + /* Build the returned address cookie. */ + endp = addr; + WT_ERR(__wt_block_addr_pack(block, &endp, objectid, new_offset, size, checksum)); + *addr_sizep = WT_PTRDIFF(endp, addr); + + WT_STAT_CONN_INCR(session, block_write); + WT_STAT_CONN_INCRV(session, block_byte_write, size); + + discard_block = false; + +err: + if (discard_block) { + __wt_spin_lock(session, &block->live_lock); + WT_TRET(__wt_block_off_free(session, block, objectid, new_offset, (wt_off_t)size)); + __wt_spin_unlock(session, &block->live_lock); + } + __wt_scr_free(session, &tmp); + return (ret); +} + +/* + * __block_dump_bucket_stat -- + * Dump out the information about available and used blocks in the given bucket (part of the + * file). */ static void -__block_dump_avail(WT_SESSION_IMPL *session, WT_BLOCK *block, bool start) +__block_dump_bucket_stat(WT_SESSION_IMPL *session, uintmax_t file_size, uintmax_t file_free, + uintmax_t bucket_size, uintmax_t bucket_free, u_int bucket_pct) +{ + uintmax_t bucket_used, free_pct, used_pct; + + free_pct = used_pct = 0; + + /* Handle rounding error in which case bucket used size can be negative. */ + bucket_used = (bucket_size > bucket_free) ? (bucket_size - bucket_free) : 0; + + if (file_free != 0) + free_pct = (bucket_free * 100) / file_free; + + if (file_size > file_free) + used_pct = (bucket_used * 100) / (file_size - file_free); + + __wt_verbose_debug(session, WT_VERB_COMPACT, + "%2u%%: %12" PRIuMAX "MB, (free: %" PRIuMAX "B, %" PRIuMAX "%%), (used: %" PRIuMAX + "MB, %" PRIuMAX "B, %" PRIuMAX "%%)", + bucket_pct, bucket_free / WT_MEGABYTE, bucket_free, free_pct, bucket_used / WT_MEGABYTE, + bucket_used, used_pct); +} + +/* + * __block_dump_file_stat -- + * Dump out the avail/used list so we can see what compaction will look like. + */ +static void +__block_dump_file_stat(WT_SESSION_IMPL *session, WT_BLOCK *block, bool start) { WT_EXT *ext; WT_EXTLIST *el; - wt_off_t decile[10], percentile[100], size, v; + wt_off_t decile[10], percentile[100], size; + uintmax_t bucket_size; u_int i; el = &block->live.avail; size = block->size; - __wt_verbose(session, WT_VERB_COMPACT, "============ %s", + __wt_verbose_debug(session, WT_VERB_COMPACT, "============ %s", start ? "testing for compaction" : "ending compaction pass"); if (!start) { - __wt_verbose( + __wt_verbose_debug( session, WT_VERB_COMPACT, "pages reviewed: %" PRIu64, block->compact_pages_reviewed); - __wt_verbose( + __wt_verbose_debug( session, WT_VERB_COMPACT, "pages skipped: %" PRIu64, block->compact_pages_skipped); - __wt_verbose( - session, WT_VERB_COMPACT, "pages written: %" PRIu64, block->compact_pages_written); + __wt_verbose_debug( + session, WT_VERB_COMPACT, "pages rewritten : %" PRIu64, block->compact_pages_rewritten); } - __wt_verbose(session, WT_VERB_COMPACT, + __wt_verbose_debug(session, WT_VERB_COMPACT, "file size %" PRIuMAX "MB (%" PRIuMAX ") with %" PRIuMAX "%% space available %" PRIuMAX "MB (%" PRIuMAX ")", (uintmax_t)size / WT_MEGABYTE, (uintmax_t)size, @@ -226,19 +396,21 @@ __block_dump_avail(WT_SESSION_IMPL *session, WT_BLOCK *block, bool start) #ifdef __VERBOSE_OUTPUT_PERCENTILE /* * The verbose output always displays 10% buckets, running this code as well also displays 1% - * buckets. + * buckets. There will be rounding error in the `used` stats because of the bucket size + * calculation. Adding 50 to minimize the rounding error. */ - for (i = 0; i < WT_ELEMENTS(percentile); ++i) { - v = percentile[i] * 512; - __wt_verbose(session, WT_VERB_COMPACT, - "%2u%%: %12" PRIuMAX "MB, (%" PRIuMAX "B, %" PRIuMAX "%%)", i, (uintmax_t)v / WT_MEGABYTE, - (uintmax_t)v, (uintmax_t)((v * 100) / (wt_off_t)el->bytes)); - } + bucket_size = (uintmax_t)((size + 50) / 100); + for (i = 0; i < WT_ELEMENTS(percentile); ++i) + __block_dump_bucket_stat(session, (uintmax_t)size, (uintmax_t)el->bytes, bucket_size, + (uintmax_t)percentile[i] * 512, i); #endif - for (i = 0; i < WT_ELEMENTS(decile); ++i) { - v = decile[i] * 512; - __wt_verbose(session, WT_VERB_COMPACT, - "%2u%%: %12" PRIuMAX "MB, (%" PRIuMAX "B, %" PRIuMAX "%%)", i * 10, - (uintmax_t)v / WT_MEGABYTE, (uintmax_t)v, (uintmax_t)((v * 100) / (wt_off_t)el->bytes)); - } + + /* + * There will be rounding error in the `used` stats because of the bucket size calculation. + * Adding 5 to minimize the rounding error. + */ + bucket_size = (uintmax_t)((size + 5) / 10); + for (i = 0; i < WT_ELEMENTS(decile); ++i) + __block_dump_bucket_stat(session, (uintmax_t)size, (uintmax_t)el->bytes, bucket_size, + (uintmax_t)decile[i] * 512, i * 10); } diff --git a/src/third_party/wiredtiger/src/block/block_ext.c b/src/third_party/wiredtiger/src/block/block_ext.c index 37916e482de..32bde280e69 100644 --- a/src/third_party/wiredtiger/src/block/block_ext.c +++ b/src/third_party/wiredtiger/src/block/block_ext.c @@ -588,6 +588,9 @@ __wt_block_free(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *addr, ret = __wt_block_off_free(session, block, objectid, offset, (wt_off_t)size); __wt_spin_unlock(session, &block->live_lock); + /* Evict the freed block from the block cache */ + __wt_blkcache_remove(session, offset, size, checksum); + return (ret); } diff --git a/src/third_party/wiredtiger/src/block/block_mgr.c b/src/third_party/wiredtiger/src/block/block_mgr.c index 433d7342295..983de3d7c52 100644 --- a/src/third_party/wiredtiger/src/block/block_mgr.c +++ b/src/third_party/wiredtiger/src/block/block_mgr.c @@ -221,6 +221,32 @@ __bm_compact_end_readonly(WT_BM *bm, WT_SESSION_IMPL *session) } /* + * __bm_compact_page_rewrite -- + * Rewrite a page for compaction. + */ +static int +__bm_compact_page_rewrite( + WT_BM *bm, WT_SESSION_IMPL *session, uint8_t *addr, size_t *addr_sizep, bool *writtenp) +{ + return (__wt_block_compact_page_rewrite(session, bm->block, addr, addr_sizep, writtenp)); +} + +/* + * __bm_compact_page_rewrite_readonly -- + * Rewrite a page for compaction; readonly version. + */ +static int +__bm_compact_page_rewrite_readonly( + WT_BM *bm, WT_SESSION_IMPL *session, uint8_t *addr, size_t *addr_sizep, bool *writtenp) +{ + WT_UNUSED(addr); + WT_UNUSED(addr_sizep); + WT_UNUSED(writtenp); + + return (__bm_readonly(bm, session)); +} + +/* * __bm_compact_page_skip -- * Return if a page is useful for compaction. */ @@ -247,6 +273,16 @@ __bm_compact_page_skip_readonly( } /* + * __bm_compact_progress -- + * Output compact progress message. + */ +static void +__bm_compact_progress(WT_BM *bm, WT_SESSION_IMPL *session, u_int *msg_countp) +{ + __wt_block_compact_progress(session, bm->block, msg_countp); +} + +/* * __bm_compact_skip -- * Return if a file can be compacted. */ @@ -584,7 +620,9 @@ __bm_method_set(WT_BM *bm, bool readonly) bm->checkpoint_unload = __bm_checkpoint_unload; bm->close = __bm_close; bm->compact_end = __bm_compact_end; + bm->compact_page_rewrite = __bm_compact_page_rewrite; bm->compact_page_skip = __bm_compact_page_skip; + bm->compact_progress = __bm_compact_progress; bm->compact_skip = __bm_compact_skip; bm->compact_start = __bm_compact_start; bm->corrupt = __wt_bm_corrupt; @@ -612,6 +650,7 @@ __bm_method_set(WT_BM *bm, bool readonly) bm->checkpoint_resolve = __bm_checkpoint_resolve_readonly; bm->checkpoint_start = __bm_checkpoint_start_readonly; bm->compact_end = __bm_compact_end_readonly; + bm->compact_page_rewrite = __bm_compact_page_rewrite_readonly; bm->compact_page_skip = __bm_compact_page_skip_readonly; bm->compact_skip = __bm_compact_skip_readonly; bm->compact_start = __bm_compact_start_readonly; diff --git a/src/third_party/wiredtiger/src/block/block_read.c b/src/third_party/wiredtiger/src/block/block_read.c index 3fec75acfe8..0ed4c05d7b2 100644 --- a/src/third_party/wiredtiger/src/block/block_read.c +++ b/src/third_party/wiredtiger/src/block/block_read.c @@ -217,8 +217,10 @@ __wt_block_read_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, uin wt_off_t offset, uint32_t size, uint32_t checksum) { WT_BLOCK_HEADER *blk, swap; + WT_DECL_RET; WT_FH *fh; size_t bufsize; + int cret; __wt_verbose(session, WT_VERB_READ, "off %" PRIuMAX ", size %" PRIu32 ", checksum %#" PRIx32, (uintmax_t)offset, size, checksum); @@ -250,8 +252,22 @@ __wt_block_read_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, uin block->name, size, block->allocsize); WT_RET(__wt_buf_init(session, buf, bufsize)); + WT_RET(__wt_block_fh(session, block, objectid, &fh)); - WT_RET(__wt_read(session, fh, offset, size, buf->mem)); + + /* Ask the block cache to give us the block. If it doesn't have it, read it. */ + if (fh->file_type != WT_FS_OPEN_FILE_TYPE_DATA) + WT_RET(__wt_read(session, fh, offset, size, buf->mem)); + else { + if ((cret = __wt_blkcache_get_or_check(session, offset, size, checksum, buf->mem)) != 0) { + WT_RET(__wt_read(session, fh, offset, size, buf->mem)); + if (cret != WT_BLKCACHE_BYPASS) { + WT_TRET_ERROR_OK( + __wt_blkcache_put(session, offset, size, checksum, buf->mem, false, false), + WT_BLKCACHE_FULL); + } + } + } buf->size = size; /* diff --git a/src/third_party/wiredtiger/src/block/block_write.c b/src/third_party/wiredtiger/src/block/block_write.c index 442d69927f8..227395c7a1a 100644 --- a/src/third_party/wiredtiger/src/block/block_write.c +++ b/src/third_party/wiredtiger/src/block/block_write.c @@ -333,6 +333,11 @@ __block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, uint3 WT_RET(ret); } + if (block->fh->file_type == WT_FS_OPEN_FILE_TYPE_DATA) { + WT_TRET_ERROR_OK( + __wt_blkcache_put(session, offset, align_size, checksum, buf->mem, checkpoint_io, true), + WT_BLKCACHE_FULL); + } /* * Optionally schedule writes for dirty pages in the system buffer cache, but only if the * current session can wait. diff --git a/src/third_party/wiredtiger/src/btree/bt_compact.c b/src/third_party/wiredtiger/src/btree/bt_compact.c index 29e83c10beb..242b1c3d77b 100644 --- a/src/third_party/wiredtiger/src/btree/bt_compact.c +++ b/src/third_party/wiredtiger/src/btree/bt_compact.c @@ -9,11 +9,11 @@ #include "wt_internal.h" /* - * __compact_rewrite -- - * Return if a modified page needs to be re-written. + * __compact_page_inmem_check_addrs -- + * Return if a clean, in-memory page needs to be re-written. */ static int -__compact_rewrite(WT_SESSION_IMPL *session, WT_REF *ref, bool *skipp) +__compact_page_inmem_check_addrs(WT_SESSION_IMPL *session, WT_REF *ref, bool *skipp) { WT_ADDR_COPY addr; WT_BM *bm; @@ -25,7 +25,7 @@ __compact_rewrite(WT_SESSION_IMPL *session, WT_REF *ref, bool *skipp) bm = S2BT(session)->bm; - /* If the page is clean, test the original addresses. */ + /* If the page is currently clean, test the original addresses. */ if (__wt_page_evict_clean(ref->page)) return (__wt_ref_addr_copy(session, ref, &addr) ? bm->compact_page_skip(bm, session, addr.addr, addr.size, skipp) : @@ -53,63 +53,237 @@ __compact_rewrite(WT_SESSION_IMPL *session, WT_REF *ref, bool *skipp) } /* - * __compact_rewrite_lock -- - * Return if a page needs to be re-written. + * __compact_page_inmem -- + * Return if an in-memory page needs to be re-written. */ static int -__compact_rewrite_lock(WT_SESSION_IMPL *session, WT_REF *ref, bool *skipp) +__compact_page_inmem(WT_SESSION_IMPL *session, WT_REF *ref, bool *skipp) { - WT_BTREE *btree; - WT_DECL_RET; + *skipp = true; /* Default skip. */ + + /* + * Ignore dirty pages, checkpoint will likely write them. There are cases where checkpoint can + * skip dirty pages: to avoid that, we could alter the transactional information of the page, + * which is what checkpoint reviews to decide if a page can be skipped. Not doing that for now, + * the repeated checkpoints that compaction requires are more than likely to pick up all dirty + * pages at some point. + * + * Check clean page addresses, and mark page and tree dirty if the page needs to be rewritten. + */ + if (__wt_page_is_modified(ref->page)) + *skipp = false; + else { + WT_RET(__compact_page_inmem_check_addrs(session, ref, skipp)); + + if (!*skipp) { + WT_RET(__wt_page_modify_init(session, ref->page)); + __wt_page_modify_set(session, ref->page); + } + } + + /* If rewriting the page, have reconciliation write new blocks. */ + if (!*skipp) + F_SET_ATOMIC(ref->page, WT_PAGE_COMPACTION_WRITE); - btree = S2BT(session); + return (0); +} + +/* + * __compact_page_replace_addr -- + * Replace a page's WT_ADDR. + */ +static int +__compact_page_replace_addr(WT_SESSION_IMPL *session, WT_REF *ref, WT_ADDR_COPY *copy) +{ + WT_ADDR *addr; + WT_CELL_UNPACK_ADDR unpack; + WT_DECL_RET; /* - * Reviewing in-memory pages requires looking at page reconciliation results, because we care - * about where the page is stored now, not where the page was stored when we first read it into - * the cache. We need to ensure we don't race with page reconciliation as it's writing the page - * modify information. There are two ways we call reconciliation: checkpoints and eviction. We - * are holding a hazard pointer that blocks eviction, but there's nothing blocking a checkpoint. - * Get the tree's flush lock which blocks threads writing pages for checkpoints. If checkpoint - * is holding the lock, quit working this file, we'll visit it again in our next pass. + * If there's no address at all (the page has never been written), allocate a new WT_ADDR + * structure, otherwise, the address has already been instantiated, replace the cookie. */ - WT_RET(__wt_spin_trylock(session, &btree->flush_lock)); + addr = ref->addr; + WT_ASSERT(session, addr != NULL); + + if (__wt_off_page(ref->home, addr)) + __wt_free(session, addr->addr); + else { + __wt_cell_unpack_addr(session, ref->home->dsk, (WT_CELL *)addr, &unpack); + + WT_RET(__wt_calloc_one(session, &addr)); + addr->ta.newest_start_durable_ts = unpack.ta.newest_start_durable_ts; + addr->ta.newest_stop_durable_ts = unpack.ta.newest_stop_durable_ts; + addr->ta.oldest_start_ts = unpack.ta.oldest_start_ts; + addr->ta.newest_txn = unpack.ta.newest_txn; + addr->ta.newest_stop_ts = unpack.ta.newest_stop_ts; + addr->ta.newest_stop_txn = unpack.ta.newest_stop_txn; + switch (unpack.raw) { + case WT_CELL_ADDR_INT: + addr->type = WT_ADDR_INT; + break; + case WT_CELL_ADDR_LEAF: + addr->type = WT_ADDR_LEAF; + break; + case WT_CELL_ADDR_LEAF_NO: + addr->type = WT_ADDR_LEAF_NO; + break; + } + } - ret = __compact_rewrite(session, ref, skipp); + WT_ERR(__wt_strndup(session, copy->addr, copy->size, &addr->addr)); + addr->size = copy->size; - /* Unblock threads writing leaf pages. */ - __wt_spin_unlock(session, &btree->flush_lock); + ref->addr = addr; + return (0); +err: + if (addr != ref->addr) + __wt_free(session, addr); return (ret); } /* - * __compact_progress -- - * Output a compact progress message. + * __compact_page -- + * Compaction for a single page. */ -static void -__compact_progress(WT_SESSION_IMPL *session) +static int +__compact_page(WT_SESSION_IMPL *session, WT_REF *ref, bool *skipp) { - struct timespec cur_time; + WT_ADDR_COPY copy; WT_BM *bm; - uint64_t time_diff; + WT_DECL_RET; + size_t addr_size; + uint8_t previous_state; - if (!WT_VERBOSE_ISSET(session, WT_VERB_COMPACT_PROGRESS)) - return; + *skipp = true; /* Default skip. */ - bm = S2BT(session)->bm; - __wt_epoch(session, &cur_time); - - /* Log one progress message every twenty seconds. */ - time_diff = WT_TIMEDIFF_SEC(cur_time, session->compact->begin); - if (time_diff / WT_PROGRESS_MSG_PERIOD > session->compact->prog_msg_count) { - __wt_verbose(session, WT_VERB_COMPACT_PROGRESS, - "Compact running for %" PRIu64 " seconds; reviewed %" PRIu64 " pages, skipped %" PRIu64 - " pages, wrote %" PRIu64 " pages", - time_diff, bm->block->compact_pages_reviewed, bm->block->compact_pages_skipped, - bm->block->compact_pages_written); - session->compact->prog_msg_count++; + /* Lock the WT_REF. */ + WT_REF_LOCK(session, ref, &previous_state); + + /* + * Skip deleted pages but consider them progress (the on-disk block is discarded by the next + * checkpoint). + */ + if (previous_state == WT_REF_DELETED) + *skipp = false; + + /* + * If it's on-disk, get a copy of the address and ask the block manager to rewrite the block if + * it's useful. This is safe because we're holding the WT_REF locked, so nobody can read the + * page giving eviction a chance to modify the address. + * + * In this path, we are holding the WT_REF lock across two OS buffer cache I/Os (the read of the + * original block and the write of the new block), plus whatever overhead that entails. It's not + * ideal, we could release the lock, but then we'd have to deal with the block having been read + * into memory while we were moving it. + */ + if (previous_state == WT_REF_DISK && __wt_ref_addr_copy(session, ref, ©)) { + bm = S2BT(session)->bm; + addr_size = copy.size; + WT_ERR(bm->compact_page_rewrite(bm, session, copy.addr, &addr_size, skipp)); + if (!*skipp) { + copy.size = (uint8_t)addr_size; + WT_ERR(__compact_page_replace_addr(session, ref, ©)); + } + } + + /* + * Ignore pages that aren't in-memory for some reason other than they're on-disk, for example, + * they might have split or been deleted while we were locking the WT_REF. This includes the + * case where we found an on-disk page and either rewrite the block successfully or failed to + * get a copy of the address (which shouldn't ever happen, but if that goes wrong, it's not our + * problem to solve). + * + * In this path, we are holding the WT_REF lock across some in-memory checks and possibly one or + * more calls to the underlying block manager which is going to search the list of extents to + * figure out if the block is worth rewriting. It's not ideal because we're blocking the + * application's worker threads: we could release the lock, but then we'd have to acquire a + * hazard pointer to ensure eviction didn't select the page. + */ + if (previous_state == WT_REF_MEM) { + WT_ERR(__compact_page_inmem(session, ref, skipp)); + } + +err: + WT_REF_UNLOCK(ref, previous_state); + + return (ret); +} + +/* + * __compact_walk_internal -- + * Walk an internal page for compaction. + */ +static int +__compact_walk_internal(WT_SESSION_IMPL *session, WT_REF *parent) +{ + WT_DECL_RET; + WT_REF *ref; + bool overall_progress, skipp; + + ref = NULL; /* [-Wconditional-uninitialized] */ + + /* + * We could corrupt a checkpoint if we moved a block that's part of the checkpoint, that is, if + * we race with checkpoint's review of the tree. Get the tree's flush lock which blocks threads + * writing pages for checkpoints, and hold it long enough to review a single internal page. Quit + * working the file if checkpoint is holding the lock, checkpoint holds the lock for relatively + * long periods. + */ + WT_RET(__wt_spin_trylock(session, &S2BT(session)->flush_lock)); + + /* + * Walk the internal page and check any leaf pages it references; skip internal pages, we'll + * visit them individually. + */ + overall_progress = false; + WT_INTL_FOREACH_BEGIN (session, parent->page, ref) { + if (F_ISSET(ref, WT_REF_FLAG_LEAF)) { + WT_ERR(__compact_page(session, ref, &skipp)); + if (!skipp) + overall_progress = true; + } + } + WT_INTL_FOREACH_END; + + /* + * If we moved a leaf page, we'll write the parent. If we didn't move a leaf page, check pages + * other than the root to see if we want to move the internal page itself. (Skip the root as a + * forced checkpoint will always rewrite it, and you can't just "move" a root page.) + */ + if (!overall_progress && !__wt_ref_is_root(parent)) { + WT_ERR(__compact_page(session, parent, &skipp)); + if (!skipp) + overall_progress = true; } + + /* If we found a page to compact, mark the parent and tree dirty and report success. */ + if (overall_progress) { + WT_ERR(__wt_page_parent_modify_set(session, ref, false)); + session->compact_state = WT_COMPACT_SUCCESS; + } + +err: + /* Unblock checkpoint threads. */ + __wt_spin_unlock(session, &S2BT(session)->flush_lock); + + return (ret); +} + +/* + * __compact_walk_page_skip -- + * Skip leaf pages, all we want are internal pages. + */ +static int +__compact_walk_page_skip(WT_SESSION_IMPL *session, WT_REF *ref, void *context, bool *skipp) +{ + WT_UNUSED(context); + WT_UNUSED(session); + + /* All we want are the internal pages. */ + *skipp = F_ISSET(ref, WT_REF_FLAG_LEAF) ? true : false; + return (0); } /* @@ -122,17 +296,21 @@ __wt_compact(WT_SESSION_IMPL *session) WT_BM *bm; WT_DECL_RET; WT_REF *ref; - u_int i; + u_int i, msg_count; bool skip; + uint64_t stats_pages_rewritten; /* Pages rewritten */ + uint64_t stats_pages_reviewed; /* Pages reviewed */ + uint64_t stats_pages_skipped; /* Pages skipped */ + bm = S2BT(session)->bm; ref = NULL; WT_STAT_DATA_INCR(session, session_compact); /* - * Check if compaction might be useful -- the API layer will quit trying to compact the data - * source if we make no progress, set a flag if the block layer thinks compaction is possible. + * Check if compaction might be useful (the API layer will quit trying to compact the data + * source if we make no progress). */ WT_RET(bm->compact_skip(bm, session, &skip)); if (skip) { @@ -145,17 +323,18 @@ __wt_compact(WT_SESSION_IMPL *session) for (i = 0;;) { /* Track progress. */ - WT_STAT_DATA_SET(session, btree_compact_pages_reviewed, bm->block->compact_pages_reviewed); - WT_STAT_DATA_SET(session, btree_compact_pages_skipped, bm->block->compact_pages_skipped); - WT_STAT_DATA_SET( - session, btree_compact_pages_write_selected, bm->block->compact_pages_written); + __wt_block_compact_get_progress_stats( + session, bm, &stats_pages_reviewed, &stats_pages_skipped, &stats_pages_rewritten); + WT_STAT_DATA_SET(session, btree_compact_pages_reviewed, stats_pages_reviewed); + WT_STAT_DATA_SET(session, btree_compact_pages_skipped, stats_pages_skipped); + WT_STAT_DATA_SET(session, btree_compact_pages_rewritten, stats_pages_rewritten); /* * Periodically check if we've timed out or eviction is stuck. Quit if eviction is stuck, * we're making the problem worse. */ if (++i > 100) { - __compact_progress(session); + bm->compact_progress(bm, session, &msg_count); WT_ERR(__wt_session_compact_check_timeout(session)); if (__wt_cache_stuck(session)) @@ -176,94 +355,16 @@ __wt_compact(WT_SESSION_IMPL *session) * evicted quickly. */ WT_ERR(__wt_tree_walk_custom_skip( - session, &ref, __wt_compact_page_skip, NULL, WT_READ_NO_GEN | WT_READ_WONT_NEED)); + session, &ref, __compact_walk_page_skip, NULL, WT_READ_NO_GEN | WT_READ_WONT_NEED)); if (ref == NULL) break; - /* - * Cheap checks that don't require locking. - * - * Ignore the root: it may not have a replacement address, and besides, if anything else - * gets written, so will it. - * - * Ignore dirty pages, checkpoint will likely write them. There are cases where checkpoint - * can skip dirty pages: to avoid that, we could alter the transactional information of the - * page, which is what checkpoint reviews to decide if a page can be skipped. Not doing that - * for now, the repeated checkpoints that compaction requires are more than likely to pick - * up all dirty pages at some point. - */ - if (__wt_ref_is_root(ref)) - continue; - if (__wt_page_is_modified(ref->page)) - continue; - - WT_ERR(__compact_rewrite_lock(session, ref, &skip)); - if (skip) - continue; - - /* Rewrite the page: mark the page and tree dirty. */ - WT_ERR(__wt_page_modify_init(session, ref->page)); - __wt_page_modify_set(session, ref->page); - - session->compact_state = WT_COMPACT_SUCCESS; - WT_STAT_DATA_INCR(session, btree_compact_pages_rewritten); + WT_WITH_PAGE_INDEX(session, ret = __compact_walk_internal(session, ref)); + WT_ERR(ret); } err: - if (ref != NULL) - WT_TRET(__wt_page_release(session, ref, 0)); + WT_TRET(__wt_page_release(session, ref, 0)); return (ret); } - -/* - * __wt_compact_page_skip -- - * Return if compaction requires we read this page. - */ -int -__wt_compact_page_skip(WT_SESSION_IMPL *session, WT_REF *ref, void *context, bool *skipp) -{ - WT_ADDR_COPY addr; - WT_BM *bm; - uint8_t previous_state; - bool diskaddr; - - WT_UNUSED(context); - - *skipp = false; /* Default to reading */ - - /* Internal pages must be read to walk the tree. */ - if (F_ISSET(ref, WT_REF_FLAG_INTERNAL)) - return (0); - - /* - * Skip deleted pages, rewriting them doesn't seem useful; in a better world we'd write the - * parent to delete the page. - */ - if (ref->state == WT_REF_DELETED) { - *skipp = true; - return (0); - } - - /* - * If the page is in-memory, we want to look at it (it may have been modified and written, and - * the current location is the interesting one in terms of compaction, not the original). - */ - if (ref->state != WT_REF_DISK) - return (0); - - /* - * Lock the WT_REF and if it's still on-disk, get a copy of the address. This is safe because - * it's an on-disk page and we're holding the WT_REF locked, so nobody can read the page giving - * either checkpoint or eviction a chance to modify the address. - */ - WT_REF_LOCK(session, ref, &previous_state); - diskaddr = previous_state == WT_REF_DISK && __wt_ref_addr_copy(session, ref, &addr); - WT_REF_UNLOCK(ref, previous_state); - if (!diskaddr) - return (0); - - /* Ask the block-manager if it's useful to rewrite the page. */ - bm = S2BT(session)->bm; - return (bm->compact_page_skip(bm, session, addr.addr, addr.size, skipp)); -} diff --git a/src/third_party/wiredtiger/src/btree/bt_curnext.c b/src/third_party/wiredtiger/src/btree/bt_curnext.c index 6953eb3dc02..b2588e85df1 100644 --- a/src/third_party/wiredtiger/src/btree/bt_curnext.c +++ b/src/third_party/wiredtiger/src/btree/bt_curnext.c @@ -334,8 +334,8 @@ restart_read: * Move to the next row-store item. */ static inline int -__cursor_row_next( - WT_CURSOR_BTREE *cbt, bool newpage, bool restart, size_t *skippedp, WT_ITEM *prefix) +__cursor_row_next(WT_CURSOR_BTREE *cbt, bool newpage, bool restart, size_t *skippedp, + WT_ITEM *prefix, bool *prefix_key_out_of_bounds) { WT_CELL_UNPACK_KV kpack; WT_INSERT *ins; @@ -348,6 +348,7 @@ __cursor_row_next( key = &cbt->iface.key; page = cbt->ref->page; session = CUR2S(cbt); + *prefix_key_out_of_bounds = false; prefix_search = prefix != NULL && F_ISSET(&cbt->iface, WT_CURSTD_PREFIX_SEARCH); *skippedp = 0; @@ -401,6 +402,7 @@ restart_read_insert: * are visiting is after our prefix. */ if (prefix_search && __wt_prefix_match(prefix, key) < 0) { + *prefix_key_out_of_bounds = true; WT_STAT_CONN_DATA_INCR(session, cursor_search_near_prefix_fast_paths); return (WT_NOTFOUND); } @@ -446,6 +448,7 @@ restart_read_page: * visiting is after our prefix. */ if (prefix_search && __wt_prefix_match(prefix, &cbt->iface.key) < 0) { + *prefix_key_out_of_bounds = true; WT_STAT_CONN_DATA_INCR(session, cursor_search_near_prefix_fast_paths); return (WT_NOTFOUND); } @@ -690,9 +693,10 @@ __wt_btcur_next_prefix(WT_CURSOR_BTREE *cbt, WT_ITEM *prefix, bool truncating) WT_SESSION_IMPL *session; size_t total_skipped, skipped; uint32_t flags; - bool newpage, restart; + bool newpage, prefix_key_out_of_bounds, restart; cursor = &cbt->iface; + prefix_key_out_of_bounds = false; session = CUR2S(cbt); total_skipped = 0; @@ -750,14 +754,18 @@ __wt_btcur_next_prefix(WT_CURSOR_BTREE *cbt, WT_ITEM *prefix, bool truncating) total_skipped += skipped; break; case WT_PAGE_ROW_LEAF: - ret = __cursor_row_next(cbt, newpage, restart, &skipped, prefix); + ret = __cursor_row_next( + cbt, newpage, restart, &skipped, prefix, &prefix_key_out_of_bounds); total_skipped += skipped; /* - * We can directly return WT_NOTFOUND here as the caller expects the cursor to be - * positioned when traversing keys for prefix search near. + * If we are doing a search near with prefix key configured, we need to check if we + * have exited the cursor row next function due to a prefix key mismatch. If so, we + * can immediately return WT_NOTFOUND and we do not have to walk onto the next page. */ - if (ret == WT_NOTFOUND && F_ISSET(&cbt->iface, WT_CURSTD_PREFIX_SEARCH)) + if (prefix_key_out_of_bounds) { + WT_ASSERT(session, ret == WT_NOTFOUND); return (WT_NOTFOUND); + } break; default: WT_ERR(__wt_illegal_value(session, page->type)); diff --git a/src/third_party/wiredtiger/src/btree/bt_curprev.c b/src/third_party/wiredtiger/src/btree/bt_curprev.c index 3941ada539c..c628f0a5669 100644 --- a/src/third_party/wiredtiger/src/btree/bt_curprev.c +++ b/src/third_party/wiredtiger/src/btree/bt_curprev.c @@ -184,6 +184,9 @@ __cursor_fix_append_prev(WT_CURSOR_BTREE *cbt, bool newpage, bool restart) else __cursor_set_recno(cbt, cbt->recno - 1); + if (F_ISSET(&cbt->iface, WT_CURSTD_KEY_ONLY)) + return (0); + /* * Fixed-width column store appends are inherently non-transactional. Even a non-visible update * by a concurrent or aborted transaction changes the effective end of the data. The effect is @@ -249,6 +252,10 @@ new_page: cbt->ins = __col_insert_search(cbt->ins_head, cbt->ins_stack, cbt->next_stack, cbt->recno); if (cbt->ins != NULL && cbt->recno != WT_INSERT_RECNO(cbt->ins)) cbt->ins = NULL; + + if (F_ISSET(&cbt->iface, WT_CURSTD_KEY_ONLY)) + return (0); + /* * FIXME-WT-6127: Now we only do transaction read if we have an update chain and it doesn't work * in durable history. Review this when we have a plan for fixed-length column store. @@ -296,6 +303,10 @@ new_page: return (WT_NOTFOUND); __cursor_set_recno(cbt, WT_INSERT_RECNO(cbt->ins)); + + if (F_ISSET(&cbt->iface, WT_CURSTD_KEY_ONLY)) + return (0); + restart_read: WT_RET(__wt_txn_read_upd_list(session, cbt, cbt->ins->upd)); if (cbt->upd_value->type == WT_UPDATE_INVALID) { @@ -372,8 +383,12 @@ restart_read: cbt->ins_head = WT_COL_UPDATE_SLOT(page, cbt->slot); cbt->ins = __col_insert_search_match(cbt->ins_head, cbt->recno); __wt_upd_value_clear(cbt->upd_value); - if (cbt->ins != NULL) + if (cbt->ins != NULL) { + if (F_ISSET(&cbt->iface, WT_CURSTD_KEY_ONLY)) + return (0); + WT_RET(__wt_txn_read_upd_list(session, cbt, cbt->ins->upd)); + } if (cbt->upd_value->type != WT_UPDATE_INVALID) { if (cbt->upd_value->type == WT_UPDATE_TOMBSTONE) { if (cbt->upd_value->tw.stop_txn != WT_TXN_NONE && @@ -434,6 +449,9 @@ restart_read: continue; } + if (F_ISSET(&cbt->iface, WT_CURSTD_KEY_ONLY)) + return (0); + WT_RET(__wt_txn_read(session, cbt, NULL, cbt->recno, cbt->ins ? cbt->ins->upd : NULL)); if (cbt->upd_value->type == WT_UPDATE_INVALID || cbt->upd_value->type == WT_UPDATE_TOMBSTONE) { @@ -471,12 +489,10 @@ restart_read: /* * __cursor_row_prev -- - * Move to the previous row-store item. Taking an optional prefix item for a special case of - * search near. + * Move to the previous row-store item. */ static inline int -__cursor_row_prev( - WT_CURSOR_BTREE *cbt, bool newpage, bool restart, size_t *skippedp, WT_ITEM *prefix) +__cursor_row_prev(WT_CURSOR_BTREE *cbt, bool newpage, bool restart, size_t *skippedp) { WT_CELL_UNPACK_KV kpack; WT_INSERT *ins; @@ -484,12 +500,10 @@ __cursor_row_prev( WT_PAGE *page; WT_ROW *rip; WT_SESSION_IMPL *session; - bool prefix_search; key = &cbt->iface.key; page = cbt->ref->page; session = CUR2S(cbt); - prefix_search = prefix != NULL && F_ISSET(&cbt->iface, WT_CURSTD_PREFIX_SEARCH); *skippedp = 0; /* If restarting after a prepare conflict, jump to the right spot. */ @@ -543,14 +557,10 @@ restart_read_insert: if ((ins = cbt->ins) != NULL) { key->data = WT_INSERT_KEY(ins); key->size = WT_INSERT_KEY_SIZE(ins); - /* - * If the cursor has prefix search configured we can early exit here if the key we are - * visiting is before our prefix. - */ - if (prefix_search && __wt_prefix_match(prefix, key) > 0) { - WT_STAT_CONN_DATA_INCR(session, cursor_search_near_prefix_fast_paths); - return (WT_NOTFOUND); - } + + if (F_ISSET(&cbt->iface, WT_CURSTD_KEY_ONLY)) + return (0); + WT_RET(__wt_txn_read_upd_list(session, cbt, ins->upd)); if (cbt->upd_value->type == WT_UPDATE_INVALID) { ++*skippedp; @@ -590,14 +600,10 @@ restart_read_insert: restart_read_page: rip = &page->pg_row[cbt->slot]; WT_RET(__cursor_row_slot_key_return(cbt, rip, &kpack)); - /* - * If the cursor has prefix search configured we can early exit here if the key we are - * visiting is before our prefix. - */ - if (prefix_search && __wt_prefix_match(prefix, &cbt->iface.key) > 0) { - WT_STAT_CONN_DATA_INCR(session, cursor_search_near_prefix_fast_paths); - return (WT_NOTFOUND); - } + + if (F_ISSET(&cbt->iface, WT_CURSTD_KEY_ONLY)) + return (0); + WT_RET( __wt_txn_read(session, cbt, &cbt->iface.key, WT_RECNO_OOB, WT_ROW_UPDATE(page, rip))); if (cbt->upd_value->type == WT_UPDATE_INVALID) { @@ -617,11 +623,11 @@ restart_read_page: } /* - * __wt_btcur_prev_prefix -- + * __wt_btcur_prev -- * Move to the previous record in the tree. */ int -__wt_btcur_prev_prefix(WT_CURSOR_BTREE *cbt, WT_ITEM *prefix, bool truncating) +__wt_btcur_prev(WT_CURSOR_BTREE *cbt, bool truncating) { WT_CURSOR *cursor; WT_DECL_RET; @@ -659,6 +665,8 @@ __wt_btcur_prev_prefix(WT_CURSOR_BTREE *cbt, WT_ITEM *prefix, bool truncating) restart = F_ISSET(cbt, WT_CBT_ITERATE_RETRY_PREV); F_CLR(cbt, WT_CBT_ITERATE_RETRY_PREV); for (newpage = false;; newpage = true, restart = false) { + /* Calls with key only flag should never restart. */ + WT_ASSERT(session, !F_ISSET(&cbt->iface, WT_CURSTD_KEY_ONLY) || !restart); page = cbt->ref == NULL ? NULL : cbt->ref->page; /* @@ -700,7 +708,7 @@ __wt_btcur_prev_prefix(WT_CURSOR_BTREE *cbt, WT_ITEM *prefix, bool truncating) total_skipped += skipped; break; case WT_PAGE_ROW_LEAF: - ret = __cursor_row_prev(cbt, newpage, restart, &skipped, prefix); + ret = __cursor_row_prev(cbt, newpage, restart, &skipped); total_skipped += skipped; /* * We can directly return WT_NOTFOUND here as the caller will reset the cursor for @@ -743,7 +751,8 @@ __wt_btcur_prev_prefix(WT_CURSOR_BTREE *cbt, WT_ITEM *prefix, bool truncating) * information to determine if something is visible on the page. If nothing is, the page is * skipped. */ - if (session->txn->isolation == WT_ISO_SNAPSHOT && + if (!F_ISSET(&cbt->iface, WT_CURSTD_KEY_ONLY) && + session->txn->isolation == WT_ISO_SNAPSHOT && !F_ISSET(&cbt->iface, WT_CURSTD_IGNORE_TOMBSTONE)) WT_ERR( __wt_tree_walk_custom_skip(session, &cbt->ref, __wt_btcur_skip_page, NULL, flags)); @@ -762,7 +771,10 @@ err: switch (ret) { case 0: - F_SET(cursor, WT_CURSTD_KEY_INT | WT_CURSTD_VALUE_INT); + if (F_ISSET(&cbt->iface, WT_CURSTD_KEY_ONLY)) + F_SET(cursor, WT_CURSTD_KEY_INT); + else + F_SET(cursor, WT_CURSTD_KEY_INT | WT_CURSTD_VALUE_INT); #ifdef HAVE_DIAGNOSTIC /* * Skip key order check, if next is called after a prev returned a prepare conflict error, @@ -790,13 +802,3 @@ err: F_CLR(cbt, WT_CBT_ITERATE_RETRY_NEXT); return (ret); } - -/* - * __wt_btcur_prev -- - * Move to the previous record in the tree. - */ -int -__wt_btcur_prev(WT_CURSOR_BTREE *cbt, bool truncating) -{ - return (__wt_btcur_prev_prefix(cbt, NULL, truncating)); -} diff --git a/src/third_party/wiredtiger/src/btree/bt_cursor.c b/src/third_party/wiredtiger/src/btree/bt_cursor.c index 6cd2b3c3fc7..f7d5c7ebac8 100644 --- a/src/third_party/wiredtiger/src/btree/bt_cursor.c +++ b/src/third_party/wiredtiger/src/btree/bt_cursor.c @@ -741,9 +741,21 @@ __wt_btcur_search_near(WT_CURSOR_BTREE *cbt, int *exactp) } /* + * It is not necessary to go backwards when search_near is used with a prefix, as cursor row + * search places us on the first key of the prefix range. All the entries before the key we + * were placed on will not match the prefix. + * + * For example, if we search with the prefix "b", the cursor will be positioned at the first + * key starting with "b". All the entries before this one will start with "a", hence not + * matching the prefix. + */ + if (F_ISSET(cursor, WT_CURSTD_PREFIX_SEARCH)) + goto done; + + /* * We walked to the end of the tree without finding a match. Walk backwards instead. */ - while ((ret = __wt_btcur_prev_prefix(cbt, &state.key, false)) != WT_NOTFOUND) { + while ((ret = __wt_btcur_prev(cbt, false)) != WT_NOTFOUND) { WT_ERR(ret); if (btree->type == BTREE_ROW) WT_ERR(__wt_compare(session, btree->collator, &cursor->key, &state.key, &exact)); diff --git a/src/third_party/wiredtiger/src/btree/bt_debug.c b/src/third_party/wiredtiger/src/btree/bt_debug.c index 2e594da3388..f6d4fb0de1c 100644 --- a/src/third_party/wiredtiger/src/btree/bt_debug.c +++ b/src/third_party/wiredtiger/src/btree/bt_debug.c @@ -1128,7 +1128,7 @@ __debug_page_metadata(WT_DBG *ds, WT_REF *ref) WT_RET(ds->f(ds, ", disk-mapped")); if (F_ISSET_ATOMIC(page, WT_PAGE_EVICT_LRU)) WT_RET(ds->f(ds, ", evict-lru")); - if (F_ISSET_ATOMIC(page, WT_PAGE_OVERFLOW_KEYS)) + if (F_ISSET_ATOMIC(page, WT_PAGE_INTL_OVERFLOW_KEYS)) WT_RET(ds->f(ds, ", overflow-keys")); if (F_ISSET_ATOMIC(page, WT_PAGE_SPLIT_INSERT)) WT_RET(ds->f(ds, ", split-insert")); diff --git a/src/third_party/wiredtiger/src/btree/bt_handle.c b/src/third_party/wiredtiger/src/btree/bt_handle.c index d5251268944..30f75eaf55d 100644 --- a/src/third_party/wiredtiger/src/btree/bt_handle.c +++ b/src/third_party/wiredtiger/src/btree/bt_handle.c @@ -847,7 +847,7 @@ __btree_page_sizes(WT_SESSION_IMPL *session) WT_CONFIG_ITEM cval; WT_CONNECTION_IMPL *conn; uint64_t cache_size; - uint32_t intl_split_size, leaf_split_size, max; + uint32_t leaf_split_size, max; const char **cfg; btree = S2BT(session); @@ -924,7 +924,6 @@ __btree_page_sizes(WT_SESSION_IMPL *session) session->dhandle->name, WT_BTREE_MIN_SPLIT_PCT)); } else btree->split_pct = (int)cval.val; - intl_split_size = __wt_split_page_size(btree->split_pct, btree->maxintlpage, btree->allocsize); leaf_split_size = __wt_split_page_size(btree->split_pct, btree->maxleafpage, btree->allocsize); /* @@ -948,23 +947,15 @@ __btree_page_sizes(WT_SESSION_IMPL *session) * item in an in-memory configuration. */ if (F_ISSET(conn, WT_CONN_IN_MEMORY)) { - btree->maxintlkey = WT_BTREE_MAX_OBJECT_SIZE; btree->maxleafkey = WT_BTREE_MAX_OBJECT_SIZE; btree->maxleafvalue = WT_BTREE_MAX_OBJECT_SIZE; return (0); } /* - * In historic versions of WiredTiger, the maximum internal/leaf page key/value sizes were set - * by the internal_item_max and leaf_item_max configuration strings. Look for those strings if - * we don't find the newer ones. + * In historic versions of WiredTiger, the maximum leaf page key/value sizes were set by the + * leaf_item_max configuration string. Look for that string if we don't find the newer ones. */ - WT_RET(__wt_config_gets(session, cfg, "internal_key_max", &cval)); - btree->maxintlkey = (uint32_t)cval.val; - if (btree->maxintlkey == 0) { - WT_RET(__wt_config_gets(session, cfg, "internal_item_max", &cval)); - btree->maxintlkey = (uint32_t)cval.val; - } WT_RET(__wt_config_gets(session, cfg, "leaf_key_max", &cval)); btree->maxleafkey = (uint32_t)cval.val; WT_RET(__wt_config_gets(session, cfg, "leaf_value_max", &cval)); @@ -976,16 +967,12 @@ __btree_page_sizes(WT_SESSION_IMPL *session) } /* - * Default/maximum for internal and leaf page keys: split-page / 10. Default for leaf page - * values: split-page / 2. + * Default max for leaf keys: split-page / 10. Default max for leaf values: split-page / 2. * * It's difficult for applications to configure this in any exact way as they have to duplicate * our calculation of how many keys must fit on a page, and given a split-percentage and page - * header, that isn't easy to do. If the maximum internal key value is too large for the page, - * reset it to the default. + * header, that isn't easy to do. */ - if (btree->maxintlkey == 0 || btree->maxintlkey > intl_split_size / 10) - btree->maxintlkey = intl_split_size / 10; if (btree->maxleafkey == 0) btree->maxleafkey = leaf_split_size / 10; if (btree->maxleafvalue == 0) diff --git a/src/third_party/wiredtiger/src/btree/bt_io.c b/src/third_party/wiredtiger/src/btree/bt_io.c index 385968a8a2c..6e22631f7d7 100644 --- a/src/third_party/wiredtiger/src/btree/bt_io.c +++ b/src/third_party/wiredtiger/src/btree/bt_io.c @@ -9,6 +9,11 @@ #include "wt_internal.h" /* + * Define a function that increments histogram statistics compression ratios. + */ +WT_STAT_COMPR_RATIO_HIST_INCR_FUNC(ratio) + +/* * __wt_bt_read -- * Read a cookie referenced block into a buffer. */ @@ -23,7 +28,7 @@ __wt_bt_read(WT_SESSION_IMPL *session, WT_ITEM *buf, const uint8_t *addr, size_t WT_ENCRYPTOR *encryptor; WT_ITEM *ip; const WT_PAGE_HEADER *dsk; - size_t result_len; + size_t compression_ratio, result_len; const char *fail_msg; btree = S2BT(session); @@ -105,6 +110,10 @@ __wt_bt_read(WT_SESSION_IMPL *session, WT_ITEM *buf, const uint8_t *addr, size_t fail_msg = "block decompression failed"; goto corrupt; } + + compression_ratio = result_len / (tmp->size - WT_BLOCK_COMPRESS_SKIP); + __wt_stat_compr_ratio_hist_incr(session, compression_ratio); + } else { /* * If we uncompressed above, the page is in the correct buffer. If we get here the data may diff --git a/src/third_party/wiredtiger/src/btree/bt_page.c b/src/third_party/wiredtiger/src/btree/bt_page.c index d37b3cdb204..9e330f31118 100644 --- a/src/third_party/wiredtiger/src/btree/bt_page.c +++ b/src/third_party/wiredtiger/src/btree/bt_page.c @@ -660,10 +660,11 @@ __inmem_row_int(WT_SESSION_IMPL *session, WT_PAGE *page, size_t *sizep) /* * We track if an internal page has backing overflow keys, as overflow keys limit the eviction - * we can do during a checkpoint. + * we can do during a checkpoint. (This is only for historical tables, reconciliation no longer + * writes overflow cookies on internal pages, no matter the size of the key.) */ if (overflow_keys) - F_SET_ATOMIC(page, WT_PAGE_OVERFLOW_KEYS); + F_SET_ATOMIC(page, WT_PAGE_INTL_OVERFLOW_KEYS); err: __wt_scr_free(session, ¤t); diff --git a/src/third_party/wiredtiger/src/btree/bt_split.c b/src/third_party/wiredtiger/src/btree/bt_split.c index 2c2b255242c..54fc452bd49 100644 --- a/src/third_party/wiredtiger/src/btree/bt_split.c +++ b/src/third_party/wiredtiger/src/btree/bt_split.c @@ -160,7 +160,7 @@ __split_ovfl_key_cleanup(WT_SESSION_IMPL *session, WT_PAGE *page, WT_REF *ref) uint32_t cell_offset; /* There's a per-page flag if there are any overflow keys at all. */ - if (!F_ISSET_ATOMIC(page, WT_PAGE_OVERFLOW_KEYS)) + if (!F_ISSET_ATOMIC(page, WT_PAGE_INTL_OVERFLOW_KEYS)) return (0); /* diff --git a/src/third_party/wiredtiger/src/btree/bt_stat.c b/src/third_party/wiredtiger/src/btree/bt_stat.c index 9e452f39d2d..6248eea848b 100644 --- a/src/third_party/wiredtiger/src/btree/bt_stat.c +++ b/src/third_party/wiredtiger/src/btree/bt_stat.c @@ -33,7 +33,6 @@ __wt_btree_stat_init(WT_SESSION_IMPL *session, WT_CURSOR_STAT *cst) WT_STAT_SET(session, stats, btree_fixed_len, btree->bitcnt); WT_STAT_SET(session, stats, btree_maximum_depth, btree->maximum_depth); - WT_STAT_SET(session, stats, btree_maxintlkey, btree->maxintlkey); WT_STAT_SET(session, stats, btree_maxintlpage, btree->maxintlpage); WT_STAT_SET(session, stats, btree_maxleafkey, btree->maxleafkey); WT_STAT_SET(session, stats, btree_maxleafpage, btree->maxleafpage); diff --git a/src/third_party/wiredtiger/src/btree/bt_sync.c b/src/third_party/wiredtiger/src/btree/bt_sync.c index 8888fa61ae7..5886d71f2be 100644 --- a/src/third_party/wiredtiger/src/btree/bt_sync.c +++ b/src/third_party/wiredtiger/src/btree/bt_sync.c @@ -385,8 +385,12 @@ __sync_page_skip(WT_SESSION_IMPL *session, WT_REF *ref, void *context, bool *ski if (ref->state != WT_REF_DISK) return (0); - /* Don't read any pages when the cache is operating in aggressive mode. */ - if (__wt_cache_aggressive(session)) { + /* + * Reading any page that is not in the cache will increase the cache size. Perform a set of + * checks to verify the cache can handle it. + */ + if (__wt_cache_aggressive(session) || __wt_cache_full(session) || __wt_cache_stuck(session) || + __wt_eviction_needed(session, false, false, NULL)) { *skipp = true; return (0); } diff --git a/src/third_party/wiredtiger/src/btree/col_modify.c b/src/third_party/wiredtiger/src/btree/col_modify.c index c14d3251042..d1b305fe254 100644 --- a/src/third_party/wiredtiger/src/btree/col_modify.c +++ b/src/third_party/wiredtiger/src/btree/col_modify.c @@ -177,7 +177,7 @@ __wt_col_modify(WT_CURSOR_BTREE *cbt, uint64_t recno, const WT_ITEM *value, WT_U } else { /* Make sure the modify can proceed. */ if (cbt->compare == 0 && upd_arg == NULL) - WT_ERR(__wt_txn_modify_check(session, cbt, NULL, NULL)); + WT_ERR(__wt_txn_modify_check(session, cbt, NULL, &prev_upd_ts)); /* Allocate the append/update list reference as necessary. */ if (append) { @@ -216,6 +216,9 @@ __wt_col_modify(WT_CURSOR_BTREE *cbt, uint64_t recno, const WT_ITEM *value, WT_U if (upd_arg == NULL) { WT_ERR(__wt_upd_alloc(session, value, modify_type, &upd, &upd_size)); +#ifdef HAVE_DIAGNOSTIC + upd->prev_durable_ts = prev_upd_ts; +#endif WT_ERR(__wt_txn_modify(session, upd)); logged = true; diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index 6ea091cd3f6..f58f4973690 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -19,6 +19,7 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_load_extension[] = { static const WT_CONFIG_CHECK confchk_WT_CONNECTION_open_session[] = { {"cache_cursors", "boolean", NULL, NULL, NULL, 0}, + {"cache_max_wait_ms", "int", NULL, "min=0", NULL, 0}, {"ignore_cache_size", "boolean", NULL, NULL, NULL, 0}, {"isolation", "string", NULL, "choices=[\"read-uncommitted\",\"read-committed\"," @@ -33,6 +34,19 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_query_timestamp[] = { NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; +static const WT_CONFIG_CHECK confchk_wiredtiger_open_block_cache_subconfigs[] = { + {"blkcache_eviction_aggression", "int", NULL, "min=1,max=7200", NULL, 0}, + {"cache_on_checkpoint", "boolean", NULL, NULL, NULL, 0}, + {"cache_on_writes", "boolean", NULL, NULL, NULL, 0}, {"enabled", "boolean", NULL, NULL, NULL, 0}, + {"full_target", "int", NULL, "min=30,max=100", NULL, 0}, + {"hashsize", "int", NULL, "min=512,max=256K", NULL, 0}, + {"max_percent_overhead", "int", NULL, "min=1,max=500", NULL, 0}, + {"nvram_path", "string", NULL, NULL, NULL, 0}, + {"percent_file_in_dram", "int", NULL, "min=0,max=100", NULL, 0}, + {"size", "int", NULL, "min=0,max=10TB", NULL, 0}, + {"system_ram", "int", NULL, "min=0,max=1024GB", NULL, 0}, {"type", "string", NULL, NULL, NULL, 0}, + {NULL, NULL, NULL, NULL, NULL, 0}}; + static const WT_CONFIG_CHECK confchk_wiredtiger_open_checkpoint_subconfigs[] = { {"log_size", "int", NULL, "min=0,max=2GB", NULL, 0}, {"wait", "int", NULL, "min=0,max=100000", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; @@ -102,6 +116,7 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure_tiered_storage_su {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = { + {"block_cache", "category", NULL, NULL, confchk_wiredtiger_open_block_cache_subconfigs, 12}, {"cache_max_wait_ms", "int", NULL, "min=0", NULL, 0}, {"cache_overhead", "int", NULL, "min=0,max=30", NULL, 0}, {"cache_size", "int", NULL, "min=1MB,max=10TB", NULL, 0}, @@ -148,11 +163,11 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = { "\"split_6\",\"split_7\"]", NULL, 0}, {"verbose", "list", NULL, - "choices=[\"api\",\"backup\",\"block\",\"checkpoint\"," - "\"checkpoint_cleanup\",\"checkpoint_progress\",\"compact\"," - "\"compact_progress\",\"error_returns\",\"evict\",\"evict_stuck\"" - ",\"evictserver\",\"fileops\",\"handleops\",\"log\"," - "\"history_store\",\"history_store_activity\",\"lsm\"," + "choices=[\"api\",\"backup\",\"block\",\"block_cache\"," + "\"checkpoint\",\"checkpoint_cleanup\",\"checkpoint_progress\"," + "\"compact\",\"compact_progress\",\"error_returns\",\"evict\"," + "\"evict_stuck\",\"evictserver\",\"fileops\",\"handleops\"," + "\"history_store\",\"history_store_activity\",\"log\",\"lsm\"," "\"lsm_manager\",\"metadata\",\"mutex\",\"overflow\",\"read\"," "\"reconcile\",\"recovery\",\"recovery_progress\",\"rts\"," "\"salvage\",\"shared_cache\",\"split\",\"temporary\"," @@ -392,6 +407,7 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_query_timestamp[] = { static const WT_CONFIG_CHECK confchk_WT_SESSION_reconfigure[] = { {"cache_cursors", "boolean", NULL, NULL, NULL, 0}, + {"cache_max_wait_ms", "int", NULL, "min=0", NULL, 0}, {"ignore_cache_size", "boolean", NULL, NULL, NULL, 0}, {"isolation", "string", NULL, "choices=[\"read-uncommitted\",\"read-committed\"," @@ -755,7 +771,7 @@ static const WT_CONFIG_CHECK confchk_tiered_meta[] = { {"log", "category", NULL, NULL, confchk_WT_SESSION_create_log_subconfigs, 1}, {"memory_page_image_max", "int", NULL, "min=0", NULL, 0}, {"memory_page_max", "int", NULL, "min=512B,max=10TB", NULL, 0}, - {"os_cache_dirty_max", "int", NULL, "min=0", NULL, 0}, + {"oldest", "string", NULL, NULL, NULL, 0}, {"os_cache_dirty_max", "int", NULL, "min=0", NULL, 0}, {"os_cache_max", "int", NULL, "min=0", NULL, 0}, {"prefix_compression", "boolean", NULL, NULL, NULL, 0}, {"prefix_compression_min", "int", NULL, "min=0", NULL, 0}, @@ -818,6 +834,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_transaction_sync_subconfigs {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { + {"block_cache", "category", NULL, NULL, confchk_wiredtiger_open_block_cache_subconfigs, 12}, {"buffer_alignment", "int", NULL, "min=-1,max=1MB", NULL, 0}, {"builtin_extension_config", "string", NULL, NULL, NULL, 0}, {"cache_cursors", "boolean", NULL, NULL, NULL, 0}, @@ -884,11 +901,11 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { {"use_environment", "boolean", NULL, NULL, NULL, 0}, {"use_environment_priv", "boolean", NULL, NULL, NULL, 0}, {"verbose", "list", NULL, - "choices=[\"api\",\"backup\",\"block\",\"checkpoint\"," - "\"checkpoint_cleanup\",\"checkpoint_progress\",\"compact\"," - "\"compact_progress\",\"error_returns\",\"evict\",\"evict_stuck\"" - ",\"evictserver\",\"fileops\",\"handleops\",\"log\"," - "\"history_store\",\"history_store_activity\",\"lsm\"," + "choices=[\"api\",\"backup\",\"block\",\"block_cache\"," + "\"checkpoint\",\"checkpoint_cleanup\",\"checkpoint_progress\"," + "\"compact\",\"compact_progress\",\"error_returns\",\"evict\"," + "\"evict_stuck\",\"evictserver\",\"fileops\",\"handleops\"," + "\"history_store\",\"history_store_activity\",\"log\",\"lsm\"," "\"lsm_manager\",\"metadata\",\"mutex\",\"overflow\",\"read\"," "\"reconcile\",\"recovery\",\"recovery_progress\",\"rts\"," "\"salvage\",\"shared_cache\",\"split\",\"temporary\"," @@ -900,6 +917,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { + {"block_cache", "category", NULL, NULL, confchk_wiredtiger_open_block_cache_subconfigs, 12}, {"buffer_alignment", "int", NULL, "min=-1,max=1MB", NULL, 0}, {"builtin_extension_config", "string", NULL, NULL, NULL, 0}, {"cache_cursors", "boolean", NULL, NULL, NULL, 0}, @@ -966,11 +984,11 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { {"use_environment", "boolean", NULL, NULL, NULL, 0}, {"use_environment_priv", "boolean", NULL, NULL, NULL, 0}, {"verbose", "list", NULL, - "choices=[\"api\",\"backup\",\"block\",\"checkpoint\"," - "\"checkpoint_cleanup\",\"checkpoint_progress\",\"compact\"," - "\"compact_progress\",\"error_returns\",\"evict\",\"evict_stuck\"" - ",\"evictserver\",\"fileops\",\"handleops\",\"log\"," - "\"history_store\",\"history_store_activity\",\"lsm\"," + "choices=[\"api\",\"backup\",\"block\",\"block_cache\"," + "\"checkpoint\",\"checkpoint_cleanup\",\"checkpoint_progress\"," + "\"compact\",\"compact_progress\",\"error_returns\",\"evict\"," + "\"evict_stuck\",\"evictserver\",\"fileops\",\"handleops\"," + "\"history_store\",\"history_store_activity\",\"log\",\"lsm\"," "\"lsm_manager\",\"metadata\",\"mutex\",\"overflow\",\"read\"," "\"reconcile\",\"recovery\",\"recovery_progress\",\"rts\"," "\"salvage\",\"shared_cache\",\"split\",\"temporary\"," @@ -982,6 +1000,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { + {"block_cache", "category", NULL, NULL, confchk_wiredtiger_open_block_cache_subconfigs, 12}, {"buffer_alignment", "int", NULL, "min=-1,max=1MB", NULL, 0}, {"builtin_extension_config", "string", NULL, NULL, NULL, 0}, {"cache_cursors", "boolean", NULL, NULL, NULL, 0}, @@ -1043,11 +1062,11 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { {"transaction_sync", "category", NULL, NULL, confchk_wiredtiger_open_transaction_sync_subconfigs, 2}, {"verbose", "list", NULL, - "choices=[\"api\",\"backup\",\"block\",\"checkpoint\"," - "\"checkpoint_cleanup\",\"checkpoint_progress\",\"compact\"," - "\"compact_progress\",\"error_returns\",\"evict\",\"evict_stuck\"" - ",\"evictserver\",\"fileops\",\"handleops\",\"log\"," - "\"history_store\",\"history_store_activity\",\"lsm\"," + "choices=[\"api\",\"backup\",\"block\",\"block_cache\"," + "\"checkpoint\",\"checkpoint_cleanup\",\"checkpoint_progress\"," + "\"compact\",\"compact_progress\",\"error_returns\",\"evict\"," + "\"evict_stuck\",\"evictserver\",\"fileops\",\"handleops\"," + "\"history_store\",\"history_store_activity\",\"log\",\"lsm\"," "\"lsm_manager\",\"metadata\",\"mutex\",\"overflow\",\"read\"," "\"reconcile\",\"recovery\",\"recovery_progress\",\"rts\"," "\"salvage\",\"shared_cache\",\"split\",\"temporary\"," @@ -1059,6 +1078,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { + {"block_cache", "category", NULL, NULL, confchk_wiredtiger_open_block_cache_subconfigs, 12}, {"buffer_alignment", "int", NULL, "min=-1,max=1MB", NULL, 0}, {"builtin_extension_config", "string", NULL, NULL, NULL, 0}, {"cache_cursors", "boolean", NULL, NULL, NULL, 0}, @@ -1120,11 +1140,11 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { {"transaction_sync", "category", NULL, NULL, confchk_wiredtiger_open_transaction_sync_subconfigs, 2}, {"verbose", "list", NULL, - "choices=[\"api\",\"backup\",\"block\",\"checkpoint\"," - "\"checkpoint_cleanup\",\"checkpoint_progress\",\"compact\"," - "\"compact_progress\",\"error_returns\",\"evict\",\"evict_stuck\"" - ",\"evictserver\",\"fileops\",\"handleops\",\"log\"," - "\"history_store\",\"history_store_activity\",\"lsm\"," + "choices=[\"api\",\"backup\",\"block\",\"block_cache\"," + "\"checkpoint\",\"checkpoint_cleanup\",\"checkpoint_progress\"," + "\"compact\",\"compact_progress\",\"error_returns\",\"evict\"," + "\"evict_stuck\",\"evictserver\",\"fileops\",\"handleops\"," + "\"history_store\",\"history_store_activity\",\"log\",\"lsm\"," "\"lsm_manager\",\"metadata\",\"mutex\",\"overflow\",\"read\"," "\"reconcile\",\"recovery\",\"recovery_progress\",\"rts\"," "\"salvage\",\"shared_cache\",\"split\",\"temporary\"," @@ -1148,10 +1168,16 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "config=,early_load=false,entry=wiredtiger_extension_init," "terminate=wiredtiger_extension_terminate", confchk_WT_CONNECTION_load_extension, 4}, - {"WT_CONNECTION.open_session", "cache_cursors=true,ignore_cache_size=false,isolation=snapshot", - confchk_WT_CONNECTION_open_session, 3}, + {"WT_CONNECTION.open_session", + "cache_cursors=true,cache_max_wait_ms=0,ignore_cache_size=false," + "isolation=snapshot", + confchk_WT_CONNECTION_open_session, 4}, {"WT_CONNECTION.query_timestamp", "get=all_durable", confchk_WT_CONNECTION_query_timestamp, 1}, {"WT_CONNECTION.reconfigure", + "block_cache=(blkcache_eviction_aggression=1800," + "cache_on_checkpoint=true,cache_on_writes=true,enabled=false," + "full_target=95,hashsize=0,max_percent_overhead=10,nvram_path=," + "percent_file_in_dram=50,size=0,system_ram=0,type=)," "cache_max_wait_ms=0,cache_overhead=8,cache_size=100MB," "checkpoint=(log_size=0,wait=0),compatibility=(release=)," "debug_mode=(checkpoint_retention=0,corruption_abort=true," @@ -1174,7 +1200,7 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "tiered_manager=(threads_max=8,threads_min=1,wait=0)," "tiered_storage=(local_retention=300,object_target_size=10M)," "timing_stress_for_test=,verbose=[]", - confchk_WT_CONNECTION_reconfigure, 29}, + confchk_WT_CONNECTION_reconfigure, 30}, {"WT_CONNECTION.rollback_to_stable", "", NULL, 0}, {"WT_CONNECTION.set_file_system", "", NULL, 0}, {"WT_CONNECTION.set_timestamp", "commit_timestamp=,durable_timestamp=,force=false," @@ -1253,8 +1279,10 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", {"WT_SESSION.prepare_transaction", "prepare_timestamp=", confchk_WT_SESSION_prepare_transaction, 1}, {"WT_SESSION.query_timestamp", "get=read", confchk_WT_SESSION_query_timestamp, 1}, - {"WT_SESSION.reconfigure", "cache_cursors=true,ignore_cache_size=false,isolation=snapshot", - confchk_WT_SESSION_reconfigure, 3}, + {"WT_SESSION.reconfigure", + "cache_cursors=true,cache_max_wait_ms=0,ignore_cache_size=false," + "isolation=snapshot", + confchk_WT_SESSION_reconfigure, 4}, {"WT_SESSION.rename", "", NULL, 0}, {"WT_SESSION.reset", "", NULL, 0}, {"WT_SESSION.reset_snapshot", "", NULL, 0}, {"WT_SESSION.rollback_transaction", "operation_timeout_ms=0", @@ -1410,15 +1438,19 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "internal_page_max=4KB,key_format=u,key_gap=10,last=0," "leaf_item_max=0,leaf_key_max=0,leaf_page_max=32KB," "leaf_value_max=0,log=(enabled=true),memory_page_image_max=0," - "memory_page_max=5MB,os_cache_dirty_max=0,os_cache_max=0," - "prefix_compression=false,prefix_compression_min=4,readonly=false" - ",split_deepen_min_child=0,split_deepen_per_child=0,split_pct=90," - "tiered_object=false,tiered_storage=(auth_token=,bucket=," - "bucket_prefix=,cache_directory=,local_retention=300,name=," - "object_target_size=10M),tiers=,value_format=u,verbose=[]," + "memory_page_max=5MB,oldest=1,os_cache_dirty_max=0,os_cache_max=0" + ",prefix_compression=false,prefix_compression_min=4," + "readonly=false,split_deepen_min_child=0,split_deepen_per_child=0" + ",split_pct=90,tiered_object=false,tiered_storage=(auth_token=," + "bucket=,bucket_prefix=,cache_directory=,local_retention=300," + "name=,object_target_size=10M),tiers=,value_format=u,verbose=[]," "version=(major=0,minor=0),write_timestamp_usage=none", - confchk_tiered_meta, 49}, + confchk_tiered_meta, 50}, {"wiredtiger_open", + "block_cache=(blkcache_eviction_aggression=1800," + "cache_on_checkpoint=true,cache_on_writes=true,enabled=false," + "full_target=95,hashsize=0,max_percent_overhead=10,nvram_path=," + "percent_file_in_dram=50,size=0,system_ram=0,type=)," "buffer_alignment=-1,builtin_extension_config=,cache_cursors=true" ",cache_max_wait_ms=0,cache_overhead=8,cache_size=100MB," "checkpoint=(log_size=0,wait=0),checkpoint_sync=true," @@ -1453,8 +1485,12 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "transaction_sync=(enabled=false,method=fsync)," "use_environment=true,use_environment_priv=false,verbose=[]," "verify_metadata=false,write_through=", - confchk_wiredtiger_open, 57}, + confchk_wiredtiger_open, 58}, {"wiredtiger_open_all", + "block_cache=(blkcache_eviction_aggression=1800," + "cache_on_checkpoint=true,cache_on_writes=true,enabled=false," + "full_target=95,hashsize=0,max_percent_overhead=10,nvram_path=," + "percent_file_in_dram=50,size=0,system_ram=0,type=)," "buffer_alignment=-1,builtin_extension_config=,cache_cursors=true" ",cache_max_wait_ms=0,cache_overhead=8,cache_size=100MB," "checkpoint=(log_size=0,wait=0),checkpoint_sync=true," @@ -1489,8 +1525,12 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "transaction_sync=(enabled=false,method=fsync)," "use_environment=true,use_environment_priv=false,verbose=[]," "verify_metadata=false,version=(major=0,minor=0),write_through=", - confchk_wiredtiger_open_all, 58}, + confchk_wiredtiger_open_all, 59}, {"wiredtiger_open_basecfg", + "block_cache=(blkcache_eviction_aggression=1800," + "cache_on_checkpoint=true,cache_on_writes=true,enabled=false," + "full_target=95,hashsize=0,max_percent_overhead=10,nvram_path=," + "percent_file_in_dram=50,size=0,system_ram=0,type=)," "buffer_alignment=-1,builtin_extension_config=,cache_cursors=true" ",cache_max_wait_ms=0,cache_overhead=8,cache_size=100MB," "checkpoint=(log_size=0,wait=0),checkpoint_sync=true," @@ -1524,8 +1564,12 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "object_target_size=10M),timing_stress_for_test=," "transaction_sync=(enabled=false,method=fsync),verbose=[]," "verify_metadata=false,version=(major=0,minor=0),write_through=", - confchk_wiredtiger_open_basecfg, 52}, + confchk_wiredtiger_open_basecfg, 53}, {"wiredtiger_open_usercfg", + "block_cache=(blkcache_eviction_aggression=1800," + "cache_on_checkpoint=true,cache_on_writes=true,enabled=false," + "full_target=95,hashsize=0,max_percent_overhead=10,nvram_path=," + "percent_file_in_dram=50,size=0,system_ram=0,type=)," "buffer_alignment=-1,builtin_extension_config=,cache_cursors=true" ",cache_max_wait_ms=0,cache_overhead=8,cache_size=100MB," "checkpoint=(log_size=0,wait=0),checkpoint_sync=true," @@ -1559,7 +1603,7 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "object_target_size=10M),timing_stress_for_test=," "transaction_sync=(enabled=false,method=fsync),verbose=[]," "verify_metadata=false,write_through=", - confchk_wiredtiger_open_usercfg, 51}, + confchk_wiredtiger_open_usercfg, 52}, {NULL, NULL, NULL, 0}}; int diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index 2e55aedcd63..21b2f0675fa 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -1990,8 +1990,8 @@ int __wt_verbose_config(WT_SESSION_IMPL *session, const char *cfg[]) { static const WT_NAME_FLAG verbtypes[] = {{"api", WT_VERB_API}, {"backup", WT_VERB_BACKUP}, - {"block", WT_VERB_BLOCK}, {"checkpoint", WT_VERB_CHECKPOINT}, - {"checkpoint_cleanup", WT_VERB_CHECKPOINT_CLEANUP}, + {"block", WT_VERB_BLOCK}, {"block_cache", WT_VERB_BLKCACHE}, + {"checkpoint", WT_VERB_CHECKPOINT}, {"checkpoint_cleanup", WT_VERB_CHECKPOINT_CLEANUP}, {"checkpoint_progress", WT_VERB_CHECKPOINT_PROGRESS}, {"compact", WT_VERB_COMPACT}, {"compact_progress", WT_VERB_COMPACT_PROGRESS}, {"error_returns", WT_VERB_ERROR_RETURNS}, {"evict", WT_VERB_EVICT}, {"evict_stuck", WT_VERB_EVICT_STUCK}, @@ -2011,20 +2011,42 @@ __wt_verbose_config(WT_SESSION_IMPL *session, const char *cfg[]) WT_CONNECTION_IMPL *conn; WT_DECL_RET; const WT_NAME_FLAG *ft; - uint64_t flags; conn = S2C(session); WT_RET(__wt_config_gets(session, cfg, "verbose", &cval)); - flags = 0; for (ft = verbtypes; ft->name != NULL; ft++) { - if ((ret = __wt_config_subgets(session, &cval, ft->name, &sval)) == 0 && sval.val != 0) - LF_SET(ft->flag); + ret = __wt_config_subgets(session, &cval, ft->name, &sval); WT_RET_NOTFOUND_OK(ret); + + if (ret == WT_NOTFOUND) + /* + * If the given event isn't specified in configuration string, default it to the + * WT_VERBOSE_WARNING verbosity level. + */ + conn->verbose[ft->flag] = WT_VERBOSE_WARNING; + else if (sval.type == WT_CONFIG_ITEM_BOOL && sval.len == 0) + /* + * If no value is associated with the event (i.e passing verbose=[checkpoint]), default + * the event to WT_VERBOSE_DEBUG. Correspondingly, all legacy uses of '__wt_verbose', + * being messages without an explicit verbosity level, will default to + * 'WT_VERBOSE_DEBUG'. + */ + conn->verbose[ft->flag] = WT_VERBOSE_DEBUG; + else if (sval.type == WT_CONFIG_ITEM_NUM && sval.val >= WT_VERBOSE_INFO && + sval.val <= WT_VERBOSE_DEBUG) + conn->verbose[ft->flag] = (WT_VERBOSE_LEVEL)sval.val; + else + /* + * We only support verbosity values in the form of positive numbers (representing + * verbosity levels e.g. [checkpoint:1,rts:0]) and boolean expressions (e.g. + * [checkpoint,rts]). Return error for all other unsupported verbosity values e.g + * negative numbers and strings. + */ + WT_RET_MSG(session, EINVAL, "Failed to parse verbose option '%s'", ft->name); } - conn->verbose = flags; return (0); } @@ -2691,12 +2713,9 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, const char *c } WT_ERR(__wt_verbose_config(session, cfg)); WT_ERR(__wt_timing_stress_config(session, cfg)); - - /* Set up operation tracking if configured. */ + WT_ERR(__wt_block_cache_setup(session, cfg, false)); WT_ERR(__wt_conn_optrack_setup(session, cfg, false)); - WT_ERR(__conn_session_size(session, cfg, &conn->session_size)); - WT_ERR(__wt_config_gets(session, cfg, "session_scratch_max", &cval)); conn->session_scratch_max = (size_t)cval.val; diff --git a/src/third_party/wiredtiger/src/conn/conn_cache.c b/src/third_party/wiredtiger/src/conn/conn_cache.c index 81fca88fe09..4337042629c 100644 --- a/src/third_party/wiredtiger/src/conn/conn_cache.c +++ b/src/third_party/wiredtiger/src/conn/conn_cache.c @@ -244,6 +244,12 @@ __wt_cache_create(WT_SESSION_IMPL *session, const char *cfg[]) */ if (cache->eviction_target >= cache->eviction_trigger) WT_RET_MSG(session, EINVAL, "eviction target must be lower than the eviction trigger"); + if (cache->eviction_dirty_target >= cache->eviction_dirty_trigger) + WT_RET_MSG( + session, EINVAL, "eviction dirty target must be lower than the eviction dirty trigger"); + if (cache->eviction_updates_target >= cache->eviction_updates_trigger) + WT_RET_MSG(session, EINVAL, + "eviction updates target must be lower than the eviction updates trigger"); WT_RET(__wt_cond_auto_alloc( session, "cache eviction server", 10000, WT_MILLION, &cache->evict_cond)); diff --git a/src/third_party/wiredtiger/src/conn/conn_open.c b/src/third_party/wiredtiger/src/conn/conn_open.c index 343e0046423..5d01264502d 100644 --- a/src/third_party/wiredtiger/src/conn/conn_open.c +++ b/src/third_party/wiredtiger/src/conn/conn_open.c @@ -86,6 +86,8 @@ __wt_connection_close(WT_CONNECTION_IMPL *conn) /* The default session is used to access data handles during close. */ F_CLR(session, WT_SESSION_NO_DATA_HANDLES); + __wt_block_cache_destroy(session); + /* * Shut down server threads. Some of these threads access btree handles and eviction, shut them * down before the eviction server, and shut all servers down before closing open data handles. diff --git a/src/third_party/wiredtiger/src/conn/conn_reconfig.c b/src/third_party/wiredtiger/src/conn/conn_reconfig.c index c8d1330d7ff..7dde0c40824 100644 --- a/src/third_party/wiredtiger/src/conn/conn_reconfig.c +++ b/src/third_party/wiredtiger/src/conn/conn_reconfig.c @@ -416,6 +416,7 @@ __wt_conn_reconfig(WT_SESSION_IMPL *session, const char **cfg) */ WT_WITH_CHECKPOINT_LOCK(session, ret = __wt_conn_compat_config(session, cfg, true)); WT_ERR(ret); + WT_ERR(__wt_block_cache_setup(session, cfg, true)); WT_ERR(__wt_conn_optrack_setup(session, cfg, true)); WT_ERR(__wt_conn_statistics_config(session, cfg)); WT_ERR(__wt_cache_config(session, cfg, true)); diff --git a/src/third_party/wiredtiger/src/conn/conn_tiered.c b/src/third_party/wiredtiger/src/conn/conn_tiered.c index 6f34064ca7e..2d4995dedd1 100644 --- a/src/third_party/wiredtiger/src/conn/conn_tiered.c +++ b/src/third_party/wiredtiger/src/conn/conn_tiered.c @@ -212,7 +212,7 @@ __tier_flush_meta( WT_RET(__wt_scr_alloc(session, 512, &buf)); dhandle = &tiered->iface; - newconfig = NULL; + newconfig = obj_value = NULL; WT_ERR(__wt_meta_track_on(session)); tracking = true; @@ -235,6 +235,7 @@ __tier_flush_meta( err: __wt_free(session, newconfig); + __wt_free(session, obj_value); if (release) WT_TRET(__wt_session_release_dhandle(session)); __wt_scr_free(session, &buf); @@ -251,11 +252,15 @@ int __wt_tier_do_flush(WT_SESSION_IMPL *session, WT_TIERED *tiered, uint32_t id, const char *local_uri, const char *obj_uri) { + WT_CONFIG_ITEM pfx; WT_DECL_RET; WT_FILE_SYSTEM *bucket_fs; WT_STORAGE_SOURCE *storage_source; - const char *local_name, *obj_name; + size_t len; + char *tmp; + const char *cfg[2], *local_name, *obj_name; + tmp = NULL; storage_source = tiered->bstorage->storage_source; bucket_fs = tiered->bstorage->file_system; @@ -263,27 +268,36 @@ __wt_tier_do_flush(WT_SESSION_IMPL *session, WT_TIERED *tiered, uint32_t id, con WT_PREFIX_SKIP_REQUIRED(session, local_name, "file:"); obj_name = obj_uri; WT_PREFIX_SKIP_REQUIRED(session, obj_name, "object:"); + cfg[0] = tiered->obj_config; + cfg[1] = NULL; + WT_RET(__wt_config_gets(session, cfg, "tiered_storage.bucket_prefix", &pfx)); + WT_ASSERT(session, pfx.len != 0); + len = strlen(obj_name) + pfx.len + 1; + WT_RET(__wt_calloc_def(session, len, &tmp)); + WT_ERR(__wt_snprintf(tmp, len, "%.*s%s", (int)pfx.len, pfx.str, obj_name)); /* This call make take a while, and may fail due to network timeout. */ - WT_RET(storage_source->ss_flush( - storage_source, &session->iface, bucket_fs, local_name, obj_name, NULL)); + WT_ERR( + storage_source->ss_flush(storage_source, &session->iface, bucket_fs, local_name, tmp, NULL)); WT_WITH_CHECKPOINT_LOCK(session, WT_WITH_SCHEMA_LOCK(session, ret = __tier_flush_meta(session, tiered, local_uri, obj_uri))); - WT_RET(ret); + WT_ERR(ret); /* * We may need a way to cleanup flushes for those not completed (after a crash), or failed (due * to previous network outage). */ - WT_RET(storage_source->ss_flush_finish( - storage_source, &session->iface, bucket_fs, local_name, obj_name, NULL)); + WT_ERR(storage_source->ss_flush_finish( + storage_source, &session->iface, bucket_fs, local_name, tmp, NULL)); /* * After successful flushing, push a work unit to drop the local object in the future. The * object will be removed locally after the local retention period expires. */ - WT_RET(__wt_tiered_put_drop_local(session, tiered, id)); - return (0); + WT_ERR(__wt_tiered_put_drop_local(session, tiered, id)); +err: + __wt_free(session, tmp); + return (ret); } /* diff --git a/src/third_party/wiredtiger/src/cursor/cur_std.c b/src/third_party/wiredtiger/src/cursor/cur_std.c index 1bf28de3f11..32e9973caaa 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_std.c +++ b/src/third_party/wiredtiger/src/cursor/cur_std.c @@ -1120,29 +1120,26 @@ err: int __wt_cursor_largest_key(WT_CURSOR *cursor) { + WT_CURSOR_BTREE *cbt; WT_DECL_ITEM(key); WT_DECL_RET; WT_SESSION_IMPL *session; - bool ignore_tombstone; - bool visible_all; + bool key_only; - ignore_tombstone = F_ISSET(cursor, WT_CURSTD_IGNORE_TOMBSTONE); - visible_all = F_ISSET(cursor, WT_CURSTD_VISIBLE_ALL); - CURSOR_API_CALL(cursor, session, largest_key, NULL); + cbt = (WT_CURSOR_BTREE *)cursor; + key_only = F_ISSET(cursor, WT_CURSTD_KEY_ONLY); + CURSOR_API_CALL(cursor, session, largest_key, CUR2BT(cbt)); WT_ERR(__wt_scr_alloc(session, 0, &key)); /* Reset the cursor to give up the cursor position. */ WT_ERR(cursor->reset(cursor)); - /* Ignore deletion. */ - F_SET(cursor, WT_CURSTD_IGNORE_TOMBSTONE); - /* Ignore visibility. */ - F_SET(cursor, WT_CURSTD_VISIBLE_ALL); + /* Set the flag to bypass value read. */ + F_SET(cursor, WT_CURSTD_KEY_ONLY); - /* Call cursor prev to get the largest key. */ - WT_ERR(cursor->prev(cursor)); - WT_ERR(ret); + /* Call btree cursor prev to get the largest key. */ + WT_ERR(__wt_btcur_prev(cbt, false)); /* Copy the key as we will reset the cursor after that. */ WT_ERR(__wt_buf_set(session, key, cursor->key.data, cursor->key.size)); @@ -1152,10 +1149,8 @@ __wt_cursor_largest_key(WT_CURSOR *cursor) F_SET(cursor, WT_CURSTD_KEY_EXT); err: - if (!ignore_tombstone) - F_CLR(cursor, WT_CURSTD_IGNORE_TOMBSTONE); - if (!visible_all) - F_CLR(cursor, WT_CURSTD_VISIBLE_ALL); + if (!key_only) + F_CLR(cursor, WT_CURSTD_KEY_ONLY); __wt_scr_free(session, &key); if (ret != 0) WT_TRET(cursor->reset(cursor)); diff --git a/src/third_party/wiredtiger/src/docs/Doxyfile b/src/third_party/wiredtiger/src/docs/Doxyfile index cfef83d0445..5377daebd20 100644 --- a/src/third_party/wiredtiger/src/docs/Doxyfile +++ b/src/third_party/wiredtiger/src/docs/Doxyfile @@ -262,7 +262,7 @@ TAB_SIZE = 8 # a double escape (\\{ and \\}) ALIASES = "arch_page_table{2}=<div class= arch_head><table><tr><th rowspan=2 style=\"width:10%;\">@htmlonly <div><a href=\"arch-index.html\"><img class=\"arch_thumbnail\" src=\"wt_diagram.png\" style=\"background-image: url(wt_diagram.png)\"></a></div>@endhtmlonly</th><th style=\"width:44%\">Data Structures</th><th style=\"width:45%\">Source Location</th></tr><tr><td><code>\1</code></td><td><code>\2</code></td></tr></table></div>" \ - "arch_page_top{2}=@page \1 \2 (Architecture Guide)" \ + "arch_page_top{2}=@page \1 \2" \ "arch_page_caution=<b>Caution: the Architecture Guide is not updated in lockstep with the code base and is not necessarily correct or complete for any specific release.</b>" \ "config{3}= @row{<tt>\1</tt>,\2,\3}" \ "configempty{2}=@param config configuration string, see @ref config_strings. No values currently permitted." \ @@ -1693,7 +1693,7 @@ MATHJAX_CODEFILE = # The default value is: YES. # This tag requires that the tag GENERATE_HTML is set to YES. -SEARCHENGINE = NO +SEARCHENGINE = YES # When the SERVER_BASED_SEARCH tag is enabled the search engine will be # implemented using a web server instead of a web client using JavaScript. There diff --git a/src/third_party/wiredtiger/src/docs/arch-data-file.dox b/src/third_party/wiredtiger/src/docs/arch-data-file.dox index 580b355d838..1448c8dc2b1 100644 --- a/src/third_party/wiredtiger/src/docs/arch-data-file.dox +++ b/src/third_party/wiredtiger/src/docs/arch-data-file.dox @@ -82,9 +82,8 @@ their position in the page. (3) Overflow Pages - overflow pages are needed when keys/values that are too large must be stored separately in the file, apart from where the item -is logically placed. Page sizes and configuration values such as -internal_key_max, leaf_key_max and leaf_value_max are used to determine -overflow items. +is logically placed. Page sizes and configuration values such as leaf_key_max +and leaf_value_max are used to determine overflow items. While the above gives a general overview of how different page types are structured, it is a simplified representation. Due to the large amount of diff --git a/src/third_party/wiredtiger/src/docs/arch-eviction.dox b/src/third_party/wiredtiger/src/docs/arch-eviction.dox index 0e3bf023be6..9e8fedb811c 100644 --- a/src/third_party/wiredtiger/src/docs/arch-eviction.dox +++ b/src/third_party/wiredtiger/src/docs/arch-eviction.dox @@ -64,7 +64,7 @@ a database via ::wiredtiger_open, or changed after open with WT_CONNECTION::reconfigure. The \c eviction_target configuration value is the level at -which WiredTiger attempts to keep the overall cache usage. Eviction worker +which WiredTiger attempts to keep the overall cache usage. Eviction worker threads are active when the cache contains at least this much content, expressed as a percentage of the total cache size. @@ -72,15 +72,18 @@ The \c eviction_trigger configuration value is the level at which application th start to perform the eviction. The \c eviction_dirty_target and \c eviction_dirty_trigger operate in a similar way to -the overall targets but only apply to dirty data in cache. In particular, application +the overall targets but only apply to dirty data in cache. In particular, application threads will be throttled if the percentage of dirty data reaches the \c eviction_dirty_trigger. Any page that has been modified since it was read from disk is considered dirty. -The \c eviction_updates_target perform eviction in worker threads when the cache contain +The \c eviction_updates_target perform eviction in worker threads when the cache contains at least this many bytes of updates The \c eviction_updates_trigger trigger application threads to perform eviction when the cache contains at least this many bytes of updates. +It is worth noting that for all eviction target and trigger sizes, the target size must be lower +than its trigger size counterpart. + */ diff --git a/src/third_party/wiredtiger/src/docs/arch-glossary.dox b/src/third_party/wiredtiger/src/docs/arch-glossary.dox index f12468aaaaf..21b1734780b 100644 --- a/src/third_party/wiredtiger/src/docs/arch-glossary.dox +++ b/src/third_party/wiredtiger/src/docs/arch-glossary.dox @@ -1,4 +1,4 @@ -/*! @page arch-glossary Architecture Guide Glossary of Terms +/*! @page arch-glossary Glossary of Terms WiredTiger has a lot of domain specific nomenclature - this page attempts to decode it. This is intended for those navigating the WiredTiger source @@ -7,7 +7,9 @@ tree - it describes terms internal to the storage engine. The definitions list how the terms are used in WiredTiger, which may not match how they are used in other settings. -@section arch-glossary-transactions Glossary (click headings to sort) +@section arch-glossary-transactions Glossary + +(click headings to sort) <!-- Please keep glossary formatted a single entry to a line, and sorted alphabetically (case insensitive). --> diff --git a/src/third_party/wiredtiger/src/docs/arch-index.dox b/src/third_party/wiredtiger/src/docs/arch-index.dox index 1a8f252fe0a..af4ff8babcc 100644 --- a/src/third_party/wiredtiger/src/docs/arch-index.dox +++ b/src/third_party/wiredtiger/src/docs/arch-index.dox @@ -1,9 +1,15 @@ /*! @page arch-index WiredTiger Architecture Guide -WiredTiger is built as a library that is linked into the application. Applications -use the API interface to direct WiredTiger's operations (see @ref basic_api). -Here is an overview of the software components in WiredTiger and how they are organized. -An arrow indicates the "from" component uses "to" component. +The WiredTiger Architecture Guide provides a comprehensive overview +of WiredTiger internals and code that should be useful for any database +engineer wanting to understand how the storage engine works. The goal +of this guide is to help both MongoDB engineers and external users +quickly understand the workings of WiredTiger. + +The relationships between the software components in WiredTiger are +illustrated in the diagram below. An arrow originating at Component A +and pointing to Component B indicates that Component B is used by +Component A. <div class="arch_diagram"> @plantuml_start{wt_diagram.png } @@ -86,14 +92,12 @@ schema -down-> meta schema -down-> btree cursor -down-> btree btree -down-> row -btree -down-> column meta -up-> cursor ' The hidden arrow helps our boxes to line up in a better way. meta -[hidden]right-> btree cursor -[hidden]right-> txn txn -down-> snapshot row -down-> cache -column -down-> cache cache -down-> history evict -down-> history history -up-> cursor @@ -120,104 +124,190 @@ wt_file -[hidden]right-> log_file @plantuml_end </div> -We go into some detail for some of the internal components. +For those unfamiliar with storage engine architecture and/or seeking an introduction +to WiredTiger, we recommend reading the guide in the order presented. You can find +much of the architecture-specific terminology explained in the @ref arch-glossary. +For an application level view of WiredTiger, head over to the @ref basic_api section +of the documentation. -@subpage arch-backup "Backup" +<div class="arch_toc"> +<h1>Table of Contents</h1> -Hot backup uses a type of cursor to backup the database. +- @subpage arch-toc-fundamentals + - @ref arch-connection + - @ref arch-session + - @ref arch-cursor + - @ref arch-transaction + - @ref arch-snapshot + - @ref arch-rts -@subpage arch-block "Block Manager" +- @subpage arch-toc-data-org + - @ref arch-schema + - @ref arch-metadata -The Block Manager manages the reading and writing of disk blocks. +- @subpage arch-toc-data-src + - @ref arch-dhandle + - @ref arch-btree + - @ref arch-row-column -@subpage arch-btree -A B-Tree is one type of underlying data source in a dhandle and is organized into pages. +- @subpage arch-toc-in-mem + - @ref arch-cache + - @ref arch-eviction -@subpage arch-cache +- @subpage arch-toc-mem-disk + - @ref arch-block + - @ref arch-data-file -Cache is represented by the various shared data structures that -make up in-memory Btrees and subordinate data structures. +- @subpage arch-toc-on-disk + - @ref arch-checkpoint + - @ref arch-hs + - @ref arch-backup -@subpage arch-checkpoint "Checkpoint" +- @subpage arch-toc-recovery + - @ref arch-logging + - @ref arch-log-file -A checkpoint is created by WiredTiger to serve as a point from which it can recover. +- @subpage arch-toc-tools + - @ref arch-python + - @ref command_line + +- @subpage arch-toc-platform + - @ref arch-fs-os -@subpage arch-connection "Connection" +- @subpage arch-glossary -A connection is a handle to a WiredTiger database instance. +</div> -@subpage arch-cursor "Cursor" +*/ -Cursors are used to get and modify data. +/*! @page arch-toc-fundamentals Fundamentals -@subpage arch-data-file "Data File Format" +@subpage arch-connection +- A connection is a handle to a WiredTiger database instance. -The format of the data file is given by structures in \c block.h . +@subpage arch-session +- A session defines the context for most operations performed in WiredTiger. + +@subpage arch-cursor +- Cursors are used to get and modify data. + +@subpage arch-transaction +- Transactions provide a powerful abstraction for multiple threads to +operate on data concurrently. -@subpage arch-dhandle "Data Handles" +@subpage arch-snapshot +- Snapshots are implemented by storing transaction ids committed before +the transaction started. + +@subpage arch-rts +- Rollback the database to a stable state by removing data that is beyond the +stable timestamp. + +*/ + +/*! @page arch-toc-data-org Data Organization + +@subpage arch-schema + +A schema defines the format of the application data in WiredTiger. + +@subpage arch-metadata + +Metadata is stored as <code>uri, config</code> key/value pairs in a designated table. + +*/ + +/*! @page arch-toc-data-src Data Sources + +@subpage arch-btree + +A B-Tree is one type of underlying data source in a dhandle and is organized into pages. + +@subpage arch-dhandle An internal structure called a Data Handle (dhandle) is used to represent and access Btrees and other data sources in WiredTiger. -@subpage arch-eviction "Eviction" +@subpage arch-row-column + +Row stores and column store are B-Trees. Row stores have a variable size key +and data while column stores have as their key a record id. + +*/ + +/*! @page arch-toc-in-mem In Memory Concepts + +@subpage arch-cache + +Cache is represented by the various shared data structures that +make up in-memory Btrees and subordinate data structures. + +@subpage arch-eviction Eviction represents the process or removing old data from the cache, writing it to disk if it is dirty. -@subpage arch-fs-os "File System and Operating System Interface" +*/ -A layer of abstraction is above all operating system calls and -a set of functions can be registered to be called for each file system -operation. +/*! @page arch-toc-mem-disk Moving Data Between Memory and Disk -@subpage arch-hs "History Store" +@subpage arch-block -The History Store tracks old versions of records. +The block manager manages the reading and writing of disk blocks. -@subpage arch-logging "Logging" +@subpage arch-data-file -WiredTiger writes all changes into a write-ahead log when configured. +The format of the data file is given by structures in \c block.h . -@subpage arch-log-file "Log File Format" +*/ -The format of a log file is defined in \c log.h . +/*! @page arch-toc-on-disk On Disk Concepts -@subpage arch-metadata "Metadata" +@subpage arch-checkpoint -Metadata is stored as <code>uri, config</code> K/V pairs in a designated table. +A checkpoint is created by WiredTiger to serve as a point from which it can recover. -@subpage arch-python "Python API" +@subpage arch-hs -WiredTiger has a Python API that is useful for scripting and experimentation. +The History Store tracks old versions of records. -@subpage arch-row-column "Row Store and Column Store" +@subpage arch-backup -Row stores and column store are B-Trees. Row stores have a variable size key and data while column -stores have as their key a record id. +Hot backup uses a type of cursor to backup the database. -@subpage arch-rts "Rollback to Stable" +*/ -Rollback to stable to remove the unstable updates from the database. +/*! @page arch-toc-recovery Recovery -@subpage arch-schema "Schema" +@subpage arch-logging -A schema defines the format of the application data in WiredTiger. +WiredTiger writes all changes into a write-ahead log when configured. -@subpage arch-session "Session" +@subpage arch-log-file -A session defines the context for most operations performed in WiredTiger. +The format of a log file is defined in \c log.h . -@subpage arch-snapshot "Snapshot" +*/ -Snapshots are implemented by storing transaction ids committed before -the transaction started. +/*! @page arch-toc-tools Tools -@subpage arch-transaction "Transactions" +@subpage arch-python -Transactions provide a powerful abstraction for multiple threads to operate on data concurrently. +WiredTiger has a Python API that is useful for scripting and experimentation. -@subpage arch-glossary +@subpage command_line -WiredTiger assigns specific meanings to certain words. Here we decode them. +The \c wt tool is a command-line utility that provides access to various pieces +of the WiredTiger functionality. + +*/ + +/*! @page arch-toc-platform Cross-Platform Support + +@subpage arch-fs-os + +A layer of abstraction is above all operating system calls and +a set of functions can be registered to be called for each file system +operation. */ diff --git a/src/third_party/wiredtiger/src/docs/cursor-ops.dox b/src/third_party/wiredtiger/src/docs/cursor-ops.dox index 9618396a77c..c6d0d30b827 100644 --- a/src/third_party/wiredtiger/src/docs/cursor-ops.dox +++ b/src/third_party/wiredtiger/src/docs/cursor-ops.dox @@ -70,6 +70,14 @@ WT_CURSOR::search_near method: @snippet ex_cursor.c cursor search near +The WT_CURSOR::search_near method can also be used with a prefix key by +setting \c prefix_search=true in the cursor configuration options. This +optimization ensures that search_near only traverses the relevant key +range with the matching prefix. Limiting the key range that search_near +can traverse through is particularly useful when there are a larger number +of records present outside of the key range which are not visible to the +search_near caller. + Cursor positions do not survive transactions: cursors that are open during WT_SESSION::begin_transaction, WT_SESSION::commit_transaction or WT_SESSION::rollback_transaction will lose their position as if diff --git a/src/third_party/wiredtiger/src/docs/images/plantuml_gen_img/wt_diagram.cmapx b/src/third_party/wiredtiger/src/docs/images/plantuml_gen_img/wt_diagram.cmapx index 69ba9be858a..db5d5182f5d 100644 --- a/src/third_party/wiredtiger/src/docs/images/plantuml_gen_img/wt_diagram.cmapx +++ b/src/third_party/wiredtiger/src/docs/images/plantuml_gen_img/wt_diagram.cmapx @@ -1,20 +1,20 @@ <map id="wt_diagram_map" name="wt_diagram_map"> <area shape="rect" id="id1" href="modules.html" title="modules.html" alt="" coords="248,128,283,144"/> -<area shape="rect" id="id2" href="arch-cache.html" title="arch-cache.html" alt="" coords="196,546,240,562"/> -<area shape="rect" id="id3" href="arch-cursor.html" title="arch-cursor.html" alt="" coords="206,224,252,240"/> -<area shape="rect" id="id4" href="arch-eviction.html" title="arch-eviction.html" alt="" coords="295,546,348,562"/> +<area shape="rect" id="id2" href="arch-cache.html" title="arch-cache.html" alt="" coords="204,546,248,562"/> +<area shape="rect" id="id3" href="arch-cursor.html" title="arch-cursor.html" alt="" coords="234,224,280,240"/> +<area shape="rect" id="id4" href="arch-eviction.html" title="arch-eviction.html" alt="" coords="303,546,356,562"/> <area shape="rect" id="id5" href="arch-logging.html" title="arch-logging.html" alt="" coords="388,650,443,666"/> -<area shape="rect" id="id6" href="arch-schema.html" title="arch-schema.html" alt="" coords="95,224,151,240"/> +<area shape="rect" id="id6" href="arch-schema.html" title="arch-schema.html" alt="" coords="123,224,179,240"/> <area shape="rect" id="id7" href="command_line.html" title="command_line.html" alt="" coords="374,23,430,39"/> <area shape="rect" id="id8" href="arch-log-file.html" title="arch-log-file.html" alt="" coords="308,865,339,897"/> <area shape="rect" id="id9" href="arch-metadata.html" title="arch-metadata.html" alt="" coords="25,328,89,344"/> <area shape="rect" id="id10" href="arch-python.html" title="arch-python.html" alt="" coords="84,23,157,39"/> -<area shape="rect" id="id11" href="arch-snapshot.html" title="arch-snapshot.html" alt="" coords="381,441,455,457"/> -<area shape="rect" id="id12" href="arch-transaction.html" title="arch-transaction.html" alt="" coords="339,328,428,344"/> +<area shape="rect" id="id11" href="arch-snapshot.html" title="arch-snapshot.html" alt="" coords="297,441,371,457"/> +<area shape="rect" id="id12" href="arch-transaction.html" title="arch-transaction.html" alt="" coords="298,328,387,344"/> <area shape="rect" id="id13" href="arch-hs.html" title="arch-hs.html" alt="" coords="93,642,140,674"/> -<area shape="rect" id="id14" href="arch-row-column.html" title="arch-row-column.html" alt="" coords="163,433,217,465"/> -<area shape="rect" id="id16" href="arch-block.html" title="arch-block.html" alt="" coords="196,642,256,674"/> -<area shape="rect" id="id17" href="arch-dhandle.html" title="arch-dhandle.html" alt="" coords="181,320,242,352"/> -<area shape="rect" id="id18" href="arch-data-file.html" title="arch-data-file.html" alt="" coords="179,865,245,897"/> -<area shape="rect" id="id19" href="arch-fs-os.html" title="arch-fs-os.html" alt="" coords="175,755,357,771"/> +<area shape="rect" id="id14" href="arch-block.html" title="arch-block.html" alt="" coords="196,642,256,674"/> +<area shape="rect" id="id15" href="arch-dhandle.html" title="arch-dhandle.html" alt="" coords="144,320,205,352"/> +<area shape="rect" id="id16" href="arch-data-file.html" title="arch-data-file.html" alt="" coords="179,865,245,897"/> +<area shape="rect" id="id17" href="arch-row-column.html" title="arch-row-column.html" alt="" coords="117,433,205,465"/> +<area shape="rect" id="id18" href="arch-fs-os.html" title="arch-fs-os.html" alt="" coords="175,755,357,771"/> </map> diff --git a/src/third_party/wiredtiger/src/docs/images/plantuml_gen_img/wt_diagram.png b/src/third_party/wiredtiger/src/docs/images/plantuml_gen_img/wt_diagram.png Binary files differindex 92bf5dc1f2e..899231658f7 100644 --- a/src/third_party/wiredtiger/src/docs/images/plantuml_gen_img/wt_diagram.png +++ b/src/third_party/wiredtiger/src/docs/images/plantuml_gen_img/wt_diagram.png diff --git a/src/third_party/wiredtiger/src/docs/spell.ok b/src/third_party/wiredtiger/src/docs/spell.ok index 6a5b7f8dd5d..f11f16ccc99 100644 --- a/src/third_party/wiredtiger/src/docs/spell.ok +++ b/src/third_party/wiredtiger/src/docs/spell.ok @@ -632,6 +632,7 @@ timeline timelines timestamp timestamps +toc todo toolchain toolchains diff --git a/src/third_party/wiredtiger/src/docs/style/wiredtiger.css b/src/third_party/wiredtiger/src/docs/style/wiredtiger.css index 9757f79b7d8..59f9da89144 100644 --- a/src/third_party/wiredtiger/src/docs/style/wiredtiger.css +++ b/src/third_party/wiredtiger/src/docs/style/wiredtiger.css @@ -1,9 +1,17 @@ /* WiredTiger-specific styling. */ +html { + font-size: small; +} + +/* Set a max page width for readability on large monitors. */ +div.contents { + max-width: 960px; +} + body, table, div, p, dl { - font-family: "Helvetica Neue", Helvetica, Arial, sans-serif; + font-family: "Calibri","Helvetica Neue", Helvetica, Arial, sans-serif; color: #373737; - font-size: 13px; - line-height: 1.3; + line-height: 1.5; } h1 { @@ -14,6 +22,35 @@ h2 { font-size: 110%; } +/* Architecture guide TOC styling. */ +div.arch_toc ul, ol { + line-height: 1.2; +} + +div.arch_toc li { + margin-top: 5px; + margin-bottom: 5px; +} + +div.arch_toc ul li:first-child { + margin-top:0; +} + +div.arch_toc li li { + margin-top: 0px; + margin-bottom: 0px; + line-height: 1.2; +} + +div.arch_toc li li a { + font-weight: normal !important; +} + +div.arch_toc h1 { + font-weight: bold; + font-size: 110%; +} + /* @group Link Styling */ a { @@ -30,6 +67,8 @@ div.arch_diagram { display: block; background: #FFFFFF; background: radial-gradient(#FFDDAA,#FFFFFF); + margin-top: 2em; + margin-bottom: 2em; } /* match just one image */ diff --git a/src/third_party/wiredtiger/src/docs/transactions.dox b/src/third_party/wiredtiger/src/docs/transactions.dox index 3c1233b0400..c594b6b15c9 100644 --- a/src/third_party/wiredtiger/src/docs/transactions.dox +++ b/src/third_party/wiredtiger/src/docs/transactions.dox @@ -246,15 +246,6 @@ For example, for the timestamp values <code>prepare_timestamp=100, timestamp is rounded up to the new prepare timestamp as <code>commit_timestamp=200</code>. -Configuring <code>roundup_timestamps=(prepared=true)</code> also allows setting -the prepared timestamps smaller than or equal to the latest active read timestamps. -Use this feature carefully as it may break repeated read. For example, consider a -transaction with a read timestamp set to 30 and a key that has a value with -timestamp 20. Before a prepared transaction (in another thread), reading the key -returns the value. Now a prepared transaction elsewhere in the system sets a prepared -timestamp at 30. The reader, trying to do a repeat read of the key, gets return a -prepared conflict error instead of the value. - Configuring <code>roundup_timestamps=(read=true)</code> causes the read timestamp to be rounded up to the oldest timestamp, if the read timestamp is greater than the oldest timestamp no change will be made. diff --git a/src/third_party/wiredtiger/src/docs/tune-page-size-and-comp.dox b/src/third_party/wiredtiger/src/docs/tune-page-size-and-comp.dox index b39a9de2594..bfff6df40fc 100644 --- a/src/third_party/wiredtiger/src/docs/tune-page-size-and-comp.dox +++ b/src/third_party/wiredtiger/src/docs/tune-page-size-and-comp.dox @@ -202,18 +202,13 @@ example of such a configuration string is as follows: @subsection key_val_max internal/leaf key/value max - Overflow items -\n Overflow items are keys and values too large to easily store on a page. Overflow -items are stored separately in the file from the page where the item logically -appears, and so reading or writing an overflow item is more expensive than an -on-page item, normally requiring additional I/O. Additionally, overflow values -are not cached in memory. This means overflow items won't affect the caching -behavior of the application. It also means that each time an overflow value is -read, it is re-read from disk. - - internal_key_max -\n The largest key stored in an internal page, in bytes. If set, keys larger than -the specified size are stored as overflow items. - - The default and the maximum allowed value are both one-tenth the size of a -newly split internal page. +\n Overflow items are keys and values too large to easily store on a leaf +page. Overflow items are stored separately in the file from the page where +the item logically appears, and so reading an overflow item is more expensive +than an on-page item, normally requiring additional I/O. Additionally, +overflow values are not cached in memory. This means overflow items won't +affect the caching behavior of the application. It also means that each +time an overflow value is read, it is re-read from disk. - leaf_key_max \n The largest key stored in a leaf page, in bytes. If set, keys larger than the specified size are stored as overflow items. @@ -244,24 +239,18 @@ aren't likely to be needed. - Applications with large keys and values, doing table scans, might increase the page size to avoid creating overflow items, as the overflow items must be read into memory in all cases, anyway. - - internal_key_max, leaf_key_max and leaf_value_max configuration values -allow applications to change the size at which a key or value will be treated -as an overflow item. + - leaf_key_max and leaf_value_max configuration values allow applications +to change the size at which a key or value will be treated as an overflow item. - Most applications should not need to tune the maximum key and value sizes. - - The value of internal_key_max is relative to the maximum internal page -size. Because the number of keys on an internal page determines the depth of the -tree, the internal_key_max value can only be adjusted within a certain range, -and the configured value will be automatically adjusted by WiredTiger, if -necessary, to ensure a reasonable number of keys fit on an internal page. - The values of leaf_key_max and leaf_value_max are not relative to the maximum leaf page size. If either is larger than the maximum page size, the page size will be ignored when the larger keys and values are being written, and a larger page will be created as necessary. - Configuration: -\n Specified as internal_key_max, leaf_key_max and leaf_value_max configuration -options to WT_SESSION::create. An example of configuration string for a large -leaf overflow value: +\n Specified as leaf_key_max and leaf_value_max configuration options to +WT_SESSION::create. An example of configuration string for a large leaf +overflow value: <pre> "key_format=S,value_format=S,leaf_page_max=16KB,leaf_value_max=256KB" diff --git a/src/third_party/wiredtiger/src/evict/evict_lru.c b/src/third_party/wiredtiger/src/evict/evict_lru.c index 0bfaf1c9f33..88344fc5c7b 100644 --- a/src/third_party/wiredtiger/src/evict/evict_lru.c +++ b/src/third_party/wiredtiger/src/evict/evict_lru.c @@ -2340,8 +2340,8 @@ __wt_cache_eviction_worker(WT_SESSION_IMPL *session, bool busy, bool readonly, d WT_TRACK_OP_DECL; WT_TXN_GLOBAL *txn_global; WT_TXN_SHARED *txn_shared; + uint64_t cache_max_wait_us, initial_progress, max_progress; uint64_t elapsed, time_start, time_stop; - uint64_t initial_progress, max_progress; bool app_thread; WT_TRACK_OP_INIT(session); @@ -2353,6 +2353,11 @@ __wt_cache_eviction_worker(WT_SESSION_IMPL *session, bool busy, bool readonly, d txn_global = &conn->txn_global; txn_shared = WT_SESSION_TXN_SHARED(session); + if (session->cache_max_wait_us != 0) + cache_max_wait_us = session->cache_max_wait_us; + else + cache_max_wait_us = cache->cache_max_wait_us; + /* * Before we enter the eviction generation, make sure this session has a cached history store * cursor, otherwise we can deadlock with a session wanting exclusive access to a handle: that @@ -2383,6 +2388,8 @@ __wt_cache_eviction_worker(WT_SESSION_IMPL *session, bool busy, bool readonly, d if (!F_ISSET(conn, WT_CONN_RECOVERING) && __wt_cache_stuck(session)) { ret = __wt_txn_is_blocking(session); if (ret == WT_ROLLBACK) { + __wt_verbose_debug( + session, WT_VERB_TRANSACTION, "Rollback reason: %s", "Cache full"); --cache->evict_aggressive_score; WT_STAT_CONN_INCR(session, txn_fail_cache); if (app_thread) @@ -2436,10 +2443,9 @@ __wt_cache_eviction_worker(WT_SESSION_IMPL *session, bool busy, bool readonly, d goto err; } /* Stop if we've exceeded the time out. */ - if (app_thread && cache->cache_max_wait_us != 0) { + if (app_thread && cache_max_wait_us != 0) { time_stop = __wt_clock(session); - if (session->cache_wait_us + WT_CLOCKDIFF_US(time_stop, time_start) > - cache->cache_max_wait_us) + if (session->cache_wait_us + WT_CLOCKDIFF_US(time_stop, time_start) > cache_max_wait_us) goto err; } } @@ -2451,7 +2457,7 @@ err: WT_STAT_CONN_INCRV(session, application_cache_time, elapsed); WT_STAT_SESSION_INCRV(session, cache_time, elapsed); session->cache_wait_us += elapsed; - if (cache->cache_max_wait_us != 0 && session->cache_wait_us > cache->cache_max_wait_us) { + if (cache_max_wait_us != 0 && session->cache_wait_us > cache_max_wait_us) { WT_TRET(WT_CACHE_FULL); WT_STAT_CONN_INCR(session, cache_timed_out_ops); } diff --git a/src/third_party/wiredtiger/src/history/hs_rec.c b/src/third_party/wiredtiger/src/history/hs_rec.c index bf9103fcc57..ec66f5b35a4 100644 --- a/src/third_party/wiredtiger/src/history/hs_rec.c +++ b/src/third_party/wiredtiger/src/history/hs_rec.c @@ -28,7 +28,7 @@ __hs_verbose_cache_stats(WT_SESSION_IMPL *session, WT_BTREE *btree) btree_id = btree->id; - if (!WT_VERBOSE_ISSET(session, WT_VERB_HS | WT_VERB_HS_ACTIVITY)) + if (!WT_VERBOSE_ISSET(session, WT_VERB_HS) && !WT_VERBOSE_ISSET(session, WT_VERB_HS_ACTIVITY)) return; conn = S2C(session); @@ -46,7 +46,9 @@ __hs_verbose_cache_stats(WT_SESSION_IMPL *session, WT_BTREE *btree) WT_IGNORE_RET_BOOL(__wt_eviction_clean_needed(session, &pct_full)); WT_IGNORE_RET_BOOL(__wt_eviction_dirty_needed(session, &pct_dirty)); - __wt_verbose(session, WT_VERB_HS | WT_VERB_HS_ACTIVITY, + __wt_verbose_multi(session, + WT_DECL_VERBOSE_MULTI_CATEGORY( + ((WT_VERBOSE_CATEGORY[]){WT_VERB_HS, WT_VERB_HS_ACTIVITY})), "Page reconciliation triggered history store write: file ID %" PRIu32 ". Current history store file size: %" PRId64 ", cache dirty: %2.3f%% , cache use: %2.3f%%", @@ -66,9 +68,7 @@ static int __hs_insert_record(WT_SESSION_IMPL *session, WT_CURSOR *cursor, WT_BTREE *btree, const WT_ITEM *key, const uint8_t type, const WT_ITEM *hs_value, WT_TIME_WINDOW *tw, bool error_on_ooo_ts) { -#ifdef HAVE_DIAGNOSTIC WT_CURSOR_BTREE *hs_cbt; -#endif WT_DECL_ITEM(hs_key); #ifdef HAVE_DIAGNOSTIC WT_DECL_ITEM(existing_val); @@ -85,7 +85,7 @@ __hs_insert_record(WT_SESSION_IMPL *session, WT_CURSOR *cursor, WT_BTREE *btree, uint64_t counter, hs_counter; uint32_t hs_btree_id; - counter = 0; + counter = hs_counter = 0; /* * We might be entering this code from application thread's context. We should make sure that we @@ -112,9 +112,10 @@ __hs_insert_record(WT_SESSION_IMPL *session, WT_CURSOR *cursor, WT_BTREE *btree, #ifdef HAVE_DIAGNOSTIC /* Allocate buffer for the existing history store value for the same key. */ WT_ERR(__wt_scr_alloc(session, 0, &existing_val)); - hs_cbt = __wt_curhs_get_cbt(cursor); #endif + hs_cbt = __wt_curhs_get_cbt(cursor); + /* Sanity check that the btree is not a history store btree. */ WT_ASSERT(session, !WT_IS_HS(btree)); @@ -183,9 +184,16 @@ __hs_insert_record(WT_SESSION_IMPL *session, WT_CURSOR *cursor, WT_BTREE *btree, * would have received WT_NOT_FOUND. In that case we need to search again with a higher * timestamp. */ - if (ret == 0) - WT_ERR_NOTFOUND_OK(cursor->next(cursor), true); - else { + if (ret == 0) { + /* + * Check if the current history store update's stop timestamp is out of order with respect + * to the update to be inserted before before moving onto the next record. + */ + if (hs_cbt->upd_value->tw.stop_ts <= tw->start_ts) + WT_ERR_NOTFOUND_OK(cursor->next(cursor), true); + else + counter = hs_counter + 1; + } else { cursor->set_key(cursor, 3, btree->id, key, tw->start_ts + 1); WT_ERR_NOTFOUND_OK(__wt_curhs_search_near_after(session, cursor), true); } @@ -867,8 +875,13 @@ __hs_delete_reinsert_from_pos(WT_SESSION_IMPL *session, WT_CURSOR *hs_cursor, ui WT_ERR(__wt_compare(session, NULL, &hs_key, key, &cmp)); WT_ASSERT(session, cmp == 0); #endif - /* We find a key that is larger or equal to the specified timestamp*/ - if (hs_ts >= ts) + /* + * We have found a key with a timestamp larger than or equal to the specified timestamp. + * Always use the start timestamp retrieved from the key instead of the start timestamp from + * the cell. The cell's start timestamp can be cleared during reconciliation if it is + * globally visible. + */ + if (hs_ts >= ts || twp->stop_ts >= ts) break; } if (ret == WT_NOTFOUND) @@ -894,30 +907,44 @@ __hs_delete_reinsert_from_pos(WT_SESSION_IMPL *session, WT_CURSOR *hs_cursor, ui * see them after they've been moved due to their transaction id. * * For example, if we're inserting an update at timestamp 3 with value ddd: - * btree key ts counter value - * 2 foo 5 0 aaa - * 2 foo 6 0 bbb - * 2 foo 7 0 ccc + * btree key ts counter value stop_ts + * 2 foo 5 0 aaa 6 + * 2 foo 6 0 bbb 7 + * 2 foo 7 0 ccc 8 * * We want to end up with this: - * btree key ts counter value - * 2 foo 3 0 aaa - * 2 foo 3 1 bbb - * 2 foo 3 2 ccc - * 2 foo 3 3 ddd + * btree key ts counter value stop_ts + * 2 foo 3 0 aaa 3 + * 2 foo 3 1 bbb 3 + * 2 foo 3 2 ccc 3 + * 2 foo 3 3 ddd 3 * * Another example, if we're inserting an update at timestamp 0 with value ddd: - * btree key ts counter value - * 2 foo 5 0 aaa - * 2 foo 6 0 bbb - * 2 foo 7 0 ccc + * btree key ts counter value stop_ts + * 2 foo 5 0 aaa 6 + * 2 foo 6 0 bbb 7 + * 2 foo 7 0 ccc 8 + * + * We want to end up with this: + * btree key ts counter value stop_ts + * 2 foo 0 0 aaa 0 + * 2 foo 0 1 bbb 0 + * 2 foo 0 2 ccc 0 + * 2 foo 0 3 ddd 0 + * + * Another example, if we're inserting an update at timestamp 3 with value ddd + * that is an out of order with a stop timestamp of 6: + * btree key ts counter value stop_ts + * 2 foo 1 0 aaa 6 + * 2 foo 6 0 bbb 7 + * 2 foo 7 0 ccc 8 * * We want to end up with this: - * btree key ts counter value - * 2 foo 0 0 aaa - * 2 foo 0 1 bbb - * 2 foo 0 2 ccc - * 2 foo 0 3 ddd + * btree key ts counter value stop_ts + * 2 foo 1 1 aaa 3 + * 2 foo 3 2 bbb 3 + * 2 foo 3 3 ccc 3 + * 2 foo 3 4 ddd 3 */ for (; ret == 0; ret = hs_cursor->next(hs_cursor)) { /* We shouldn't have crossed the btree and user key search space. */ @@ -933,8 +960,14 @@ __hs_delete_reinsert_from_pos(WT_SESSION_IMPL *session, WT_CURSOR *hs_cursor, ui * Our strategy to rectify this is to remove all records for the same key with a timestamp * higher or equal than the specified timestamp and reinsert them at the smaller timestamp, * which is the timestamp of the update we are about to insert to the history store. + * + * It is possible that the cursor next call can find an update that was reinserted when it + * had an out of order tombstone with respect to the new update. Continue the search by + * ignoring them. */ - WT_ASSERT(session, hs_ts >= ts); + __wt_hs_upd_time_window(hs_cursor, &twp); + if (hs_ts < ts && twp->stop_ts < ts) + continue; if (reinsert) { /* @@ -963,7 +996,16 @@ __hs_delete_reinsert_from_pos(WT_SESSION_IMPL *session, WT_CURSOR *hs_cursor, ui __wt_timestamp_to_string(hs_cbt->upd_value->tw.durable_stop_ts, ts_string[3]), __wt_timestamp_to_string(ts, ts_string[4])); - hs_insert_tw.start_ts = hs_insert_tw.durable_start_ts = ts - 1; + /* + * Use the original start time window's timestamps if it isn't out of order with respect + * to the new update. + */ + if (hs_cbt->upd_value->tw.start_ts >= ts) + hs_insert_tw.start_ts = hs_insert_tw.durable_start_ts = ts - 1; + else { + hs_insert_tw.start_ts = hs_cbt->upd_value->tw.start_ts; + hs_insert_tw.durable_start_ts = hs_cbt->upd_value->tw.durable_start_ts; + } hs_insert_tw.start_txn = hs_cbt->upd_value->tw.start_txn; /* diff --git a/src/third_party/wiredtiger/src/include/block.h b/src/third_party/wiredtiger/src/include/block.h index 7b4fa732027..a2f6753884f 100644 --- a/src/third_party/wiredtiger/src/include/block.h +++ b/src/third_party/wiredtiger/src/include/block.h @@ -181,8 +181,10 @@ struct __wt_bm { int (*checkpoint_unload)(WT_BM *, WT_SESSION_IMPL *); int (*close)(WT_BM *, WT_SESSION_IMPL *); int (*compact_end)(WT_BM *, WT_SESSION_IMPL *); + int (*compact_page_rewrite)(WT_BM *, WT_SESSION_IMPL *, uint8_t *, size_t *, bool *); int (*compact_page_skip)(WT_BM *, WT_SESSION_IMPL *, const uint8_t *, size_t, bool *); int (*compact_skip)(WT_BM *, WT_SESSION_IMPL *, bool *); + void (*compact_progress)(WT_BM *, WT_SESSION_IMPL *, u_int *); int (*compact_start)(WT_BM *, WT_SESSION_IMPL *); int (*corrupt)(WT_BM *, WT_SESSION_IMPL *, const uint8_t *, size_t); int (*free)(WT_BM *, WT_SESSION_IMPL *, const uint8_t *, size_t); @@ -276,10 +278,10 @@ struct __wt_block { WT_CKPT *final_ckpt; /* Final live checkpoint write */ /* Compaction support */ - int compact_pct_tenths; /* Percent to compact */ - uint64_t compact_pages_reviewed; /* Pages reviewed */ - uint64_t compact_pages_skipped; /* Pages skipped */ - uint64_t compact_pages_written; /* Pages rewritten */ + int compact_pct_tenths; /* Percent to compact */ + uint64_t compact_pages_rewritten; /* Pages rewritten */ + uint64_t compact_pages_reviewed; /* Pages reviewed */ + uint64_t compact_pages_skipped; /* Pages skipped */ /* Salvage support */ wt_off_t slvg_off; /* Salvage file offset */ diff --git a/src/third_party/wiredtiger/src/include/block_cache.h b/src/third_party/wiredtiger/src/include/block_cache.h new file mode 100644 index 00000000000..0686a3eaf66 --- /dev/null +++ b/src/third_party/wiredtiger/src/include/block_cache.h @@ -0,0 +1,155 @@ +/*- + * Copyright (c) 2014-2020 MongoDB, Inc. + * Copyright (c) 2008-2014 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +/* + * WiredTiger's block cache. It is used to cache blocks identical to those that live on disk in a + * faster storage medium, such as NVRAM. + */ + +#ifdef HAVE_LIBMEMKIND +#include <memkind.h> +#endif + +/* + * Determines now often we compute the total size of the files open in the block manager. + */ +#define BLKCACHE_FILESIZE_EST_FREQ 5000 + +#define BLKCACHE_HASHSIZE_DEFAULT 32768 +#define BLKCACHE_HASHSIZE_MIN 512 +#define BLKCACHE_HASHSIZE_MAX WT_GIGABYTE + +#define WT_BLKCACHE_FULL -2 +#define WT_BLKCACHE_BYPASS -3 + +#define BLKCACHE_MINREF_INCREMENT 20 +#define BLKCACHE_EVICT_OTHER 0 +#define BLKCACHE_NOT_EVICTION_CANDIDATE 1 + +/* + * WT_BLKCACHE_ID -- + * File ID, checksum, offset and size uniquely identify a block. + */ +WT_PACKED_STRUCT_BEGIN(__wt_blkcache_id) + uint32_t fid; + uint32_t checksum; + uint32_t size; + wt_off_t offset; +WT_PACKED_STRUCT_END +#define WT_BLKCACHE_ID_SIZE (sizeof(wt_off_t) + 3 * sizeof(uint32_t)) + +/* + * WT_BLKCACHE_ITEM -- + * Block cache item. It links with other items in the same hash bucket. + */ +struct __wt_blkcache_item { + struct __wt_blkcache_id id; + TAILQ_ENTRY(__wt_blkcache_item) hashq; + void *data; + uint32_t num_references; + + /* + * This counter is incremented every time a block is referenced and decremented every time the + * eviction thread sweeps through the cache. This counter will be low for blocks that have not + * been reused or for blocks that were reused in the past but lost their appeal. In this sense, + * this counter is a metric combining frequency and recency, and hence its name. + */ + int32_t freq_rec_counter; +}; + +/* + * WT_BLKCACHE_BUCKET_METADATA -- + * The metadata indicating the number of bytes in cache is accumulated per + * bucket, because we do locking per bucket. Then the eviction thread accumulates + * per-bucket data into a global metadata value that is stored in the block + * cache structure. + */ + +struct __wt_blkcache_bucket_metadata { + WT_CACHE_LINE_PAD_BEGIN + volatile uint64_t bucket_num_data_blocks; /* Number of blocks in the bucket */ + volatile uint64_t bucket_bytes_used; /* Bytes in the bucket */ + WT_CACHE_LINE_PAD_END +}; + +/* + * WT_BLKCACHE -- + * Block cache metadata includes the hashtable of cached items, number of cached data blocks + * and the total amount of space they occupy. + */ +struct __wt_blkcache { + /* Locked: Block manager cache. Locks are per-bucket. */ + TAILQ_HEAD(__wt_blkcache_hash, __wt_blkcache_item) * hash; + WT_SPINLOCK *hash_locks; + WT_BLKCACHE_BUCKET_METADATA *bucket_metadata; + + wt_thread_t evict_thread_tid; + volatile bool blkcache_exiting; /* If destroying the cache */ + int32_t evict_aggressive; /* Seconds an unused block stays in the cache */ + + bool cache_on_checkpoint; /* Don't cache blocks written by checkpoints */ + bool cache_on_writes; /* Cache blocks on writes */ + +#ifdef HAVE_LIBMEMKIND + struct memkind *pmem_kind; /* NVRAM connection */ +#endif + char *nvram_device_path; /* The absolute path of the file system on NVRAM device */ + + uint64_t full_target; /* Number of bytes in the block cache that triggers eviction */ + u_int overhead_pct; /* Overhead percentage that suppresses population and eviction */ + + size_t estimated_file_size; /* Estimated size of all files used by the workload. */ + int refs_since_filesize_estimated; /* Counter for recalculating the aggregate file size */ + + /* + * This fraction tells us the good enough ratio of file data cached in the DRAM resident OS + * buffer cache, which makes the use of this block cache unnecessary. Suppose we set that + * fraction to 50%. Then if half of our file data fits into system DRAM, we consider this block + * cache unhelpful. + * + * E.g., if the fraction is set to 50%, our aggregate file size is 500GB, and we have 300GB of + * RAM, then we will not use this block cache, because we know that half of our files (250GB) + * must be cached by the OS in DRAM. + */ + u_int percent_file_in_os_cache; + + u_int hash_size; /* Number of block cache hash buckets */ + u_int type; /* Type of block cache (NVRAM or DRAM) */ + volatile uint64_t bytes_used; /* Bytes in the block cache */ + volatile uint64_t num_data_blocks; /* Number of blocks in the block cache */ + uint64_t max_bytes; /* Block cache size */ + uint64_t system_ram; /* Configured size of system RAM */ + + uint32_t min_num_references; /* The per-block number of references triggering eviction. */ + + /* + * Various metrics helping us measure the overhead and decide if to bypass the cache. We access + * some of them without synchronization despite races. These serve as heuristics, and we don't + * need precise values for them to be useful. If, because of races, we lose updates of these + * values, assuming that we lose them at the same rate for all variables, the ratio should + * remain roughly accurate. We care about the ratio. + */ + uint64_t lookups; + uint64_t inserts; + uint64_t removals; + + /* Histograms keeping track of number of references to each block */ +#define BLKCACHE_HIST_BUCKETS 11 +#define BLKCACHE_HIST_BOUNDARY 10 + uint32_t cache_references[BLKCACHE_HIST_BUCKETS]; + uint32_t cache_references_removed_blocks[BLKCACHE_HIST_BUCKETS]; + uint32_t cache_references_evicted_blocks[BLKCACHE_HIST_BUCKETS]; +}; + +#define BLKCACHE_UNCONFIGURED 0 +#define BLKCACHE_DRAM 1 +#define BLKCACHE_NVRAM 2 + +#define BLKCACHE_RM_EXIT 1 +#define BLKCACHE_RM_FREE 2 +#define BLKCACHE_RM_EVICTION 3 diff --git a/src/third_party/wiredtiger/src/include/btmem.h b/src/third_party/wiredtiger/src/include/btmem.h index 6382a1d5f59..8b4ee968dc4 100644 --- a/src/third_party/wiredtiger/src/include/btmem.h +++ b/src/third_party/wiredtiger/src/include/btmem.h @@ -660,16 +660,17 @@ struct __wt_page { uint8_t type; /* Page type */ /* AUTOMATIC FLAG VALUE GENERATION START 0 */ -#define WT_PAGE_BUILD_KEYS 0x01u /* Keys have been built in memory */ -#define WT_PAGE_DISK_ALLOC 0x02u /* Disk image in allocated memory */ -#define WT_PAGE_DISK_MAPPED 0x04u /* Disk image in mapped memory */ -#define WT_PAGE_EVICT_LRU 0x08u /* Page is on the LRU queue */ -#define WT_PAGE_EVICT_NO_PROGRESS 0x10u /* Eviction doesn't count as progress */ -#define WT_PAGE_OVERFLOW_KEYS 0x20u /* Page has overflow keys */ -#define WT_PAGE_SPLIT_INSERT 0x40u /* A leaf page was split for append */ -#define WT_PAGE_UPDATE_IGNORE 0x80u /* Ignore updates on page discard */ - /* AUTOMATIC FLAG VALUE GENERATION STOP 8 */ - uint8_t flags_atomic; /* Atomic flags, use F_*_ATOMIC */ +#define WT_PAGE_BUILD_KEYS 0x001u /* Keys have been built in memory */ +#define WT_PAGE_COMPACTION_WRITE 0x002u /* Writing the page for compaction */ +#define WT_PAGE_DISK_ALLOC 0x004u /* Disk image in allocated memory */ +#define WT_PAGE_DISK_MAPPED 0x008u /* Disk image in mapped memory */ +#define WT_PAGE_EVICT_LRU 0x010u /* Page is on the LRU queue */ +#define WT_PAGE_EVICT_NO_PROGRESS 0x020u /* Eviction doesn't count as progress */ +#define WT_PAGE_INTL_OVERFLOW_KEYS 0x040u /* Internal page has overflow keys (historic only) */ +#define WT_PAGE_SPLIT_INSERT 0x080u /* A leaf page was split for append */ +#define WT_PAGE_UPDATE_IGNORE 0x100u /* Ignore updates on page discard */ + /* AUTOMATIC FLAG VALUE GENERATION STOP 9 */ + uint8_t flags_atomic; /* Atomic flags, use F_*_ATOMIC */ uint8_t unused[2]; /* Unused padding */ diff --git a/src/third_party/wiredtiger/src/include/btree.h b/src/third_party/wiredtiger/src/include/btree.h index 72fe0ec5594..a20cbfc9046 100644 --- a/src/third_party/wiredtiger/src/include/btree.h +++ b/src/third_party/wiredtiger/src/include/btree.h @@ -119,7 +119,6 @@ struct __wt_btree { uint32_t allocsize; /* Allocation size */ uint32_t maxintlpage; /* Internal page max size */ - uint32_t maxintlkey; /* Internal page max key size */ uint32_t maxleafpage; /* Leaf page max size */ uint32_t maxleafkey; /* Leaf page max key size */ uint32_t maxleafvalue; /* Leaf page max value size */ diff --git a/src/third_party/wiredtiger/src/include/btree_inline.h b/src/third_party/wiredtiger/src/include/btree_inline.h index 7b9a2b74a5f..a079a133bd1 100644 --- a/src/third_party/wiredtiger/src/include/btree_inline.h +++ b/src/third_party/wiredtiger/src/include/btree_inline.h @@ -1675,13 +1675,14 @@ __wt_page_can_evict(WT_SESSION_IMPL *session, WT_REF *ref, bool *inmem_splitp) /* * We can't split or evict multiblock row-store pages where the parent's key for the page is an - * overflow item, because the split into the parent frees the backing blocks for any - * no-longer-used overflow keys, which will corrupt the checkpoint's block management. + * overflow item, because the split into the parent frees the backing blocks for no-longer-used + * overflow keys, which will corrupt the checkpoint's block management. (This is only for + * historical tables, reconciliation no longer writes overflow cookies on internal pages, no + * matter the size of the key.) */ - if (!__wt_btree_can_evict_dirty(session) && F_ISSET_ATOMIC(ref->home, WT_PAGE_OVERFLOW_KEYS)) { - WT_STAT_CONN_INCR(session, cache_eviction_fail_parent_has_overflow_items); + if (!__wt_btree_can_evict_dirty(session) && + F_ISSET_ATOMIC(ref->home, WT_PAGE_INTL_OVERFLOW_KEYS)) return (false); - } /* * Check for in-memory splits before other eviction tests. If the page should split in-memory, diff --git a/src/third_party/wiredtiger/src/include/compact.h b/src/third_party/wiredtiger/src/include/compact.h index 637f4a60e51..bf1a31b2167 100644 --- a/src/third_party/wiredtiger/src/include/compact.h +++ b/src/third_party/wiredtiger/src/include/compact.h @@ -7,10 +7,9 @@ */ struct __wt_compact_state { - uint32_t lsm_count; /* Number of LSM trees seen */ - uint32_t file_count; /* Number of files seen */ - uint64_t max_time; /* Configured timeout */ - uint64_t prog_msg_count; /* Progress message count */ + uint32_t lsm_count; /* Number of LSM trees seen */ + uint32_t file_count; /* Number of files seen */ + uint64_t max_time; /* Configured timeout */ struct timespec begin; /* Starting time */ }; diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h index 5b8f7e9f8e4..89283c3cc19 100644 --- a/src/third_party/wiredtiger/src/include/connection.h +++ b/src/third_party/wiredtiger/src/include/connection.h @@ -295,6 +295,8 @@ struct __wt_connection_impl { TAILQ_HEAD(__wt_blockhash, __wt_block) * blockhash; TAILQ_HEAD(__wt_block_qh, __wt_block) blockqh; + WT_BLKCACHE blkcache; /* Block cache */ + /* Locked: handles in each bucket */ uint64_t *dh_bucket_count; uint64_t dhandle_count; /* Locked: handles in the queue */ @@ -550,47 +552,8 @@ struct __wt_connection_impl { /* AUTOMATIC FLAG VALUE GENERATION STOP 64 */ uint64_t debug_flags; -/* AUTOMATIC FLAG VALUE GENERATION START 0 */ -#define WT_VERB_API 0x0000000001u -#define WT_VERB_BACKUP 0x0000000002u -#define WT_VERB_BLOCK 0x0000000004u -#define WT_VERB_CHECKPOINT 0x0000000008u -#define WT_VERB_CHECKPOINT_CLEANUP 0x0000000010u -#define WT_VERB_CHECKPOINT_PROGRESS 0x0000000020u -#define WT_VERB_COMPACT 0x0000000040u -#define WT_VERB_COMPACT_PROGRESS 0x0000000080u -#define WT_VERB_ERROR_RETURNS 0x0000000100u -#define WT_VERB_EVICT 0x0000000200u -#define WT_VERB_EVICTSERVER 0x0000000400u -#define WT_VERB_EVICT_STUCK 0x0000000800u -#define WT_VERB_FILEOPS 0x0000001000u -#define WT_VERB_HANDLEOPS 0x0000002000u -#define WT_VERB_HS 0x0000004000u -#define WT_VERB_HS_ACTIVITY 0x0000008000u -#define WT_VERB_LOG 0x0000010000u -#define WT_VERB_LSM 0x0000020000u -#define WT_VERB_LSM_MANAGER 0x0000040000u -#define WT_VERB_METADATA 0x0000080000u -#define WT_VERB_MUTEX 0x0000100000u -#define WT_VERB_OVERFLOW 0x0000200000u -#define WT_VERB_READ 0x0000400000u -#define WT_VERB_RECONCILE 0x0000800000u -#define WT_VERB_RECOVERY 0x0001000000u -#define WT_VERB_RECOVERY_PROGRESS 0x0002000000u -#define WT_VERB_RTS 0x0004000000u -#define WT_VERB_SALVAGE 0x0008000000u -#define WT_VERB_SHARED_CACHE 0x0010000000u -#define WT_VERB_SPLIT 0x0020000000u -#define WT_VERB_TEMPORARY 0x0040000000u -#define WT_VERB_THREAD_GROUP 0x0080000000u -#define WT_VERB_TIERED 0x0100000000u -#define WT_VERB_TIMESTAMP 0x0200000000u -#define WT_VERB_TRANSACTION 0x0400000000u -#define WT_VERB_VERIFY 0x0800000000u -#define WT_VERB_VERSION 0x1000000000u -#define WT_VERB_WRITE 0x2000000000u - /* AUTOMATIC FLAG VALUE GENERATION STOP 64 */ - uint64_t verbose; + /* Verbose settings for our various categories. */ + WT_VERBOSE_LEVEL verbose[WT_VERB_NUM_CATEGORIES]; /* * Variable with flags for which subsystems the diagnostic stress timing delays have been requested. diff --git a/src/third_party/wiredtiger/src/include/error.h b/src/third_party/wiredtiger/src/include/error.h index ef3019405da..7851b449ba2 100644 --- a/src/third_party/wiredtiger/src/include/error.h +++ b/src/third_party/wiredtiger/src/include/error.h @@ -168,20 +168,3 @@ WT_RET_PANIC(session, v, __VA_ARGS__); \ } while (0) #endif - -/* Verbose messages. */ -#define WT_VERBOSE_ISSET(session, flag) (FLD_ISSET(S2C(session)->verbose, flag)) - -/* - * __wt_verbose -- - * Display a verbose message. Not an inlined function because you can't inline functions taking - * variadic arguments and we don't want to make a function call in production systems just to - * find out a verbose flag isn't set. The macro must take a format string and at least one - * additional argument, there's no portable way to remove the comma before an empty __VA_ARGS__ - * value. - */ -#define __wt_verbose(session, flag, fmt, ...) \ - do { \ - if (WT_VERBOSE_ISSET(session, flag)) \ - __wt_verbose_worker(session, "[" #flag "] " fmt, __VA_ARGS__); \ - } while (0) diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index 0f4f3d84a67..3270c75d107 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -90,6 +90,11 @@ extern int __wt_backup_open(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_bad_object_type(WT_SESSION_IMPL *session, const char *uri) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_blkcache_get_or_check(WT_SESSION_IMPL *session, wt_off_t offset, size_t size, + uint32_t checksum, void *data_ptr) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_blkcache_put(WT_SESSION_IMPL *session, wt_off_t offset, size_t size, + uint32_t checksum, void *data, bool checkpoint_io, bool write) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_addr_invalid(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *addr, size_t addr_size, bool live) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_addr_pack(WT_BLOCK *block, uint8_t **pp, uint32_t objectid, wt_off_t offset, @@ -101,6 +106,8 @@ extern int __wt_block_addr_unpack(WT_SESSION_IMPL *session, WT_BLOCK *block, con WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_alloc(WT_SESSION_IMPL *session, WT_BLOCK *block, wt_off_t *offp, wt_off_t size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_cache_setup(WT_SESSION_IMPL *session, const char *cfg[], bool reconfig) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_checkpoint(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, WT_CKPT *ckptbase, bool data_checksum) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_checkpoint_final(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, @@ -129,6 +136,8 @@ extern int __wt_block_close(WT_SESSION_IMPL *session, WT_BLOCK *block) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_compact_end(WT_SESSION_IMPL *session, WT_BLOCK *block) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_compact_page_rewrite(WT_SESSION_IMPL *session, WT_BLOCK *block, uint8_t *addr, + size_t *addr_sizep, bool *skipp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_compact_page_skip( WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *addr, size_t addr_size, bool *skipp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -275,8 +284,6 @@ extern int __wt_btcur_next_random(WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_btcur_prev(WT_CURSOR_BTREE *cbt, bool truncating) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_btcur_prev_prefix(WT_CURSOR_BTREE *cbt, WT_ITEM *prefix, bool truncating) - WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_btcur_range_truncate(WT_CURSOR_BTREE *start, WT_CURSOR_BTREE *stop) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_btcur_remove(WT_CURSOR_BTREE *cbt, bool positioned) @@ -397,8 +404,6 @@ extern int __wt_collator_config(WT_SESSION_IMPL *session, const char *uri, WT_CO WT_CONFIG_ITEM *metadata, WT_COLLATOR **collatorp, int *ownp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_compact(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_compact_page_skip(WT_SESSION_IMPL *session, WT_REF *ref, void *context, bool *skipp) - WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_compressor_config(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *cval, WT_COMPRESSOR **compressorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_cond_auto_alloc(WT_SESSION_IMPL *session, const char *name, uint64_t min, @@ -1236,10 +1241,10 @@ extern int __wt_rec_row_int(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE * WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_rec_row_leaf(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_REF *pageref, WT_SALVAGE_COOKIE *salvage) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_rec_split(WT_SESSION_IMPL *session, WT_RECONCILE *r, size_t next_len, bool forced) +extern int __wt_rec_split(WT_SESSION_IMPL *session, WT_RECONCILE *r, size_t next_len) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_rec_split_crossing_bnd(WT_SESSION_IMPL *session, WT_RECONCILE *r, size_t next_len) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_rec_split_crossing_bnd(WT_SESSION_IMPL *session, WT_RECONCILE *r, size_t next_len, - bool forced) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_rec_split_finish(WT_SESSION_IMPL *session, WT_RECONCILE *r) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_rec_split_init(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page, @@ -1685,7 +1690,14 @@ extern void *__wt_ext_scr_alloc(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session extern void __wt_abort(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern void __wt_backup_destroy(WT_SESSION_IMPL *session); +extern void __wt_blkcache_remove( + WT_SESSION_IMPL *session, wt_off_t offset, size_t size, uint32_t checksum); +extern void __wt_block_cache_destroy(WT_SESSION_IMPL *session); extern void __wt_block_ckpt_destroy(WT_SESSION_IMPL *session, WT_BLOCK_CKPT *ci); +extern void __wt_block_compact_get_progress_stats(WT_SESSION_IMPL *session, WT_BM *bm, + uint64_t *pages_reviewedp, uint64_t *pages_skippedp, uint64_t *pages_rewrittenp); +extern void __wt_block_compact_progress( + WT_SESSION_IMPL *session, WT_BLOCK *block, u_int *msg_countp); extern void __wt_block_configure_first_fit(WT_BLOCK *block, bool on); extern void __wt_block_ext_free(WT_SESSION_IMPL *session, WT_EXT *ext); extern void __wt_block_extlist_free(WT_SESSION_IMPL *session, WT_EXTLIST *el); @@ -2315,6 +2327,8 @@ static inline void __wt_rec_cell_build_addr(WT_SESSION_IMPL *session, WT_RECONCI static inline void __wt_rec_image_copy(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_REC_KV *kv); static inline void __wt_rec_incr( WT_SESSION_IMPL *session, WT_RECONCILE *r, uint32_t v, size_t size); +static inline void __wt_rec_time_window_clear_obsolete( + WT_SESSION_IMPL *session, WT_TIME_WINDOW *tw, WT_RECONCILE *r); static inline void __wt_ref_key(WT_PAGE *page, WT_REF *ref, void *keyp, size_t *sizep); static inline void __wt_ref_key_clear(WT_REF *ref); static inline void __wt_ref_key_onpage_set(WT_PAGE *page, WT_REF *ref, WT_CELL_UNPACK_ADDR *unpack); @@ -2334,8 +2348,6 @@ static inline void __wt_spin_lock(WT_SESSION_IMPL *session, WT_SPINLOCK *t); static inline void __wt_spin_lock_track(WT_SESSION_IMPL *session, WT_SPINLOCK *t); static inline void __wt_spin_unlock(WT_SESSION_IMPL *session, WT_SPINLOCK *t); static inline void __wt_struct_size_adjust(WT_SESSION_IMPL *session, size_t *sizep); -static inline void __wt_time_window_clear_obsolete( - WT_SESSION_IMPL *session, WT_TIME_WINDOW *tw, uint64_t oldest_id, wt_timestamp_t oldest_ts); static inline void __wt_timing_stress(WT_SESSION_IMPL *session, u_int flag); static inline void __wt_tree_modify_set(WT_SESSION_IMPL *session); static inline void __wt_txn_cursor_op(WT_SESSION_IMPL *session); diff --git a/src/third_party/wiredtiger/src/include/hardware.h b/src/third_party/wiredtiger/src/include/hardware.h index 762613f0373..a313fbb816b 100644 --- a/src/third_party/wiredtiger/src/include/hardware.h +++ b/src/third_party/wiredtiger/src/include/hardware.h @@ -33,6 +33,8 @@ #define F_SET_ATOMIC(p, mask) \ do { \ uint8_t __orig; \ + if (F_ISSET_ATOMIC(p, mask)) \ + break; \ do { \ __orig = (p)->flags_atomic; \ } while (!__wt_atomic_cas8(&(p)->flags_atomic, __orig, __orig | (uint8_t)(mask))); \ @@ -41,6 +43,8 @@ #define F_CLR_ATOMIC(p, mask) \ do { \ uint8_t __orig; \ + if (!F_ISSET_ATOMIC(p, mask)) \ + break; \ do { \ __orig = (p)->flags_atomic; \ } while (!__wt_atomic_cas8(&(p)->flags_atomic, __orig, __orig & ~(uint8_t)(mask))); \ diff --git a/src/third_party/wiredtiger/src/include/reconcile_inline.h b/src/third_party/wiredtiger/src/include/reconcile_inline.h index 8bf71e66ab9..d9c47822e0f 100644 --- a/src/third_party/wiredtiger/src/include/reconcile_inline.h +++ b/src/third_party/wiredtiger/src/include/reconcile_inline.h @@ -20,6 +20,22 @@ #define WT_REC_SPLIT_MIN_ITEMS_USE_MEM 10 /* + * WT_REC_TW_START_VISIBLE_ALL + * Check if the provided time window's start is globally visible as per the saved state on the + * reconciliation structure. + * + * An update is considered to be globally visible when its transaction id is less than the + * pinned id, and when its start timestamp is less than or equal to the pinned timestamp. + * Due to a difference in transaction id based visibility and timestamp visibility the timestamp + * comparison is inclusive whereas the transaction id comparison isn't. + */ +#define WT_REC_TW_START_VISIBLE_ALL(r, tw) \ + (WT_TXNID_LT((tw)->start_txn, (r)->rec_start_oldest_id) && \ + ((tw)->durable_start_ts == WT_TS_NONE || \ + ((r)->rec_start_pinned_ts != WT_TS_NONE && \ + (tw)->durable_start_ts <= (r)->rec_start_pinned_ts))) + +/* * __rec_cell_addr_stats -- * Track statistics for time values associated with an address. */ @@ -425,3 +441,30 @@ __wt_rec_dict_replace( } return (0); } + +/* + * __wt_rec_time_window_clear_obsolete -- + * Where possible modify time window values to avoid writing obsolete values to the cell later. + */ +static inline void +__wt_rec_time_window_clear_obsolete(WT_SESSION_IMPL *session, WT_TIME_WINDOW *tw, WT_RECONCILE *r) +{ + /* + * In memory database don't need to avoid writing values to the cell. If we remove this check we + * create an extra update on the end of the chain later in reconciliation as we'll re-append the + * disk image value to the update chain. + */ + if (!tw->prepare && !F_ISSET(S2C(session), WT_CONN_IN_MEMORY)) { + /* + * Check if the start of the time window is globally visible, and if so remove unnecessary + * values. + */ + if (WT_REC_TW_START_VISIBLE_ALL(r, tw)) { + /* The durable timestamp should never be less than the start timestamp. */ + WT_ASSERT(session, tw->start_ts <= tw->durable_start_ts); + + tw->start_ts = tw->durable_start_ts = WT_TS_NONE; + tw->start_txn = WT_TXN_NONE; + } + } +} diff --git a/src/third_party/wiredtiger/src/include/session.h b/src/third_party/wiredtiger/src/include/session.h index 5206910f139..62d3158e09a 100644 --- a/src/third_party/wiredtiger/src/include/session.h +++ b/src/third_party/wiredtiger/src/include/session.h @@ -165,7 +165,8 @@ struct __wt_session_impl { void *salvage_track; /* Sessions have an associated statistics bucket based on its ID. */ - u_int stat_bucket; /* Statistics bucket offset */ + u_int stat_bucket; /* Statistics bucket offset */ + uint64_t cache_max_wait_us; /* Maximum time an operation waits for space in cache */ #ifdef HAVE_DIAGNOSTIC uint8_t dump_raw; /* Configure debugging page dump */ diff --git a/src/third_party/wiredtiger/src/include/stat.h b/src/third_party/wiredtiger/src/include/stat.h index 309d0b50805..69357367d71 100644 --- a/src/third_party/wiredtiger/src/include/stat.h +++ b/src/third_party/wiredtiger/src/include/stat.h @@ -303,6 +303,25 @@ __wt_stats_clear(void *stats_arg, int slot) WT_STAT_CONN_INCR(session, stat##_gt10000); \ } +#define WT_STAT_COMPR_RATIO_HIST_INCR_FUNC(ratio) \ + static inline void __wt_stat_compr_ratio_hist_incr(WT_SESSION_IMPL *session, uint64_t ratio) \ + { \ + if (ratio < 2) \ + WT_STAT_DATA_INCR(session, compress_hist_ratio_2); \ + else if (ratio < 4) \ + WT_STAT_DATA_INCR(session, compress_hist_ratio_4); \ + else if (ratio < 8) \ + WT_STAT_DATA_INCR(session, compress_hist_ratio_8); \ + else if (ratio < 16) \ + WT_STAT_DATA_INCR(session, compress_hist_ratio_16); \ + else if (ratio < 32) \ + WT_STAT_DATA_INCR(session, compress_hist_ratio_32); \ + else if (ratio < 64) \ + WT_STAT_DATA_INCR(session, compress_hist_ratio_64); \ + else \ + WT_STAT_DATA_INCR(session, compress_hist_ratio_max); \ + } + /* * DO NOT EDIT: automatically built by dist/stat.py. */ @@ -323,6 +342,27 @@ struct __wt_connection_stats { int64_t lsm_work_units_done; int64_t lsm_work_units_created; int64_t lsm_work_queue_max; + int64_t block_cache_blocks_update; + int64_t block_cache_bytes_update; + int64_t block_cache_blocks_evicted; + int64_t block_cache_bypass_filesize; + int64_t block_cache_data_refs; + int64_t block_cache_not_evicted_overhead; + int64_t block_cache_bypass_writealloc; + int64_t block_cache_bypass_overhead_put; + int64_t block_cache_bypass_get; + int64_t block_cache_bypass_put; + int64_t block_cache_eviction_passes; + int64_t block_cache_hits; + int64_t block_cache_misses; + int64_t block_cache_bypass_chkpt; + int64_t block_cache_blocks_removed; + int64_t block_cache_blocks; + int64_t block_cache_blocks_insert_read; + int64_t block_cache_blocks_insert_write; + int64_t block_cache_bytes; + int64_t block_cache_bytes_insert_read; + int64_t block_cache_bytes_insert_write; int64_t block_preload; int64_t block_read; int64_t block_write; @@ -458,7 +498,6 @@ struct __wt_connection_stats { int64_t cache_eviction_pages_seen; int64_t cache_eviction_pages_already_queued; int64_t cache_eviction_fail; - int64_t cache_eviction_fail_parent_has_overflow_items; int64_t cache_eviction_fail_active_children_on_an_internal_page; int64_t cache_eviction_fail_in_reconciliation; int64_t cache_eviction_fail_checkpoint_out_of_order_ts; @@ -656,7 +695,6 @@ struct __wt_connection_stats { int64_t rec_time_window_bytes_ts; int64_t rec_time_window_bytes_txn; int64_t rec_page_delete_fast; - int64_t rec_overflow_key_internal; int64_t rec_overflow_key_leaf; int64_t rec_maximum_seconds; int64_t rec_pages; @@ -842,7 +880,7 @@ struct __wt_dsrc_stats { int64_t btree_checkpoint_generation; int64_t btree_clean_checkpoint_timer; int64_t btree_compact_pages_reviewed; - int64_t btree_compact_pages_write_selected; + int64_t btree_compact_pages_rewritten; int64_t btree_compact_pages_skipped; int64_t btree_compact_skipped; int64_t btree_column_fix; @@ -851,7 +889,6 @@ struct __wt_dsrc_stats { int64_t btree_column_deleted; int64_t btree_column_variable; int64_t btree_fixed_len; - int64_t btree_maxintlkey; int64_t btree_maxintlpage; int64_t btree_maxleafkey; int64_t btree_maxleafpage; @@ -859,7 +896,6 @@ struct __wt_dsrc_stats { int64_t btree_maximum_depth; int64_t btree_entries; int64_t btree_overflow; - int64_t btree_compact_pages_rewritten; int64_t btree_row_empty_values; int64_t btree_row_internal; int64_t btree_row_leaf; @@ -952,6 +988,13 @@ struct __wt_dsrc_stats { int64_t compress_precomp_leaf_max_page_size; int64_t compress_read; int64_t compress_write; + int64_t compress_hist_ratio_max; + int64_t compress_hist_ratio_16; + int64_t compress_hist_ratio_2; + int64_t compress_hist_ratio_32; + int64_t compress_hist_ratio_4; + int64_t compress_hist_ratio_64; + int64_t compress_hist_ratio_8; int64_t compress_write_fail; int64_t compress_write_too_small; int64_t cursor_next_skip_total; @@ -994,7 +1037,6 @@ struct __wt_dsrc_stats { int64_t rec_page_delete_fast; int64_t rec_suffix_compression; int64_t rec_multiblock_internal; - int64_t rec_overflow_key_internal; int64_t rec_prefix_compression; int64_t rec_multiblock_leaf; int64_t rec_overflow_key_leaf; diff --git a/src/third_party/wiredtiger/src/include/tiered.h b/src/third_party/wiredtiger/src/include/tiered.h index 9758042daad..1fa146aba5c 100644 --- a/src/third_party/wiredtiger/src/include/tiered.h +++ b/src/third_party/wiredtiger/src/include/tiered.h @@ -127,6 +127,7 @@ struct __wt_tiered { uint32_t current_id; /* Current object id number */ uint32_t next_id; /* Next object number */ + uint32_t oldest_id; /* Oldest object id number */ /* AUTOMATIC FLAG VALUE GENERATION START 0 */ #define WT_TIERED_FLAG_UNUSED 0x1u diff --git a/src/third_party/wiredtiger/src/include/timestamp_inline.h b/src/third_party/wiredtiger/src/include/timestamp_inline.h index 43f8c59cdf4..fd27a1f8a90 100644 --- a/src/third_party/wiredtiger/src/include/timestamp_inline.h +++ b/src/third_party/wiredtiger/src/include/timestamp_inline.h @@ -157,37 +157,3 @@ if ((source)->prepare != 0) \ (dest)->prepare = 1; \ } while (0) - -/* - * __wt_time_window_clear_obsolete -- - * Where possible modify time window values to avoid writing obsolete values to the cell later. - */ -static inline void -__wt_time_window_clear_obsolete( - WT_SESSION_IMPL *session, WT_TIME_WINDOW *tw, uint64_t oldest_id, wt_timestamp_t oldest_ts) -{ - /* - * In memory database don't need to avoid writing values to the cell. If we remove this check we - * create an extra update on the end of the chain later in reconciliation as we'll re-append the - * disk image value to the update chain. - */ - if (!tw->prepare && !F_ISSET(S2C(session), WT_CONN_IN_MEMORY)) { - if (tw->stop_txn == WT_TXN_MAX && tw->start_txn < oldest_id) - tw->start_txn = WT_TXN_NONE; - /* Avoid retrieving the pinned timestamp unless we need it. */ - if (tw->stop_ts == WT_TS_MAX) { - /* - * The durable stop timestamp should be it's default value whenever the stop timestamp - * is. - */ - WT_ASSERT(session, tw->durable_stop_ts == WT_TS_NONE); - /* - * The durable start timestamp is always greater than or equal to the start timestamp, - * as such we must check it against the pinned timestamp and not the start timestamp. - */ - WT_ASSERT(session, tw->start_ts <= tw->durable_start_ts); - if (tw->durable_start_ts < oldest_ts) - tw->start_ts = tw->durable_start_ts = WT_TS_NONE; - } - } -} diff --git a/src/third_party/wiredtiger/src/include/txn_inline.h b/src/third_party/wiredtiger/src/include/txn_inline.h index a6b13708057..1201ff5326d 100644 --- a/src/third_party/wiredtiger/src/include/txn_inline.h +++ b/src/third_party/wiredtiger/src/include/txn_inline.h @@ -901,10 +901,7 @@ __wt_txn_read_upd_list_internal(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, continue; } - if (F_ISSET(&cbt->iface, WT_CURSTD_VISIBLE_ALL)) - upd_visible = WT_VISIBLE_TRUE; - else - upd_visible = __wt_txn_upd_visible_type(session, upd); + upd_visible = __wt_txn_upd_visible_type(session, upd); if (upd_visible == WT_VISIBLE_TRUE) break; @@ -1044,11 +1041,9 @@ retry: /* * We return the onpage value in the following cases: * 1. The record is from the history store. - * 2. It has the WT_CURSTD_VISIBLE_ALL flag set. - * 3. It is visible to the reader. + * 2. It is visible to the reader. */ - if (WT_IS_HS(session->dhandle) || F_ISSET(&cbt->iface, WT_CURSTD_VISIBLE_ALL) || - __wt_txn_tw_start_visible(session, &tw)) { + if (WT_IS_HS(session->dhandle) || __wt_txn_tw_start_visible(session, &tw)) { if (cbt->upd_value->skip_buf) { cbt->upd_value->buf.data = NULL; cbt->upd_value->buf.size = 0; @@ -1317,13 +1312,16 @@ static inline int __wt_txn_modify_check( WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_UPDATE *upd, wt_timestamp_t *prev_tsp) { + WT_DECL_ITEM(buf); WT_DECL_RET; WT_TIME_WINDOW tw; WT_TXN *txn; WT_TXN_GLOBAL *txn_global; + uint32_t snap_count; + char ts_string[WT_TS_INT_STRING_SIZE]; bool ignore_prepare_set, rollback, tw_found; - rollback = false; + rollback = tw_found = false; txn = session->txn; txn_global = &S2C(session)->txn_global; @@ -1342,6 +1340,9 @@ __wt_txn_modify_check( F_CLR(txn, WT_TXN_IGNORE_PREPARE); for (; upd != NULL && !__wt_txn_upd_visible(session, upd); upd = upd->next) { if (upd->txnid != WT_TXN_ABORTED) { + __wt_verbose_debug(session, WT_VERB_TRANSACTION, + "Conflict with update with txn id %" PRIu64 " at timestamp: %s", upd->txnid, + __wt_timestamp_to_string(upd->start_ts, ts_string)); rollback = true; break; } @@ -1357,14 +1358,39 @@ __wt_txn_modify_check( if (!rollback && upd == NULL) { __wt_read_cell_time_window(cbt, &tw, &tw_found); if (tw_found) { - if (WT_TIME_WINDOW_HAS_STOP(&tw)) + if (WT_TIME_WINDOW_HAS_STOP(&tw)) { rollback = !__wt_txn_tw_stop_visible(session, &tw); - else + if (rollback) + __wt_verbose_debug(session, WT_VERB_TRANSACTION, + "Conflict with update %" PRIu64 " at stop timestamp: %s", tw.stop_txn, + __wt_timestamp_to_string(tw.stop_ts, ts_string)); + } else { rollback = !__wt_txn_tw_start_visible(session, &tw); + if (rollback) + __wt_verbose_debug(session, WT_VERB_TRANSACTION, + "Conflict with update %" PRIu64 " at start timestamp: %s", tw.start_txn, + __wt_timestamp_to_string(tw.start_ts, ts_string)); + } } } if (rollback) { + /* Dump information about the txn snapshot. */ + if (WT_VERBOSE_LEVEL_ISSET(session, WT_VERB_TRANSACTION, WT_VERBOSE_DEBUG)) { + WT_ERR(__wt_scr_alloc(session, 1024, &buf)); + WT_ERR(__wt_buf_fmt(session, buf, + "snapshot_min=%" PRIu64 ", snapshot_max=%" PRIu64 ", snapshot_count=%" PRIu32, + txn->snap_min, txn->snap_max, txn->snapshot_count)); + if (txn->snapshot_count > 0) { + WT_ERR(__wt_buf_catfmt(session, buf, ", snapshots=[")); + for (snap_count = 0; snap_count < txn->snapshot_count - 1; ++snap_count) + WT_ERR( + __wt_buf_catfmt(session, buf, "%" PRIu64 ",", txn->snapshot[snap_count])); + WT_ERR(__wt_buf_catfmt(session, buf, "%" PRIu64 "]", txn->snapshot[snap_count])); + } + __wt_verbose_debug(session, WT_VERB_TRANSACTION, "%s", (const char *)buf->data); + } + WT_STAT_CONN_DATA_INCR(session, txn_update_conflict); ret = __wt_txn_rollback_required(session, "conflict between concurrent operations"); } @@ -1373,17 +1399,24 @@ __wt_txn_modify_check( * Don't access the update from an uncommitted transaction as it can produce wrong timestamp * results. */ - if (!rollback && prev_tsp != NULL && upd != NULL) { - /* - * The durable timestamp must be greater than or equal to the commit timestamp unless it is - * an in-progress prepared update. - */ - WT_ASSERT( - session, upd->durable_ts >= upd->start_ts || upd->prepare_state == WT_PREPARE_INPROGRESS); - *prev_tsp = upd->durable_ts; + if (!rollback && prev_tsp != NULL) { + if (upd != NULL) { + /* + * The durable timestamp must be greater than or equal to the commit timestamp unless it + * is an in-progress prepared update. + */ + WT_ASSERT(session, + upd->durable_ts >= upd->start_ts || upd->prepare_state == WT_PREPARE_INPROGRESS); + *prev_tsp = upd->durable_ts; + } else if (tw_found) + *prev_tsp = WT_TIME_WINDOW_HAS_STOP(&tw) ? tw.durable_stop_ts : tw.durable_start_ts; } + if (ignore_prepare_set) F_SET(txn, WT_TXN_IGNORE_PREPARE); + +err: + __wt_scr_free(session, &buf); return (ret); } diff --git a/src/third_party/wiredtiger/src/include/verbose.h b/src/third_party/wiredtiger/src/include/verbose.h new file mode 100644 index 00000000000..0e2bb954eea --- /dev/null +++ b/src/third_party/wiredtiger/src/include/verbose.h @@ -0,0 +1,175 @@ +/*- + * Copyright (c) 2014-present MongoDB, Inc. + * Copyright (c) 2008-2014 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +/* Permitted verbose event categories that can be used when defining a verbose message. */ +typedef enum { + WT_VERB_API = 0, + WT_VERB_BACKUP, + WT_VERB_BLKCACHE, + WT_VERB_BLOCK, + WT_VERB_CHECKPOINT, + WT_VERB_CHECKPOINT_CLEANUP, + WT_VERB_CHECKPOINT_PROGRESS, + WT_VERB_COMPACT, + WT_VERB_COMPACT_PROGRESS, + WT_VERB_ERROR_RETURNS, + WT_VERB_EVICT, + WT_VERB_EVICTSERVER, + WT_VERB_EVICT_STUCK, + WT_VERB_FILEOPS, + WT_VERB_HANDLEOPS, + WT_VERB_HS, + WT_VERB_HS_ACTIVITY, + WT_VERB_LOG, + WT_VERB_LSM, + WT_VERB_LSM_MANAGER, + WT_VERB_METADATA, + WT_VERB_MUTEX, + WT_VERB_OVERFLOW, + WT_VERB_READ, + WT_VERB_RECONCILE, + WT_VERB_RECOVERY, + WT_VERB_RECOVERY_PROGRESS, + WT_VERB_RTS, + WT_VERB_SALVAGE, + WT_VERB_SHARED_CACHE, + WT_VERB_SPLIT, + WT_VERB_TEMPORARY, + WT_VERB_THREAD_GROUP, + WT_VERB_TIERED, + WT_VERB_TIMESTAMP, + WT_VERB_TRANSACTION, + WT_VERB_VERIFY, + WT_VERB_VERSION, + WT_VERB_WRITE, + /* This entry needs to be the last in order to track the number of category items. */ + WT_VERB_NUM_CATEGORIES, +} WT_VERBOSE_CATEGORY; + +/* + * Permitted verbosity levels; to be used when defining verbose messages. The levels define a range + * of severity categories, with WT_VERBOSE_ERROR being the lowest, most critical level (used by + * messages on critical error paths) and WT_VERBOSE_DEBUG being the highest verbosity/informational + * level (mostly adopted for debugging). + */ +typedef enum { + WT_VERBOSE_ERROR = -2, + WT_VERBOSE_WARNING, + WT_VERBOSE_INFO, + WT_VERBOSE_DEBUG +} WT_VERBOSE_LEVEL; + +/* + * Default verbosity level. WT_VERBOSE_DEBUG being the default level assigned to verbose messages + * prior to the introduction of verbosity levels. + */ +#define WT_VERBOSE_DEFAULT WT_VERBOSE_DEBUG + +/* + * WT_VERBOSE_MULTI_CATEGORY -- + * Simple structure to represent a set of verbose categories. + */ +struct __wt_verbose_multi_category { + WT_VERBOSE_CATEGORY *categories; + uint32_t cnt; +}; + +/* Generate a set of verbose categories. */ +#define WT_DECL_VERBOSE_MULTI_CATEGORY(items) \ + ((WT_VERBOSE_MULTI_CATEGORY){.categories = (items), .cnt = WT_ELEMENTS(items)}) + +/* Check if a given verbosity level satisfies the verbosity level of a category. */ +#define WT_VERBOSE_LEVEL_ISSET(session, category, level) (level <= S2C(session)->verbose[category]) + +/* + * Given this verbosity check is without an explicit verbosity level, the macro checks whether the + * given category satisfies the default verbosity level. + */ +#define WT_VERBOSE_ISSET(session, category) \ + WT_VERBOSE_LEVEL_ISSET(session, category, WT_VERBOSE_DEFAULT) + +/* + * __wt_verbose_level -- + * Display a verbose message considering a category and a verbosity level. + */ +#define __wt_verbose_level(session, category, level, fmt, ...) \ + do { \ + if (WT_VERBOSE_LEVEL_ISSET(session, category, level)) \ + __wt_verbose_worker(session, "[" #category "] " fmt, __VA_ARGS__); \ + } while (0) + +/* + * __wt_verbose_error -- + * Wrapper to __wt_verbose_level defaulting the verbosity level to WT_VERBOSE_ERROR. + */ +#define __wt_verbose_error(session, category, fmt, ...) \ + __wt_verbose_level(session, category, WT_VERBOSE_ERROR, fmt, __VA_ARGS__) + +/* + * __wt_verbose_warning -- + * Wrapper to __wt_verbose_level defaulting the verbosity level to WT_VERBOSE_WARNING. + */ +#define __wt_verbose_warning(session, category, fmt, ...) \ + __wt_verbose_level(session, category, WT_VERBOSE_WARNING, fmt, __VA_ARGS__) + +/* + * __wt_verbose_info -- + * Wrapper to __wt_verbose_level defaulting the verbosity level to WT_VERBOSE_INFO. + */ +#define __wt_verbose_info(session, category, fmt, ...) \ + __wt_verbose_level(session, category, WT_VERBOSE_INFO, fmt, __VA_ARGS__) + +/* + * __wt_verbose_debug -- + * Wrapper to __wt_verbose_level using the default verbosity level. + */ +#define __wt_verbose_debug(session, category, fmt, ...) \ + __wt_verbose_level(session, category, WT_VERBOSE_DEBUG, fmt, __VA_ARGS__) + +/* + * __wt_verbose -- + * Display a verbose message using the default verbosity level. Not an inlined function because + * you can't inline functions taking variadic arguments and we don't want to make a function + * call in production systems just to find out a verbose flag isn't set. The macro must take a + * format string and at least one additional argument, there's no portable way to remove the + * comma before an empty __VA_ARGS__ value. + */ +#define __wt_verbose(session, category, fmt, ...) \ + __wt_verbose_level(session, category, WT_VERBOSE_DEFAULT, fmt, __VA_ARGS__) + +/* + * __wt_verbose_level_multi -- + * Display a verbose message, given a set of multiple verbose categories. A verbose message will + * be displayed if at least one category in the set satisfies the required verbosity level. + */ +#define __wt_verbose_level_multi(session, multi_category, level, fmt, ...) \ + do { \ + uint32_t __v_idx; \ + for (__v_idx = 0; __v_idx < multi_category.cnt; __v_idx++) { \ + if (WT_VERBOSE_LEVEL_ISSET(session, multi_category.categories[__v_idx], level)) { \ + __wt_verbose_worker(session, "[" #multi_category "] " fmt, __VA_ARGS__); \ + break; \ + } \ + } \ + } while (0) + +/* + * __wt_verbose_multi -- + * Display a verbose message, given a set of multiple verbose categories using the default + * verbosity level. + */ +#define __wt_verbose_multi(session, multi_category, fmt, ...) \ + do { \ + uint32_t __v_idx; \ + for (__v_idx = 0; __v_idx < multi_category.cnt; __v_idx++) { \ + if (WT_VERBOSE_ISSET(session, multi_category.categories[__v_idx])) { \ + __wt_verbose_worker(session, "[" #multi_category "] " fmt, __VA_ARGS__); \ + break; \ + } \ + } \ + } while (0) diff --git a/src/third_party/wiredtiger/src/include/verify_build.h b/src/third_party/wiredtiger/src/include/verify_build.h index af0e81c141d..1b83b6e1813 100644 --- a/src/third_party/wiredtiger/src/include/verify_build.h +++ b/src/third_party/wiredtiger/src/include/verify_build.h @@ -49,6 +49,7 @@ static inline void __wt_verify_build(void) { /* Check specific structures weren't padded. */ + WT_SIZE_CHECK(WT_BLKCACHE_ID, WT_BLKCACHE_ID_SIZE); WT_SIZE_CHECK(WT_BLOCK_DESC, WT_BLOCK_DESC_SIZE); WT_SIZE_CHECK(WT_REF, WT_REF_SIZE); diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index ec50d871f90..928ae7d6148 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -727,15 +727,15 @@ struct __wt_cursor { #define WT_CURSTD_JOINED 0x0008000u #define WT_CURSTD_KEY_EXT 0x0010000u /* Key points out of tree. */ #define WT_CURSTD_KEY_INT 0x0020000u /* Key points into tree. */ -#define WT_CURSTD_META_INUSE 0x0040000u -#define WT_CURSTD_OPEN 0x0080000u -#define WT_CURSTD_OVERWRITE 0x0100000u -#define WT_CURSTD_PREFIX_SEARCH 0x0200000u -#define WT_CURSTD_RAW 0x0400000u -#define WT_CURSTD_RAW_SEARCH 0x0800000u -#define WT_CURSTD_VALUE_EXT 0x1000000u /* Value points out of tree. */ -#define WT_CURSTD_VALUE_INT 0x2000000u /* Value points into tree. */ -#define WT_CURSTD_VISIBLE_ALL 0x4000000u +#define WT_CURSTD_KEY_ONLY 0x0040000u +#define WT_CURSTD_META_INUSE 0x0080000u +#define WT_CURSTD_OPEN 0x0100000u +#define WT_CURSTD_OVERWRITE 0x0200000u +#define WT_CURSTD_PREFIX_SEARCH 0x0400000u +#define WT_CURSTD_RAW 0x0800000u +#define WT_CURSTD_RAW_SEARCH 0x1000000u +#define WT_CURSTD_VALUE_EXT 0x2000000u /* Value points out of tree. */ +#define WT_CURSTD_VALUE_INT 0x4000000u /* Value points into tree. */ /* AUTOMATIC FLAG VALUE GENERATION STOP 32 */ #define WT_CURSTD_KEY_SET (WT_CURSTD_KEY_EXT | WT_CURSTD_KEY_INT) #define WT_CURSTD_VALUE_SET (WT_CURSTD_VALUE_EXT | WT_CURSTD_VALUE_INT) @@ -799,6 +799,10 @@ struct __wt_session { * keep it available to be reused for later calls to WT_SESSION::open_cursor. Cached * cursors may be eventually closed. This value is inherited from ::wiredtiger_open \c * cache_cursors., a boolean flag; default \c true.} + * @config{cache_max_wait_ms, the maximum number of milliseconds an application thread will + * wait for space to be available in cache before giving up. Default value will be the + * global setting of the connection config., an integer greater than or equal to 0; default + * \c 0.} * @config{ignore_cache_size, when set\, operations performed by this session ignore the * cache size and are not blocked when the cache is full. Note that use of this option for * operations that create cache pressure can starve ordinary sessions that obey the cache @@ -1170,11 +1174,8 @@ struct __wt_session { * @config{ ),,} * @config{internal_item_max, This option is no longer supported\, retained for backward * compatibility., an integer greater than or equal to 0; default \c 0.} - * @config{internal_key_max, the largest key stored in an internal node\, in bytes. If - * set\, keys larger than the specified size are stored as overflow items (which may require - * additional I/O to access). The default and the maximum allowed value are both one-tenth - * the size of a newly split internal page., an integer greater than or equal to 0; default - * \c 0.} + * @config{internal_key_max, This option is no longer supported\, retained for backward + * compatibility., an integer greater than or equal to 0; default \c 0.} * @config{internal_key_truncate, configure internal key truncation\, discarding unnecessary * trailing bytes on internal keys (ignored for custom collators)., a boolean flag; default * \c true.} @@ -1695,12 +1696,11 @@ struct __wt_session { * and the commit timestamp of this transaction can be rounded up. If the prepare timestamp * is less than the oldest timestamp\, the prepare timestamp will be rounded to the oldest * timestamp. If the commit timestamp is less than the prepare timestamp\, the commit - * timestamp will be rounded up to the prepare timestamp. Allows setting the prepared - * timestamp smaller than or equal to the latest active read timestamp., a boolean flag; - * default \c false.} - * @config{ read, if the read timestamp is less - * than the oldest timestamp\, the read timestamp will be rounded up to the oldest - * timestamp., a boolean flag; default \c false.} + * timestamp will be rounded up to the prepare timestamp., a boolean flag; default \c + * false.} + * @config{ read, if the read timestamp is less than the + * oldest timestamp\, the read timestamp will be rounded up to the oldest timestamp., a + * boolean flag; default \c false.} * @config{ ),,} * @config{sync, whether to sync log records when the transaction commits\, inherited from * ::wiredtiger_open \c transaction_sync., a boolean flag; default empty.} @@ -2012,6 +2012,41 @@ struct __wt_connection { * * @param connection the connection handle * @configstart{WT_CONNECTION.reconfigure, see dist/api_data.py} + * @config{block_cache = (, block cache configuration options., a set of related + * configuration options defined below.} + * @config{ + * blkcache_eviction_aggression, seconds an unused block remains in the cache before it is + * evicted., an integer between 1 and 7200; default \c 1800.} + * @config{ cache_on_checkpoint, cache blocks written by a + * checkpoint., a boolean flag; default \c true.} + * @config{ + * cache_on_writes, cache blocks as they are written (other than checkpoint blocks)., a + * boolean flag; default \c true.} + * @config{ enabled, enable block + * cache., a boolean flag; default \c false.} + * @config{ full_target, + * the fraction of the block cache that must be full before eviction will remove unused + * blocks., an integer between 30 and 100; default \c 95.} + * @config{ + * hashsize, number of buckets in the hashtable that keeps track of blocks., an integer + * between 512 and 256K; default \c 0.} + * @config{ + * max_percent_overhead, maximum tolerated overhead expressed as the number of blocks added + * and removed as percent of blocks looked up; cache population and eviction will be + * suppressed if the overhead exceeds the supplied threshold., an integer between 1 and 500; + * default \c 10.} + * @config{ nvram_path, the absolute path to the file + * system mounted on the NVRAM device., a string; default empty.} + * @config{ percent_file_in_dram, bypass cache for a file if the set + * percentage of the file fits in system DRAM (as specified by block_cache.system_ram)., an + * integer between 0 and 100; default \c 50.} + * @config{ size, maximum + * memory to allocate for the block cache., an integer between 0 and 10TB; default \c 0.} + * @config{ system_ram, the bytes of system DRAM available for + * caching filesystem blocks., an integer between 0 and 1024GB; default \c 0.} + * @config{ type, cache location: DRAM or NVRAM., a string; default + * empty.} + * @config{ ),,} * @config{cache_max_wait_ms, the maximum number of milliseconds an application thread will * wait for space to be available in cache before giving up. Default will wait forever., an * integer greater than or equal to 0; default \c 0.} @@ -2105,31 +2140,37 @@ struct __wt_connection { * @config{eviction_dirty_target, perform eviction in worker threads when the cache contains * at least this much dirty content. It is a percentage of the cache size if the value is * within the range of 1 to 100 or an absolute size when greater than 100. The value is not - * allowed to exceed the \c cache_size., an integer between 1 and 10TB; default \c 5.} + * allowed to exceed the \c cache_size and has to be lower than its counterpart \c + * eviction_dirty_trigger., an integer between 1 and 10TB; default \c 5.} * @config{eviction_dirty_trigger, trigger application threads to perform eviction when the * cache contains at least this much dirty content. It is a percentage of the cache size if * the value is within the range of 1 to 100 or an absolute size when greater than 100. The - * value is not allowed to exceed the \c cache_size. This setting only alters behavior if - * it is lower than eviction_trigger., an integer between 1 and 10TB; default \c 20.} + * value is not allowed to exceed the \c cache_size and has to be greater than its + * counterpart \c eviction_dirty_target. This setting only alters behavior if it is lower + * than eviction_trigger., an integer between 1 and 10TB; default \c 20.} * @config{eviction_target, perform eviction in worker threads when the cache contains at * least this much content. It is a percentage of the cache size if the value is within the * range of 10 to 100 or an absolute size when greater than 100. The value is not allowed to - * exceed the \c cache_size., an integer between 10 and 10TB; default \c 80.} + * exceed the \c cache_size and has to be lower than its counterpart \c eviction_trigger., + * an integer between 10 and 10TB; default \c 80.} * @config{eviction_trigger, trigger application threads to perform eviction when the cache * contains at least this much content. It is a percentage of the cache size if the value * is within the range of 10 to 100 or an absolute size when greater than 100. The value is - * not allowed to exceed the \c cache_size., an integer between 10 and 10TB; default \c 95.} + * not allowed to exceed the \c cache_size and has to be greater than its counterpart \c + * eviction_target., an integer between 10 and 10TB; default \c 95.} * @config{eviction_updates_target, perform eviction in worker threads when the cache * contains at least this many bytes of updates. It is a percentage of the cache size if * the value is within the range of 0 to 100 or an absolute size when greater than 100. * Calculated as half of \c eviction_dirty_target by default. The value is not allowed to - * exceed the \c cache_size., an integer between 0 and 10TB; default \c 0.} + * exceed the \c cache_size and has to be lower than its counterpart \c + * eviction_updates_trigger., an integer between 0 and 10TB; default \c 0.} * @config{eviction_updates_trigger, trigger application threads to perform eviction when * the cache contains at least this many bytes of updates. It is a percentage of the cache * size if the value is within the range of 1 to 100 or an absolute size when greater than * 100\. Calculated as half of \c eviction_dirty_trigger by default. The value is not - * allowed to exceed the \c cache_size. This setting only alters behavior if it is lower - * than \c eviction_trigger., an integer between 0 and 10TB; default \c 0.} + * allowed to exceed the \c cache_size and has to be greater than its counterpart \c + * eviction_updates_target. This setting only alters behavior if it is lower than \c + * eviction_trigger., an integer between 0 and 10TB; default \c 0.} * @config{file_manager = (, control how file handles are managed., a set of related * configuration options defined below.} * @config{ @@ -2259,16 +2300,19 @@ struct __wt_connection { * approximate size of objects before creating them on the tiered storage tier., an integer * between 100K and 10TB; default \c 10M.} * @config{ ),,} - * @config{verbose, enable messages for various events. Options are given as a list\, such - * as <code>"verbose=[evictserver\,read]"</code>., a list\, with values chosen from the - * following options: \c "api"\, \c "backup"\, \c "block"\, \c "checkpoint"\, \c - * "checkpoint_cleanup"\, \c "checkpoint_progress"\, \c "compact"\, \c "compact_progress"\, - * \c "error_returns"\, \c "evict"\, \c "evict_stuck"\, \c "evictserver"\, \c "fileops"\, \c - * "handleops"\, \c "log"\, \c "history_store"\, \c "history_store_activity"\, \c "lsm"\, \c - * "lsm_manager"\, \c "metadata"\, \c "mutex"\, \c "overflow"\, \c "read"\, \c "reconcile"\, - * \c "recovery"\, \c "recovery_progress"\, \c "rts"\, \c "salvage"\, \c "shared_cache"\, \c - * "split"\, \c "temporary"\, \c "thread_group"\, \c "tiered"\, \c "timestamp"\, \c - * "transaction"\, \c "verify"\, \c "version"\, \c "write"; default \c [].} + * @config{verbose, enable messages for various subsystems and operations. Options are + * given as a list\, where each message type can optionally define an associated verbosity + * level\, such as <code>"verbose=[evictserver\,read:1\,rts:0]"</code>. Verbosity levels + * that can be provided include <code>0</code> (INFO) and <code>1</code> (DEBUG)., a list\, + * with values chosen from the following options: \c "api"\, \c "backup"\, \c "block"\, \c + * "block_cache"\, \c "checkpoint"\, \c "checkpoint_cleanup"\, \c "checkpoint_progress"\, \c + * "compact"\, \c "compact_progress"\, \c "error_returns"\, \c "evict"\, \c "evict_stuck"\, + * \c "evictserver"\, \c "fileops"\, \c "handleops"\, \c "history_store"\, \c + * "history_store_activity"\, \c "log"\, \c "lsm"\, \c "lsm_manager"\, \c "metadata"\, \c + * "mutex"\, \c "overflow"\, \c "read"\, \c "reconcile"\, \c "recovery"\, \c + * "recovery_progress"\, \c "rts"\, \c "salvage"\, \c "shared_cache"\, \c "split"\, \c + * "temporary"\, \c "thread_group"\, \c "tiered"\, \c "timestamp"\, \c "transaction"\, \c + * "verify"\, \c "version"\, \c "write"; default \c [].} * @configend * @errors */ @@ -2335,6 +2379,10 @@ struct __wt_connection { * keep it available to be reused for later calls to WT_SESSION::open_cursor. Cached * cursors may be eventually closed. This value is inherited from ::wiredtiger_open \c * cache_cursors., a boolean flag; default \c true.} + * @config{cache_max_wait_ms, the maximum number of milliseconds an application thread will + * wait for space to be available in cache before giving up. Default value will be the + * global setting of the connection config., an integer greater than or equal to 0; default + * \c 0.} * @config{ignore_cache_size, when set\, operations performed by this session ignore the * cache size and are not blocked when the cache is full. Note that use of this option for * operations that create cache pressure can starve ordinary sessions that obey the cache @@ -2660,6 +2708,39 @@ struct __wt_connection { * event handler is installed that writes error messages to stderr. See * @ref event_message_handling for more information. * @configstart{wiredtiger_open, see dist/api_data.py} + * @config{block_cache = (, block cache configuration options., a set of related configuration + * options defined below.} + * @config{ blkcache_eviction_aggression, seconds an + * unused block remains in the cache before it is evicted., an integer between 1 and 7200; default + * \c 1800.} + * @config{ cache_on_checkpoint, cache blocks written by a + * checkpoint., a boolean flag; default \c true.} + * @config{ cache_on_writes, + * cache blocks as they are written (other than checkpoint blocks)., a boolean flag; default \c + * true.} + * @config{ enabled, enable block cache., a boolean flag; default \c + * false.} + * @config{ full_target, the fraction of the block cache that must be + * full before eviction will remove unused blocks., an integer between 30 and 100; default \c 95.} + * @config{ hashsize, number of buckets in the hashtable that keeps track of + * blocks., an integer between 512 and 256K; default \c 0.} + * @config{ + * max_percent_overhead, maximum tolerated overhead expressed as the number of blocks added and + * removed as percent of blocks looked up; cache population and eviction will be suppressed if the + * overhead exceeds the supplied threshold., an integer between 1 and 500; default \c 10.} + * @config{ nvram_path, the absolute path to the file system mounted on the + * NVRAM device., a string; default empty.} + * @config{ percent_file_in_dram, + * bypass cache for a file if the set percentage of the file fits in system DRAM (as specified by + * block_cache.system_ram)., an integer between 0 and 100; default \c 50.} + * @config{ size, maximum memory to allocate for the block cache., an integer + * between 0 and 10TB; default \c 0.} + * @config{ system_ram, the bytes of + * system DRAM available for caching filesystem blocks., an integer between 0 and 1024GB; default \c + * 0.} + * @config{ type, cache location: DRAM or NVRAM., a string; default + * empty.} + * @config{ ),,} * @config{buffer_alignment, in-memory alignment (in bytes) for buffers used for I/O. The default * value of -1 indicates a platform-specific alignment value should be used (4KB on Linux systems * when direct I/O is configured\, zero elsewhere)., an integer between -1 and 1MB; default \c -1.} @@ -2802,31 +2883,36 @@ struct __wt_connection { * @config{eviction_dirty_target, perform eviction in worker threads when the cache contains at * least this much dirty content. It is a percentage of the cache size if the value is within the * range of 1 to 100 or an absolute size when greater than 100. The value is not allowed to exceed - * the \c cache_size., an integer between 1 and 10TB; default \c 5.} + * the \c cache_size and has to be lower than its counterpart \c eviction_dirty_trigger., an integer + * between 1 and 10TB; default \c 5.} * @config{eviction_dirty_trigger, trigger application threads to perform eviction when the cache * contains at least this much dirty content. It is a percentage of the cache size if the value is * within the range of 1 to 100 or an absolute size when greater than 100. The value is not allowed - * to exceed the \c cache_size. This setting only alters behavior if it is lower than - * eviction_trigger., an integer between 1 and 10TB; default \c 20.} + * to exceed the \c cache_size and has to be greater than its counterpart \c eviction_dirty_target. + * This setting only alters behavior if it is lower than eviction_trigger., an integer between 1 and + * 10TB; default \c 20.} * @config{eviction_target, perform eviction in worker threads when the cache contains at least this * much content. It is a percentage of the cache size if the value is within the range of 10 to 100 - * or an absolute size when greater than 100. The value is not allowed to exceed the \c cache_size., - * an integer between 10 and 10TB; default \c 80.} + * or an absolute size when greater than 100. The value is not allowed to exceed the \c cache_size + * and has to be lower than its counterpart \c eviction_trigger., an integer between 10 and 10TB; + * default \c 80.} * @config{eviction_trigger, trigger application threads to perform eviction when the cache contains * at least this much content. It is a percentage of the cache size if the value is within the * range of 10 to 100 or an absolute size when greater than 100. The value is not allowed to exceed - * the \c cache_size., an integer between 10 and 10TB; default \c 95.} + * the \c cache_size and has to be greater than its counterpart \c eviction_target., an integer + * between 10 and 10TB; default \c 95.} * @config{eviction_updates_target, perform eviction in worker threads when the cache contains at * least this many bytes of updates. It is a percentage of the cache size if the value is within * the range of 0 to 100 or an absolute size when greater than 100. Calculated as half of \c - * eviction_dirty_target by default. The value is not allowed to exceed the \c cache_size., an - * integer between 0 and 10TB; default \c 0.} + * eviction_dirty_target by default. The value is not allowed to exceed the \c cache_size and has + * to be lower than its counterpart \c eviction_updates_trigger., an integer between 0 and 10TB; + * default \c 0.} * @config{eviction_updates_trigger, trigger application threads to perform eviction when the cache * contains at least this many bytes of updates. It is a percentage of the cache size if the value * is within the range of 1 to 100 or an absolute size when greater than 100\. Calculated as half of - * \c eviction_dirty_trigger by default. The value is not allowed to exceed the \c cache_size. - * This setting only alters behavior if it is lower than \c eviction_trigger., an integer between 0 - * and 10TB; default \c 0.} + * \c eviction_dirty_trigger by default. The value is not allowed to exceed the \c cache_size and + * has to be greater than its counterpart \c eviction_updates_target. This setting only alters + * behavior if it is lower than \c eviction_trigger., an integer between 0 and 10TB; default \c 0.} * @config{exclusive, fail if the database already exists\, generally used with the \c create * option., a boolean flag; default \c false.} * @config{extensions, list of shared library extensions to load (using dlopen). Any values @@ -3020,16 +3106,18 @@ struct __wt_connection { * @config{use_environment_priv, use the \c WIREDTIGER_CONFIG and \c WIREDTIGER_HOME environment * variables even if the process is running with special privileges. See @ref home for more * information., a boolean flag; default \c false.} - * @config{verbose, enable messages for various events. Options are given as a list\, such as - * <code>"verbose=[evictserver\,read]"</code>., a list\, with values chosen from the following - * options: \c "api"\, \c "backup"\, \c "block"\, \c "checkpoint"\, \c "checkpoint_cleanup"\, \c - * "checkpoint_progress"\, \c "compact"\, \c "compact_progress"\, \c "error_returns"\, \c "evict"\, - * \c "evict_stuck"\, \c "evictserver"\, \c "fileops"\, \c "handleops"\, \c "log"\, \c - * "history_store"\, \c "history_store_activity"\, \c "lsm"\, \c "lsm_manager"\, \c "metadata"\, \c - * "mutex"\, \c "overflow"\, \c "read"\, \c "reconcile"\, \c "recovery"\, \c "recovery_progress"\, - * \c "rts"\, \c "salvage"\, \c "shared_cache"\, \c "split"\, \c "temporary"\, \c "thread_group"\, - * \c "tiered"\, \c "timestamp"\, \c "transaction"\, \c "verify"\, \c "version"\, \c "write"; - * default \c [].} + * @config{verbose, enable messages for various subsystems and operations. Options are given as a + * list\, where each message type can optionally define an associated verbosity level\, such as + * <code>"verbose=[evictserver\,read:1\,rts:0]"</code>. Verbosity levels that can be provided + * include <code>0</code> (INFO) and <code>1</code> (DEBUG)., a list\, with values chosen from the + * following options: \c "api"\, \c "backup"\, \c "block"\, \c "block_cache"\, \c "checkpoint"\, \c + * "checkpoint_cleanup"\, \c "checkpoint_progress"\, \c "compact"\, \c "compact_progress"\, \c + * "error_returns"\, \c "evict"\, \c "evict_stuck"\, \c "evictserver"\, \c "fileops"\, \c + * "handleops"\, \c "history_store"\, \c "history_store_activity"\, \c "log"\, \c "lsm"\, \c + * "lsm_manager"\, \c "metadata"\, \c "mutex"\, \c "overflow"\, \c "read"\, \c "reconcile"\, \c + * "recovery"\, \c "recovery_progress"\, \c "rts"\, \c "salvage"\, \c "shared_cache"\, \c "split"\, + * \c "temporary"\, \c "thread_group"\, \c "tiered"\, \c "timestamp"\, \c "transaction"\, \c + * "verify"\, \c "version"\, \c "write"; default \c [].} * @config{verify_metadata, open connection and verify any WiredTiger metadata. This API allows * verification and detection of corruption in WiredTiger metadata., a boolean flag; default \c * false.} @@ -5024,1190 +5112,1234 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_CONN_LSM_WORK_UNITS_CREATED 1008 /*! LSM: tree queue hit maximum */ #define WT_STAT_CONN_LSM_WORK_QUEUE_MAX 1009 +/*! block-manager: block cache cached blocks updated */ +#define WT_STAT_CONN_BLOCK_CACHE_BLOCKS_UPDATE 1010 +/*! block-manager: block cache cached bytes updated */ +#define WT_STAT_CONN_BLOCK_CACHE_BYTES_UPDATE 1011 +/*! block-manager: block cache evicted blocks */ +#define WT_STAT_CONN_BLOCK_CACHE_BLOCKS_EVICTED 1012 +/*! block-manager: block cache file size causing bypass */ +#define WT_STAT_CONN_BLOCK_CACHE_BYPASS_FILESIZE 1013 +/*! block-manager: block cache lookups */ +#define WT_STAT_CONN_BLOCK_CACHE_DATA_REFS 1014 +/*! + * block-manager: block cache number of blocks not evicted due to + * overhead + */ +#define WT_STAT_CONN_BLOCK_CACHE_NOT_EVICTED_OVERHEAD 1015 +/*! + * block-manager: block cache number of bypasses because no-write- + * allocate setting was on + */ +#define WT_STAT_CONN_BLOCK_CACHE_BYPASS_WRITEALLOC 1016 +/*! block-manager: block cache number of bypasses due to overhead on put */ +#define WT_STAT_CONN_BLOCK_CACHE_BYPASS_OVERHEAD_PUT 1017 +/*! block-manager: block cache number of bypasses on get */ +#define WT_STAT_CONN_BLOCK_CACHE_BYPASS_GET 1018 +/*! + * block-manager: block cache number of bypasses on put because file is + * too small + */ +#define WT_STAT_CONN_BLOCK_CACHE_BYPASS_PUT 1019 +/*! block-manager: block cache number of eviction passes */ +#define WT_STAT_CONN_BLOCK_CACHE_EVICTION_PASSES 1020 +/*! block-manager: block cache number of hits including existence checks */ +#define WT_STAT_CONN_BLOCK_CACHE_HITS 1021 +/*! block-manager: block cache number of misses including existence checks */ +#define WT_STAT_CONN_BLOCK_CACHE_MISSES 1022 +/*! block-manager: block cache number of put bypasses on checkpoint I/O */ +#define WT_STAT_CONN_BLOCK_CACHE_BYPASS_CHKPT 1023 +/*! block-manager: block cache removed blocks */ +#define WT_STAT_CONN_BLOCK_CACHE_BLOCKS_REMOVED 1024 +/*! block-manager: block cache total blocks */ +#define WT_STAT_CONN_BLOCK_CACHE_BLOCKS 1025 +/*! block-manager: block cache total blocks inserted on read path */ +#define WT_STAT_CONN_BLOCK_CACHE_BLOCKS_INSERT_READ 1026 +/*! block-manager: block cache total blocks inserted on write path */ +#define WT_STAT_CONN_BLOCK_CACHE_BLOCKS_INSERT_WRITE 1027 +/*! block-manager: block cache total bytes */ +#define WT_STAT_CONN_BLOCK_CACHE_BYTES 1028 +/*! block-manager: block cache total bytes inserted on read path */ +#define WT_STAT_CONN_BLOCK_CACHE_BYTES_INSERT_READ 1029 +/*! block-manager: block cache total bytes inserted on write path */ +#define WT_STAT_CONN_BLOCK_CACHE_BYTES_INSERT_WRITE 1030 /*! block-manager: blocks pre-loaded */ -#define WT_STAT_CONN_BLOCK_PRELOAD 1010 +#define WT_STAT_CONN_BLOCK_PRELOAD 1031 /*! block-manager: blocks read */ -#define WT_STAT_CONN_BLOCK_READ 1011 +#define WT_STAT_CONN_BLOCK_READ 1032 /*! block-manager: blocks written */ -#define WT_STAT_CONN_BLOCK_WRITE 1012 +#define WT_STAT_CONN_BLOCK_WRITE 1033 /*! block-manager: bytes read */ -#define WT_STAT_CONN_BLOCK_BYTE_READ 1013 +#define WT_STAT_CONN_BLOCK_BYTE_READ 1034 /*! block-manager: bytes read via memory map API */ -#define WT_STAT_CONN_BLOCK_BYTE_READ_MMAP 1014 +#define WT_STAT_CONN_BLOCK_BYTE_READ_MMAP 1035 /*! block-manager: bytes read via system call API */ -#define WT_STAT_CONN_BLOCK_BYTE_READ_SYSCALL 1015 +#define WT_STAT_CONN_BLOCK_BYTE_READ_SYSCALL 1036 /*! block-manager: bytes written */ -#define WT_STAT_CONN_BLOCK_BYTE_WRITE 1016 +#define WT_STAT_CONN_BLOCK_BYTE_WRITE 1037 /*! block-manager: bytes written for checkpoint */ -#define WT_STAT_CONN_BLOCK_BYTE_WRITE_CHECKPOINT 1017 +#define WT_STAT_CONN_BLOCK_BYTE_WRITE_CHECKPOINT 1038 /*! block-manager: bytes written via memory map API */ -#define WT_STAT_CONN_BLOCK_BYTE_WRITE_MMAP 1018 +#define WT_STAT_CONN_BLOCK_BYTE_WRITE_MMAP 1039 /*! block-manager: bytes written via system call API */ -#define WT_STAT_CONN_BLOCK_BYTE_WRITE_SYSCALL 1019 +#define WT_STAT_CONN_BLOCK_BYTE_WRITE_SYSCALL 1040 /*! block-manager: mapped blocks read */ -#define WT_STAT_CONN_BLOCK_MAP_READ 1020 +#define WT_STAT_CONN_BLOCK_MAP_READ 1041 /*! block-manager: mapped bytes read */ -#define WT_STAT_CONN_BLOCK_BYTE_MAP_READ 1021 +#define WT_STAT_CONN_BLOCK_BYTE_MAP_READ 1042 /*! * block-manager: number of times the file was remapped because it * changed size via fallocate or truncate */ -#define WT_STAT_CONN_BLOCK_REMAP_FILE_RESIZE 1022 +#define WT_STAT_CONN_BLOCK_REMAP_FILE_RESIZE 1043 /*! block-manager: number of times the region was remapped via write */ -#define WT_STAT_CONN_BLOCK_REMAP_FILE_WRITE 1023 +#define WT_STAT_CONN_BLOCK_REMAP_FILE_WRITE 1044 /*! cache: application threads page read from disk to cache count */ -#define WT_STAT_CONN_CACHE_READ_APP_COUNT 1024 +#define WT_STAT_CONN_CACHE_READ_APP_COUNT 1045 /*! cache: application threads page read from disk to cache time (usecs) */ -#define WT_STAT_CONN_CACHE_READ_APP_TIME 1025 +#define WT_STAT_CONN_CACHE_READ_APP_TIME 1046 /*! cache: application threads page write from cache to disk count */ -#define WT_STAT_CONN_CACHE_WRITE_APP_COUNT 1026 +#define WT_STAT_CONN_CACHE_WRITE_APP_COUNT 1047 /*! cache: application threads page write from cache to disk time (usecs) */ -#define WT_STAT_CONN_CACHE_WRITE_APP_TIME 1027 +#define WT_STAT_CONN_CACHE_WRITE_APP_TIME 1048 /*! cache: bytes allocated for updates */ -#define WT_STAT_CONN_CACHE_BYTES_UPDATES 1028 +#define WT_STAT_CONN_CACHE_BYTES_UPDATES 1049 /*! cache: bytes belonging to page images in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_IMAGE 1029 +#define WT_STAT_CONN_CACHE_BYTES_IMAGE 1050 /*! cache: bytes belonging to the history store table in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_HS 1030 +#define WT_STAT_CONN_CACHE_BYTES_HS 1051 /*! cache: bytes currently in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_INUSE 1031 +#define WT_STAT_CONN_CACHE_BYTES_INUSE 1052 /*! cache: bytes dirty in the cache cumulative */ -#define WT_STAT_CONN_CACHE_BYTES_DIRTY_TOTAL 1032 +#define WT_STAT_CONN_CACHE_BYTES_DIRTY_TOTAL 1053 /*! cache: bytes not belonging to page images in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_OTHER 1033 +#define WT_STAT_CONN_CACHE_BYTES_OTHER 1054 /*! cache: bytes read into cache */ -#define WT_STAT_CONN_CACHE_BYTES_READ 1034 +#define WT_STAT_CONN_CACHE_BYTES_READ 1055 /*! cache: bytes written from cache */ -#define WT_STAT_CONN_CACHE_BYTES_WRITE 1035 +#define WT_STAT_CONN_CACHE_BYTES_WRITE 1056 /*! cache: cache overflow score */ -#define WT_STAT_CONN_CACHE_LOOKASIDE_SCORE 1036 +#define WT_STAT_CONN_CACHE_LOOKASIDE_SCORE 1057 /*! cache: checkpoint blocked page eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_CHECKPOINT 1037 +#define WT_STAT_CONN_CACHE_EVICTION_CHECKPOINT 1058 /*! * cache: checkpoint of history store file blocked non-history store page * eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_BLOCKED_CHECKPOINT_HS 1038 +#define WT_STAT_CONN_CACHE_EVICTION_BLOCKED_CHECKPOINT_HS 1059 /*! cache: eviction calls to get a page */ -#define WT_STAT_CONN_CACHE_EVICTION_GET_REF 1039 +#define WT_STAT_CONN_CACHE_EVICTION_GET_REF 1060 /*! cache: eviction calls to get a page found queue empty */ -#define WT_STAT_CONN_CACHE_EVICTION_GET_REF_EMPTY 1040 +#define WT_STAT_CONN_CACHE_EVICTION_GET_REF_EMPTY 1061 /*! cache: eviction calls to get a page found queue empty after locking */ -#define WT_STAT_CONN_CACHE_EVICTION_GET_REF_EMPTY2 1041 +#define WT_STAT_CONN_CACHE_EVICTION_GET_REF_EMPTY2 1062 /*! cache: eviction currently operating in aggressive mode */ -#define WT_STAT_CONN_CACHE_EVICTION_AGGRESSIVE_SET 1042 +#define WT_STAT_CONN_CACHE_EVICTION_AGGRESSIVE_SET 1063 /*! cache: eviction empty score */ -#define WT_STAT_CONN_CACHE_EVICTION_EMPTY_SCORE 1043 +#define WT_STAT_CONN_CACHE_EVICTION_EMPTY_SCORE 1064 /*! * cache: eviction gave up due to detecting an out of order on disk value * behind the last update on the chain */ -#define WT_STAT_CONN_CACHE_EVICTION_BLOCKED_OOO_CHECKPOINT_RACE_1 1044 +#define WT_STAT_CONN_CACHE_EVICTION_BLOCKED_OOO_CHECKPOINT_RACE_1 1065 /*! * cache: eviction gave up due to detecting an out of order tombstone * ahead of the selected on disk update */ -#define WT_STAT_CONN_CACHE_EVICTION_BLOCKED_OOO_CHECKPOINT_RACE_2 1045 +#define WT_STAT_CONN_CACHE_EVICTION_BLOCKED_OOO_CHECKPOINT_RACE_2 1066 /*! * cache: eviction gave up due to detecting an out of order tombstone * ahead of the selected on disk update after validating the update chain */ -#define WT_STAT_CONN_CACHE_EVICTION_BLOCKED_OOO_CHECKPOINT_RACE_3 1046 +#define WT_STAT_CONN_CACHE_EVICTION_BLOCKED_OOO_CHECKPOINT_RACE_3 1067 /*! * cache: eviction gave up due to detecting out of order timestamps on * the update chain after the selected on disk update */ -#define WT_STAT_CONN_CACHE_EVICTION_BLOCKED_OOO_CHECKPOINT_RACE_4 1047 +#define WT_STAT_CONN_CACHE_EVICTION_BLOCKED_OOO_CHECKPOINT_RACE_4 1068 /*! cache: eviction passes of a file */ -#define WT_STAT_CONN_CACHE_EVICTION_WALK_PASSES 1048 +#define WT_STAT_CONN_CACHE_EVICTION_WALK_PASSES 1069 /*! cache: eviction server candidate queue empty when topping up */ -#define WT_STAT_CONN_CACHE_EVICTION_QUEUE_EMPTY 1049 +#define WT_STAT_CONN_CACHE_EVICTION_QUEUE_EMPTY 1070 /*! cache: eviction server candidate queue not empty when topping up */ -#define WT_STAT_CONN_CACHE_EVICTION_QUEUE_NOT_EMPTY 1050 +#define WT_STAT_CONN_CACHE_EVICTION_QUEUE_NOT_EMPTY 1071 /*! cache: eviction server evicting pages */ -#define WT_STAT_CONN_CACHE_EVICTION_SERVER_EVICTING 1051 +#define WT_STAT_CONN_CACHE_EVICTION_SERVER_EVICTING 1072 /*! * cache: eviction server slept, because we did not make progress with * eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_SERVER_SLEPT 1052 +#define WT_STAT_CONN_CACHE_EVICTION_SERVER_SLEPT 1073 /*! cache: eviction server unable to reach eviction goal */ -#define WT_STAT_CONN_CACHE_EVICTION_SLOW 1053 +#define WT_STAT_CONN_CACHE_EVICTION_SLOW 1074 /*! cache: eviction server waiting for a leaf page */ -#define WT_STAT_CONN_CACHE_EVICTION_WALK_LEAF_NOTFOUND 1054 +#define WT_STAT_CONN_CACHE_EVICTION_WALK_LEAF_NOTFOUND 1075 /*! cache: eviction state */ -#define WT_STAT_CONN_CACHE_EVICTION_STATE 1055 +#define WT_STAT_CONN_CACHE_EVICTION_STATE 1076 /*! * cache: eviction walk most recent sleeps for checkpoint handle * gathering */ -#define WT_STAT_CONN_CACHE_EVICTION_WALK_SLEEPS 1056 +#define WT_STAT_CONN_CACHE_EVICTION_WALK_SLEEPS 1077 /*! cache: eviction walk target pages histogram - 0-9 */ -#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT10 1057 +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT10 1078 /*! cache: eviction walk target pages histogram - 10-31 */ -#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT32 1058 +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT32 1079 /*! cache: eviction walk target pages histogram - 128 and higher */ -#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_GE128 1059 +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_GE128 1080 /*! cache: eviction walk target pages histogram - 32-63 */ -#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT64 1060 +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT64 1081 /*! cache: eviction walk target pages histogram - 64-128 */ -#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT128 1061 +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT128 1082 /*! * cache: eviction walk target pages reduced due to history store cache * pressure */ -#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_REDUCED 1062 +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_REDUCED 1083 /*! cache: eviction walk target strategy both clean and dirty pages */ -#define WT_STAT_CONN_CACHE_EVICTION_TARGET_STRATEGY_BOTH_CLEAN_AND_DIRTY 1063 +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_STRATEGY_BOTH_CLEAN_AND_DIRTY 1084 /*! cache: eviction walk target strategy only clean pages */ -#define WT_STAT_CONN_CACHE_EVICTION_TARGET_STRATEGY_CLEAN 1064 +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_STRATEGY_CLEAN 1085 /*! cache: eviction walk target strategy only dirty pages */ -#define WT_STAT_CONN_CACHE_EVICTION_TARGET_STRATEGY_DIRTY 1065 +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_STRATEGY_DIRTY 1086 /*! cache: eviction walks abandoned */ -#define WT_STAT_CONN_CACHE_EVICTION_WALKS_ABANDONED 1066 +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_ABANDONED 1087 /*! cache: eviction walks gave up because they restarted their walk twice */ -#define WT_STAT_CONN_CACHE_EVICTION_WALKS_STOPPED 1067 +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_STOPPED 1088 /*! * cache: eviction walks gave up because they saw too many pages and * found no candidates */ -#define WT_STAT_CONN_CACHE_EVICTION_WALKS_GAVE_UP_NO_TARGETS 1068 +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_GAVE_UP_NO_TARGETS 1089 /*! * cache: eviction walks gave up because they saw too many pages and * found too few candidates */ -#define WT_STAT_CONN_CACHE_EVICTION_WALKS_GAVE_UP_RATIO 1069 +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_GAVE_UP_RATIO 1090 /*! cache: eviction walks reached end of tree */ -#define WT_STAT_CONN_CACHE_EVICTION_WALKS_ENDED 1070 +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_ENDED 1091 /*! cache: eviction walks restarted */ -#define WT_STAT_CONN_CACHE_EVICTION_WALK_RESTART 1071 +#define WT_STAT_CONN_CACHE_EVICTION_WALK_RESTART 1092 /*! cache: eviction walks started from root of tree */ -#define WT_STAT_CONN_CACHE_EVICTION_WALK_FROM_ROOT 1072 +#define WT_STAT_CONN_CACHE_EVICTION_WALK_FROM_ROOT 1093 /*! cache: eviction walks started from saved location in tree */ -#define WT_STAT_CONN_CACHE_EVICTION_WALK_SAVED_POS 1073 +#define WT_STAT_CONN_CACHE_EVICTION_WALK_SAVED_POS 1094 /*! cache: eviction worker thread active */ -#define WT_STAT_CONN_CACHE_EVICTION_ACTIVE_WORKERS 1074 +#define WT_STAT_CONN_CACHE_EVICTION_ACTIVE_WORKERS 1095 /*! cache: eviction worker thread created */ -#define WT_STAT_CONN_CACHE_EVICTION_WORKER_CREATED 1075 +#define WT_STAT_CONN_CACHE_EVICTION_WORKER_CREATED 1096 /*! cache: eviction worker thread evicting pages */ -#define WT_STAT_CONN_CACHE_EVICTION_WORKER_EVICTING 1076 +#define WT_STAT_CONN_CACHE_EVICTION_WORKER_EVICTING 1097 /*! cache: eviction worker thread removed */ -#define WT_STAT_CONN_CACHE_EVICTION_WORKER_REMOVED 1077 +#define WT_STAT_CONN_CACHE_EVICTION_WORKER_REMOVED 1098 /*! cache: eviction worker thread stable number */ -#define WT_STAT_CONN_CACHE_EVICTION_STABLE_STATE_WORKERS 1078 +#define WT_STAT_CONN_CACHE_EVICTION_STABLE_STATE_WORKERS 1099 /*! cache: files with active eviction walks */ -#define WT_STAT_CONN_CACHE_EVICTION_WALKS_ACTIVE 1079 +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_ACTIVE 1100 /*! cache: files with new eviction walks started */ -#define WT_STAT_CONN_CACHE_EVICTION_WALKS_STARTED 1080 +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_STARTED 1101 /*! cache: force re-tuning of eviction workers once in a while */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_RETUNE 1081 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_RETUNE 1102 /*! * cache: forced eviction - history store pages failed to evict while * session has history store cursor open */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_HS_FAIL 1082 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_HS_FAIL 1103 /*! * cache: forced eviction - history store pages selected while session * has history store cursor open */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_HS 1083 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_HS 1104 /*! * cache: forced eviction - history store pages successfully evicted * while session has history store cursor open */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_HS_SUCCESS 1084 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_HS_SUCCESS 1105 /*! cache: forced eviction - pages evicted that were clean count */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_CLEAN 1085 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_CLEAN 1106 /*! cache: forced eviction - pages evicted that were clean time (usecs) */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_CLEAN_TIME 1086 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_CLEAN_TIME 1107 /*! cache: forced eviction - pages evicted that were dirty count */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_DIRTY 1087 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_DIRTY 1108 /*! cache: forced eviction - pages evicted that were dirty time (usecs) */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_DIRTY_TIME 1088 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_DIRTY_TIME 1109 /*! * cache: forced eviction - pages selected because of a large number of * updates to a single item */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_LONG_UPDATE_LIST 1089 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_LONG_UPDATE_LIST 1110 /*! * cache: forced eviction - pages selected because of too many deleted * items count */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_DELETE 1090 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_DELETE 1111 /*! cache: forced eviction - pages selected count */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE 1091 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE 1112 /*! cache: forced eviction - pages selected unable to be evicted count */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_FAIL 1092 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_FAIL 1113 /*! cache: forced eviction - pages selected unable to be evicted time */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_FAIL_TIME 1093 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_FAIL_TIME 1114 /*! cache: hazard pointer blocked page eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_HAZARD 1094 +#define WT_STAT_CONN_CACHE_EVICTION_HAZARD 1115 /*! cache: hazard pointer check calls */ -#define WT_STAT_CONN_CACHE_HAZARD_CHECKS 1095 +#define WT_STAT_CONN_CACHE_HAZARD_CHECKS 1116 /*! cache: hazard pointer check entries walked */ -#define WT_STAT_CONN_CACHE_HAZARD_WALKS 1096 +#define WT_STAT_CONN_CACHE_HAZARD_WALKS 1117 /*! cache: hazard pointer maximum array length */ -#define WT_STAT_CONN_CACHE_HAZARD_MAX 1097 +#define WT_STAT_CONN_CACHE_HAZARD_MAX 1118 /*! cache: history store score */ -#define WT_STAT_CONN_CACHE_HS_SCORE 1098 +#define WT_STAT_CONN_CACHE_HS_SCORE 1119 /*! cache: history store table insert calls */ -#define WT_STAT_CONN_CACHE_HS_INSERT 1099 +#define WT_STAT_CONN_CACHE_HS_INSERT 1120 /*! cache: history store table insert calls that returned restart */ -#define WT_STAT_CONN_CACHE_HS_INSERT_RESTART 1100 +#define WT_STAT_CONN_CACHE_HS_INSERT_RESTART 1121 /*! cache: history store table max on-disk size */ -#define WT_STAT_CONN_CACHE_HS_ONDISK_MAX 1101 +#define WT_STAT_CONN_CACHE_HS_ONDISK_MAX 1122 /*! cache: history store table on-disk size */ -#define WT_STAT_CONN_CACHE_HS_ONDISK 1102 +#define WT_STAT_CONN_CACHE_HS_ONDISK 1123 /*! * cache: history store table out-of-order resolved updates that lose * their durable timestamp */ -#define WT_STAT_CONN_CACHE_HS_ORDER_LOSE_DURABLE_TIMESTAMP 1103 +#define WT_STAT_CONN_CACHE_HS_ORDER_LOSE_DURABLE_TIMESTAMP 1124 /*! * cache: history store table out-of-order updates that were fixed up by * reinserting with the fixed timestamp */ -#define WT_STAT_CONN_CACHE_HS_ORDER_REINSERT 1104 +#define WT_STAT_CONN_CACHE_HS_ORDER_REINSERT 1125 /*! cache: history store table reads */ -#define WT_STAT_CONN_CACHE_HS_READ 1105 +#define WT_STAT_CONN_CACHE_HS_READ 1126 /*! cache: history store table reads missed */ -#define WT_STAT_CONN_CACHE_HS_READ_MISS 1106 +#define WT_STAT_CONN_CACHE_HS_READ_MISS 1127 /*! cache: history store table reads requiring squashed modifies */ -#define WT_STAT_CONN_CACHE_HS_READ_SQUASH 1107 +#define WT_STAT_CONN_CACHE_HS_READ_SQUASH 1128 /*! * cache: history store table truncation by rollback to stable to remove * an unstable update */ -#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE_RTS_UNSTABLE 1108 +#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE_RTS_UNSTABLE 1129 /*! * cache: history store table truncation by rollback to stable to remove * an update */ -#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE_RTS 1109 +#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE_RTS 1130 /*! cache: history store table truncation to remove an update */ -#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE 1110 +#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE 1131 /*! * cache: history store table truncation to remove range of updates due * to key being removed from the data page during reconciliation */ -#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE_ONPAGE_REMOVAL 1111 +#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE_ONPAGE_REMOVAL 1132 /*! * cache: history store table truncation to remove range of updates due * to out-of-order timestamp update on data page */ -#define WT_STAT_CONN_CACHE_HS_ORDER_REMOVE 1112 +#define WT_STAT_CONN_CACHE_HS_ORDER_REMOVE 1133 /*! cache: history store table writes requiring squashed modifies */ -#define WT_STAT_CONN_CACHE_HS_WRITE_SQUASH 1113 +#define WT_STAT_CONN_CACHE_HS_WRITE_SQUASH 1134 /*! cache: in-memory page passed criteria to be split */ -#define WT_STAT_CONN_CACHE_INMEM_SPLITTABLE 1114 +#define WT_STAT_CONN_CACHE_INMEM_SPLITTABLE 1135 /*! cache: in-memory page splits */ -#define WT_STAT_CONN_CACHE_INMEM_SPLIT 1115 +#define WT_STAT_CONN_CACHE_INMEM_SPLIT 1136 /*! cache: internal pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL 1116 +#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL 1137 /*! cache: internal pages queued for eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL_PAGES_QUEUED 1117 +#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL_PAGES_QUEUED 1138 /*! cache: internal pages seen by eviction walk */ -#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL_PAGES_SEEN 1118 +#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL_PAGES_SEEN 1139 /*! cache: internal pages seen by eviction walk that are already queued */ -#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL_PAGES_ALREADY_QUEUED 1119 +#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL_PAGES_ALREADY_QUEUED 1140 /*! cache: internal pages split during eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_INTERNAL 1120 +#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_INTERNAL 1141 /*! cache: leaf pages split during eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_LEAF 1121 +#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_LEAF 1142 /*! cache: maximum bytes configured */ -#define WT_STAT_CONN_CACHE_BYTES_MAX 1122 +#define WT_STAT_CONN_CACHE_BYTES_MAX 1143 /*! cache: maximum page size at eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_MAXIMUM_PAGE_SIZE 1123 +#define WT_STAT_CONN_CACHE_EVICTION_MAXIMUM_PAGE_SIZE 1144 /*! cache: modified pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_DIRTY 1124 +#define WT_STAT_CONN_CACHE_EVICTION_DIRTY 1145 /*! cache: modified pages evicted by application threads */ -#define WT_STAT_CONN_CACHE_EVICTION_APP_DIRTY 1125 +#define WT_STAT_CONN_CACHE_EVICTION_APP_DIRTY 1146 /*! cache: operations timed out waiting for space in cache */ -#define WT_STAT_CONN_CACHE_TIMED_OUT_OPS 1126 +#define WT_STAT_CONN_CACHE_TIMED_OUT_OPS 1147 /*! cache: overflow pages read into cache */ -#define WT_STAT_CONN_CACHE_READ_OVERFLOW 1127 +#define WT_STAT_CONN_CACHE_READ_OVERFLOW 1148 /*! cache: page split during eviction deepened the tree */ -#define WT_STAT_CONN_CACHE_EVICTION_DEEPEN 1128 +#define WT_STAT_CONN_CACHE_EVICTION_DEEPEN 1149 /*! cache: page written requiring history store records */ -#define WT_STAT_CONN_CACHE_WRITE_HS 1129 +#define WT_STAT_CONN_CACHE_WRITE_HS 1150 /*! cache: pages currently held in the cache */ -#define WT_STAT_CONN_CACHE_PAGES_INUSE 1130 +#define WT_STAT_CONN_CACHE_PAGES_INUSE 1151 /*! cache: pages evicted by application threads */ -#define WT_STAT_CONN_CACHE_EVICTION_APP 1131 +#define WT_STAT_CONN_CACHE_EVICTION_APP 1152 /*! cache: pages evicted in parallel with checkpoint */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_IN_PARALLEL_WITH_CHECKPOINT 1132 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_IN_PARALLEL_WITH_CHECKPOINT 1153 /*! cache: pages queued for eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED 1133 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED 1154 /*! cache: pages queued for eviction post lru sorting */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_POST_LRU 1134 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_POST_LRU 1155 /*! cache: pages queued for urgent eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_URGENT 1135 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_URGENT 1156 /*! cache: pages queued for urgent eviction during walk */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_OLDEST 1136 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_OLDEST 1157 /*! * cache: pages queued for urgent eviction from history store due to high * dirty content */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_URGENT_HS_DIRTY 1137 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_URGENT_HS_DIRTY 1158 /*! cache: pages read into cache */ -#define WT_STAT_CONN_CACHE_READ 1138 +#define WT_STAT_CONN_CACHE_READ 1159 /*! cache: pages read into cache after truncate */ -#define WT_STAT_CONN_CACHE_READ_DELETED 1139 +#define WT_STAT_CONN_CACHE_READ_DELETED 1160 /*! cache: pages read into cache after truncate in prepare state */ -#define WT_STAT_CONN_CACHE_READ_DELETED_PREPARED 1140 +#define WT_STAT_CONN_CACHE_READ_DELETED_PREPARED 1161 /*! cache: pages requested from the cache */ -#define WT_STAT_CONN_CACHE_PAGES_REQUESTED 1141 +#define WT_STAT_CONN_CACHE_PAGES_REQUESTED 1162 /*! cache: pages seen by eviction walk */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_SEEN 1142 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_SEEN 1163 /*! cache: pages seen by eviction walk that are already queued */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_ALREADY_QUEUED 1143 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_ALREADY_QUEUED 1164 /*! cache: pages selected for eviction unable to be evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_FAIL 1144 -/*! - * cache: pages selected for eviction unable to be evicted as the parent - * page has overflow items - */ -#define WT_STAT_CONN_CACHE_EVICTION_FAIL_PARENT_HAS_OVERFLOW_ITEMS 1145 +#define WT_STAT_CONN_CACHE_EVICTION_FAIL 1165 /*! * cache: pages selected for eviction unable to be evicted because of * active children on an internal page */ -#define WT_STAT_CONN_CACHE_EVICTION_FAIL_ACTIVE_CHILDREN_ON_AN_INTERNAL_PAGE 1146 +#define WT_STAT_CONN_CACHE_EVICTION_FAIL_ACTIVE_CHILDREN_ON_AN_INTERNAL_PAGE 1166 /*! * cache: pages selected for eviction unable to be evicted because of * failure in reconciliation */ -#define WT_STAT_CONN_CACHE_EVICTION_FAIL_IN_RECONCILIATION 1147 +#define WT_STAT_CONN_CACHE_EVICTION_FAIL_IN_RECONCILIATION 1167 /*! * cache: pages selected for eviction unable to be evicted because of * race between checkpoint and out of order timestamps handling */ -#define WT_STAT_CONN_CACHE_EVICTION_FAIL_CHECKPOINT_OUT_OF_ORDER_TS 1148 +#define WT_STAT_CONN_CACHE_EVICTION_FAIL_CHECKPOINT_OUT_OF_ORDER_TS 1168 /*! cache: pages walked for eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_WALK 1149 +#define WT_STAT_CONN_CACHE_EVICTION_WALK 1169 /*! cache: pages written from cache */ -#define WT_STAT_CONN_CACHE_WRITE 1150 +#define WT_STAT_CONN_CACHE_WRITE 1170 /*! cache: pages written requiring in-memory restoration */ -#define WT_STAT_CONN_CACHE_WRITE_RESTORE 1151 +#define WT_STAT_CONN_CACHE_WRITE_RESTORE 1171 /*! cache: percentage overhead */ -#define WT_STAT_CONN_CACHE_OVERHEAD 1152 +#define WT_STAT_CONN_CACHE_OVERHEAD 1172 /*! cache: the number of times full update inserted to history store */ -#define WT_STAT_CONN_CACHE_HS_INSERT_FULL_UPDATE 1153 +#define WT_STAT_CONN_CACHE_HS_INSERT_FULL_UPDATE 1173 /*! cache: the number of times reverse modify inserted to history store */ -#define WT_STAT_CONN_CACHE_HS_INSERT_REVERSE_MODIFY 1154 +#define WT_STAT_CONN_CACHE_HS_INSERT_REVERSE_MODIFY 1174 /*! cache: tracked bytes belonging to internal pages in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_INTERNAL 1155 +#define WT_STAT_CONN_CACHE_BYTES_INTERNAL 1175 /*! cache: tracked bytes belonging to leaf pages in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_LEAF 1156 +#define WT_STAT_CONN_CACHE_BYTES_LEAF 1176 /*! cache: tracked dirty bytes in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_DIRTY 1157 +#define WT_STAT_CONN_CACHE_BYTES_DIRTY 1177 /*! cache: tracked dirty pages in the cache */ -#define WT_STAT_CONN_CACHE_PAGES_DIRTY 1158 +#define WT_STAT_CONN_CACHE_PAGES_DIRTY 1178 /*! cache: unmodified pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_CLEAN 1159 +#define WT_STAT_CONN_CACHE_EVICTION_CLEAN 1179 /*! capacity: background fsync file handles considered */ -#define WT_STAT_CONN_FSYNC_ALL_FH_TOTAL 1160 +#define WT_STAT_CONN_FSYNC_ALL_FH_TOTAL 1180 /*! capacity: background fsync file handles synced */ -#define WT_STAT_CONN_FSYNC_ALL_FH 1161 +#define WT_STAT_CONN_FSYNC_ALL_FH 1181 /*! capacity: background fsync time (msecs) */ -#define WT_STAT_CONN_FSYNC_ALL_TIME 1162 +#define WT_STAT_CONN_FSYNC_ALL_TIME 1182 /*! capacity: bytes read */ -#define WT_STAT_CONN_CAPACITY_BYTES_READ 1163 +#define WT_STAT_CONN_CAPACITY_BYTES_READ 1183 /*! capacity: bytes written for checkpoint */ -#define WT_STAT_CONN_CAPACITY_BYTES_CKPT 1164 +#define WT_STAT_CONN_CAPACITY_BYTES_CKPT 1184 /*! capacity: bytes written for eviction */ -#define WT_STAT_CONN_CAPACITY_BYTES_EVICT 1165 +#define WT_STAT_CONN_CAPACITY_BYTES_EVICT 1185 /*! capacity: bytes written for log */ -#define WT_STAT_CONN_CAPACITY_BYTES_LOG 1166 +#define WT_STAT_CONN_CAPACITY_BYTES_LOG 1186 /*! capacity: bytes written total */ -#define WT_STAT_CONN_CAPACITY_BYTES_WRITTEN 1167 +#define WT_STAT_CONN_CAPACITY_BYTES_WRITTEN 1187 /*! capacity: threshold to call fsync */ -#define WT_STAT_CONN_CAPACITY_THRESHOLD 1168 +#define WT_STAT_CONN_CAPACITY_THRESHOLD 1188 /*! capacity: time waiting due to total capacity (usecs) */ -#define WT_STAT_CONN_CAPACITY_TIME_TOTAL 1169 +#define WT_STAT_CONN_CAPACITY_TIME_TOTAL 1189 /*! capacity: time waiting during checkpoint (usecs) */ -#define WT_STAT_CONN_CAPACITY_TIME_CKPT 1170 +#define WT_STAT_CONN_CAPACITY_TIME_CKPT 1190 /*! capacity: time waiting during eviction (usecs) */ -#define WT_STAT_CONN_CAPACITY_TIME_EVICT 1171 +#define WT_STAT_CONN_CAPACITY_TIME_EVICT 1191 /*! capacity: time waiting during logging (usecs) */ -#define WT_STAT_CONN_CAPACITY_TIME_LOG 1172 +#define WT_STAT_CONN_CAPACITY_TIME_LOG 1192 /*! capacity: time waiting during read (usecs) */ -#define WT_STAT_CONN_CAPACITY_TIME_READ 1173 +#define WT_STAT_CONN_CAPACITY_TIME_READ 1193 /*! checkpoint-cleanup: pages added for eviction */ -#define WT_STAT_CONN_CC_PAGES_EVICT 1174 +#define WT_STAT_CONN_CC_PAGES_EVICT 1194 /*! checkpoint-cleanup: pages removed */ -#define WT_STAT_CONN_CC_PAGES_REMOVED 1175 +#define WT_STAT_CONN_CC_PAGES_REMOVED 1195 /*! checkpoint-cleanup: pages skipped during tree walk */ -#define WT_STAT_CONN_CC_PAGES_WALK_SKIPPED 1176 +#define WT_STAT_CONN_CC_PAGES_WALK_SKIPPED 1196 /*! checkpoint-cleanup: pages visited */ -#define WT_STAT_CONN_CC_PAGES_VISITED 1177 +#define WT_STAT_CONN_CC_PAGES_VISITED 1197 /*! connection: auto adjusting condition resets */ -#define WT_STAT_CONN_COND_AUTO_WAIT_RESET 1178 +#define WT_STAT_CONN_COND_AUTO_WAIT_RESET 1198 /*! connection: auto adjusting condition wait calls */ -#define WT_STAT_CONN_COND_AUTO_WAIT 1179 +#define WT_STAT_CONN_COND_AUTO_WAIT 1199 /*! * connection: auto adjusting condition wait raced to update timeout and * skipped updating */ -#define WT_STAT_CONN_COND_AUTO_WAIT_SKIPPED 1180 +#define WT_STAT_CONN_COND_AUTO_WAIT_SKIPPED 1200 /*! connection: detected system time went backwards */ -#define WT_STAT_CONN_TIME_TRAVEL 1181 +#define WT_STAT_CONN_TIME_TRAVEL 1201 /*! connection: files currently open */ -#define WT_STAT_CONN_FILE_OPEN 1182 +#define WT_STAT_CONN_FILE_OPEN 1202 /*! connection: hash bucket array size for data handles */ -#define WT_STAT_CONN_BUCKETS_DH 1183 +#define WT_STAT_CONN_BUCKETS_DH 1203 /*! connection: hash bucket array size general */ -#define WT_STAT_CONN_BUCKETS 1184 +#define WT_STAT_CONN_BUCKETS 1204 /*! connection: memory allocations */ -#define WT_STAT_CONN_MEMORY_ALLOCATION 1185 +#define WT_STAT_CONN_MEMORY_ALLOCATION 1205 /*! connection: memory frees */ -#define WT_STAT_CONN_MEMORY_FREE 1186 +#define WT_STAT_CONN_MEMORY_FREE 1206 /*! connection: memory re-allocations */ -#define WT_STAT_CONN_MEMORY_GROW 1187 +#define WT_STAT_CONN_MEMORY_GROW 1207 /*! connection: pthread mutex condition wait calls */ -#define WT_STAT_CONN_COND_WAIT 1188 +#define WT_STAT_CONN_COND_WAIT 1208 /*! connection: pthread mutex shared lock read-lock calls */ -#define WT_STAT_CONN_RWLOCK_READ 1189 +#define WT_STAT_CONN_RWLOCK_READ 1209 /*! connection: pthread mutex shared lock write-lock calls */ -#define WT_STAT_CONN_RWLOCK_WRITE 1190 +#define WT_STAT_CONN_RWLOCK_WRITE 1210 /*! connection: total fsync I/Os */ -#define WT_STAT_CONN_FSYNC_IO 1191 +#define WT_STAT_CONN_FSYNC_IO 1211 /*! connection: total read I/Os */ -#define WT_STAT_CONN_READ_IO 1192 +#define WT_STAT_CONN_READ_IO 1212 /*! connection: total write I/Os */ -#define WT_STAT_CONN_WRITE_IO 1193 +#define WT_STAT_CONN_WRITE_IO 1213 /*! cursor: Total number of entries skipped by cursor next calls */ -#define WT_STAT_CONN_CURSOR_NEXT_SKIP_TOTAL 1194 +#define WT_STAT_CONN_CURSOR_NEXT_SKIP_TOTAL 1214 /*! cursor: Total number of entries skipped by cursor prev calls */ -#define WT_STAT_CONN_CURSOR_PREV_SKIP_TOTAL 1195 +#define WT_STAT_CONN_CURSOR_PREV_SKIP_TOTAL 1215 /*! * cursor: Total number of entries skipped to position the history store * cursor */ -#define WT_STAT_CONN_CURSOR_SKIP_HS_CUR_POSITION 1196 +#define WT_STAT_CONN_CURSOR_SKIP_HS_CUR_POSITION 1216 /*! * cursor: Total number of times a search near has exited due to prefix * config */ -#define WT_STAT_CONN_CURSOR_SEARCH_NEAR_PREFIX_FAST_PATHS 1197 +#define WT_STAT_CONN_CURSOR_SEARCH_NEAR_PREFIX_FAST_PATHS 1217 /*! cursor: cached cursor count */ -#define WT_STAT_CONN_CURSOR_CACHED_COUNT 1198 +#define WT_STAT_CONN_CURSOR_CACHED_COUNT 1218 /*! cursor: cursor bulk loaded cursor insert calls */ -#define WT_STAT_CONN_CURSOR_INSERT_BULK 1199 +#define WT_STAT_CONN_CURSOR_INSERT_BULK 1219 /*! cursor: cursor close calls that result in cache */ -#define WT_STAT_CONN_CURSOR_CACHE 1200 +#define WT_STAT_CONN_CURSOR_CACHE 1220 /*! cursor: cursor create calls */ -#define WT_STAT_CONN_CURSOR_CREATE 1201 +#define WT_STAT_CONN_CURSOR_CREATE 1221 /*! cursor: cursor insert calls */ -#define WT_STAT_CONN_CURSOR_INSERT 1202 +#define WT_STAT_CONN_CURSOR_INSERT 1222 /*! cursor: cursor insert key and value bytes */ -#define WT_STAT_CONN_CURSOR_INSERT_BYTES 1203 +#define WT_STAT_CONN_CURSOR_INSERT_BYTES 1223 /*! cursor: cursor modify calls */ -#define WT_STAT_CONN_CURSOR_MODIFY 1204 +#define WT_STAT_CONN_CURSOR_MODIFY 1224 /*! cursor: cursor modify key and value bytes affected */ -#define WT_STAT_CONN_CURSOR_MODIFY_BYTES 1205 +#define WT_STAT_CONN_CURSOR_MODIFY_BYTES 1225 /*! cursor: cursor modify value bytes modified */ -#define WT_STAT_CONN_CURSOR_MODIFY_BYTES_TOUCH 1206 +#define WT_STAT_CONN_CURSOR_MODIFY_BYTES_TOUCH 1226 /*! cursor: cursor next calls */ -#define WT_STAT_CONN_CURSOR_NEXT 1207 +#define WT_STAT_CONN_CURSOR_NEXT 1227 /*! * cursor: cursor next calls that skip due to a globally visible history * store tombstone */ -#define WT_STAT_CONN_CURSOR_NEXT_HS_TOMBSTONE 1208 +#define WT_STAT_CONN_CURSOR_NEXT_HS_TOMBSTONE 1228 /*! * cursor: cursor next calls that skip greater than or equal to 100 * entries */ -#define WT_STAT_CONN_CURSOR_NEXT_SKIP_GE_100 1209 +#define WT_STAT_CONN_CURSOR_NEXT_SKIP_GE_100 1229 /*! cursor: cursor next calls that skip less than 100 entries */ -#define WT_STAT_CONN_CURSOR_NEXT_SKIP_LT_100 1210 +#define WT_STAT_CONN_CURSOR_NEXT_SKIP_LT_100 1230 /*! cursor: cursor operation restarted */ -#define WT_STAT_CONN_CURSOR_RESTART 1211 +#define WT_STAT_CONN_CURSOR_RESTART 1231 /*! cursor: cursor prev calls */ -#define WT_STAT_CONN_CURSOR_PREV 1212 +#define WT_STAT_CONN_CURSOR_PREV 1232 /*! * cursor: cursor prev calls that skip due to a globally visible history * store tombstone */ -#define WT_STAT_CONN_CURSOR_PREV_HS_TOMBSTONE 1213 +#define WT_STAT_CONN_CURSOR_PREV_HS_TOMBSTONE 1233 /*! * cursor: cursor prev calls that skip greater than or equal to 100 * entries */ -#define WT_STAT_CONN_CURSOR_PREV_SKIP_GE_100 1214 +#define WT_STAT_CONN_CURSOR_PREV_SKIP_GE_100 1234 /*! cursor: cursor prev calls that skip less than 100 entries */ -#define WT_STAT_CONN_CURSOR_PREV_SKIP_LT_100 1215 +#define WT_STAT_CONN_CURSOR_PREV_SKIP_LT_100 1235 /*! cursor: cursor remove calls */ -#define WT_STAT_CONN_CURSOR_REMOVE 1216 +#define WT_STAT_CONN_CURSOR_REMOVE 1236 /*! cursor: cursor remove key bytes removed */ -#define WT_STAT_CONN_CURSOR_REMOVE_BYTES 1217 +#define WT_STAT_CONN_CURSOR_REMOVE_BYTES 1237 /*! cursor: cursor reserve calls */ -#define WT_STAT_CONN_CURSOR_RESERVE 1218 +#define WT_STAT_CONN_CURSOR_RESERVE 1238 /*! cursor: cursor reset calls */ -#define WT_STAT_CONN_CURSOR_RESET 1219 +#define WT_STAT_CONN_CURSOR_RESET 1239 /*! cursor: cursor search calls */ -#define WT_STAT_CONN_CURSOR_SEARCH 1220 +#define WT_STAT_CONN_CURSOR_SEARCH 1240 /*! cursor: cursor search history store calls */ -#define WT_STAT_CONN_CURSOR_SEARCH_HS 1221 +#define WT_STAT_CONN_CURSOR_SEARCH_HS 1241 /*! cursor: cursor search near calls */ -#define WT_STAT_CONN_CURSOR_SEARCH_NEAR 1222 +#define WT_STAT_CONN_CURSOR_SEARCH_NEAR 1242 /*! cursor: cursor sweep buckets */ -#define WT_STAT_CONN_CURSOR_SWEEP_BUCKETS 1223 +#define WT_STAT_CONN_CURSOR_SWEEP_BUCKETS 1243 /*! cursor: cursor sweep cursors closed */ -#define WT_STAT_CONN_CURSOR_SWEEP_CLOSED 1224 +#define WT_STAT_CONN_CURSOR_SWEEP_CLOSED 1244 /*! cursor: cursor sweep cursors examined */ -#define WT_STAT_CONN_CURSOR_SWEEP_EXAMINED 1225 +#define WT_STAT_CONN_CURSOR_SWEEP_EXAMINED 1245 /*! cursor: cursor sweeps */ -#define WT_STAT_CONN_CURSOR_SWEEP 1226 +#define WT_STAT_CONN_CURSOR_SWEEP 1246 /*! cursor: cursor truncate calls */ -#define WT_STAT_CONN_CURSOR_TRUNCATE 1227 +#define WT_STAT_CONN_CURSOR_TRUNCATE 1247 /*! cursor: cursor update calls */ -#define WT_STAT_CONN_CURSOR_UPDATE 1228 +#define WT_STAT_CONN_CURSOR_UPDATE 1248 /*! cursor: cursor update key and value bytes */ -#define WT_STAT_CONN_CURSOR_UPDATE_BYTES 1229 +#define WT_STAT_CONN_CURSOR_UPDATE_BYTES 1249 /*! cursor: cursor update value size change */ -#define WT_STAT_CONN_CURSOR_UPDATE_BYTES_CHANGED 1230 +#define WT_STAT_CONN_CURSOR_UPDATE_BYTES_CHANGED 1250 /*! cursor: cursors reused from cache */ -#define WT_STAT_CONN_CURSOR_REOPEN 1231 +#define WT_STAT_CONN_CURSOR_REOPEN 1251 /*! cursor: open cursor count */ -#define WT_STAT_CONN_CURSOR_OPEN_COUNT 1232 +#define WT_STAT_CONN_CURSOR_OPEN_COUNT 1252 /*! data-handle: connection data handle size */ -#define WT_STAT_CONN_DH_CONN_HANDLE_SIZE 1233 +#define WT_STAT_CONN_DH_CONN_HANDLE_SIZE 1253 /*! data-handle: connection data handles currently active */ -#define WT_STAT_CONN_DH_CONN_HANDLE_COUNT 1234 +#define WT_STAT_CONN_DH_CONN_HANDLE_COUNT 1254 /*! data-handle: connection sweep candidate became referenced */ -#define WT_STAT_CONN_DH_SWEEP_REF 1235 +#define WT_STAT_CONN_DH_SWEEP_REF 1255 /*! data-handle: connection sweep dhandles closed */ -#define WT_STAT_CONN_DH_SWEEP_CLOSE 1236 +#define WT_STAT_CONN_DH_SWEEP_CLOSE 1256 /*! data-handle: connection sweep dhandles removed from hash list */ -#define WT_STAT_CONN_DH_SWEEP_REMOVE 1237 +#define WT_STAT_CONN_DH_SWEEP_REMOVE 1257 /*! data-handle: connection sweep time-of-death sets */ -#define WT_STAT_CONN_DH_SWEEP_TOD 1238 +#define WT_STAT_CONN_DH_SWEEP_TOD 1258 /*! data-handle: connection sweeps */ -#define WT_STAT_CONN_DH_SWEEPS 1239 +#define WT_STAT_CONN_DH_SWEEPS 1259 /*! * data-handle: connection sweeps skipped due to checkpoint gathering * handles */ -#define WT_STAT_CONN_DH_SWEEP_SKIP_CKPT 1240 +#define WT_STAT_CONN_DH_SWEEP_SKIP_CKPT 1260 /*! data-handle: session dhandles swept */ -#define WT_STAT_CONN_DH_SESSION_HANDLES 1241 +#define WT_STAT_CONN_DH_SESSION_HANDLES 1261 /*! data-handle: session sweep attempts */ -#define WT_STAT_CONN_DH_SESSION_SWEEPS 1242 +#define WT_STAT_CONN_DH_SESSION_SWEEPS 1262 /*! lock: checkpoint lock acquisitions */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_COUNT 1243 +#define WT_STAT_CONN_LOCK_CHECKPOINT_COUNT 1263 /*! lock: checkpoint lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_APPLICATION 1244 +#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_APPLICATION 1264 /*! lock: checkpoint lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_INTERNAL 1245 +#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_INTERNAL 1265 /*! lock: dhandle lock application thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_APPLICATION 1246 +#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_APPLICATION 1266 /*! lock: dhandle lock internal thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_INTERNAL 1247 +#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_INTERNAL 1267 /*! lock: dhandle read lock acquisitions */ -#define WT_STAT_CONN_LOCK_DHANDLE_READ_COUNT 1248 +#define WT_STAT_CONN_LOCK_DHANDLE_READ_COUNT 1268 /*! lock: dhandle write lock acquisitions */ -#define WT_STAT_CONN_LOCK_DHANDLE_WRITE_COUNT 1249 +#define WT_STAT_CONN_LOCK_DHANDLE_WRITE_COUNT 1269 /*! * lock: durable timestamp queue lock application thread time waiting * (usecs) */ -#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WAIT_APPLICATION 1250 +#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WAIT_APPLICATION 1270 /*! * lock: durable timestamp queue lock internal thread time waiting * (usecs) */ -#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WAIT_INTERNAL 1251 +#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WAIT_INTERNAL 1271 /*! lock: durable timestamp queue read lock acquisitions */ -#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_READ_COUNT 1252 +#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_READ_COUNT 1272 /*! lock: durable timestamp queue write lock acquisitions */ -#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WRITE_COUNT 1253 +#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WRITE_COUNT 1273 /*! lock: metadata lock acquisitions */ -#define WT_STAT_CONN_LOCK_METADATA_COUNT 1254 +#define WT_STAT_CONN_LOCK_METADATA_COUNT 1274 /*! lock: metadata lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_METADATA_WAIT_APPLICATION 1255 +#define WT_STAT_CONN_LOCK_METADATA_WAIT_APPLICATION 1275 /*! lock: metadata lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_METADATA_WAIT_INTERNAL 1256 +#define WT_STAT_CONN_LOCK_METADATA_WAIT_INTERNAL 1276 /*! * lock: read timestamp queue lock application thread time waiting * (usecs) */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_APPLICATION 1257 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_APPLICATION 1277 /*! lock: read timestamp queue lock internal thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_INTERNAL 1258 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_INTERNAL 1278 /*! lock: read timestamp queue read lock acquisitions */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_READ_COUNT 1259 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_READ_COUNT 1279 /*! lock: read timestamp queue write lock acquisitions */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WRITE_COUNT 1260 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WRITE_COUNT 1280 /*! lock: schema lock acquisitions */ -#define WT_STAT_CONN_LOCK_SCHEMA_COUNT 1261 +#define WT_STAT_CONN_LOCK_SCHEMA_COUNT 1281 /*! lock: schema lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_APPLICATION 1262 +#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_APPLICATION 1282 /*! lock: schema lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_INTERNAL 1263 +#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_INTERNAL 1283 /*! * lock: table lock application thread time waiting for the table lock * (usecs) */ -#define WT_STAT_CONN_LOCK_TABLE_WAIT_APPLICATION 1264 +#define WT_STAT_CONN_LOCK_TABLE_WAIT_APPLICATION 1284 /*! * lock: table lock internal thread time waiting for the table lock * (usecs) */ -#define WT_STAT_CONN_LOCK_TABLE_WAIT_INTERNAL 1265 +#define WT_STAT_CONN_LOCK_TABLE_WAIT_INTERNAL 1285 /*! lock: table read lock acquisitions */ -#define WT_STAT_CONN_LOCK_TABLE_READ_COUNT 1266 +#define WT_STAT_CONN_LOCK_TABLE_READ_COUNT 1286 /*! lock: table write lock acquisitions */ -#define WT_STAT_CONN_LOCK_TABLE_WRITE_COUNT 1267 +#define WT_STAT_CONN_LOCK_TABLE_WRITE_COUNT 1287 /*! lock: txn global lock application thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_APPLICATION 1268 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_APPLICATION 1288 /*! lock: txn global lock internal thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_INTERNAL 1269 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_INTERNAL 1289 /*! lock: txn global read lock acquisitions */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_READ_COUNT 1270 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_READ_COUNT 1290 /*! lock: txn global write lock acquisitions */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WRITE_COUNT 1271 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WRITE_COUNT 1291 /*! log: busy returns attempting to switch slots */ -#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1272 +#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1292 /*! log: force archive time sleeping (usecs) */ -#define WT_STAT_CONN_LOG_FORCE_ARCHIVE_SLEEP 1273 +#define WT_STAT_CONN_LOG_FORCE_ARCHIVE_SLEEP 1293 /*! log: log bytes of payload data */ -#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1274 +#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1294 /*! log: log bytes written */ -#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1275 +#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1295 /*! log: log files manually zero-filled */ -#define WT_STAT_CONN_LOG_ZERO_FILLS 1276 +#define WT_STAT_CONN_LOG_ZERO_FILLS 1296 /*! log: log flush operations */ -#define WT_STAT_CONN_LOG_FLUSH 1277 +#define WT_STAT_CONN_LOG_FLUSH 1297 /*! log: log force write operations */ -#define WT_STAT_CONN_LOG_FORCE_WRITE 1278 +#define WT_STAT_CONN_LOG_FORCE_WRITE 1298 /*! log: log force write operations skipped */ -#define WT_STAT_CONN_LOG_FORCE_WRITE_SKIP 1279 +#define WT_STAT_CONN_LOG_FORCE_WRITE_SKIP 1299 /*! log: log records compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1280 +#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1300 /*! log: log records not compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1281 +#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1301 /*! log: log records too small to compress */ -#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1282 +#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1302 /*! log: log release advances write LSN */ -#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1283 +#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1303 /*! log: log scan operations */ -#define WT_STAT_CONN_LOG_SCANS 1284 +#define WT_STAT_CONN_LOG_SCANS 1304 /*! log: log scan records requiring two reads */ -#define WT_STAT_CONN_LOG_SCAN_REREADS 1285 +#define WT_STAT_CONN_LOG_SCAN_REREADS 1305 /*! log: log server thread advances write LSN */ -#define WT_STAT_CONN_LOG_WRITE_LSN 1286 +#define WT_STAT_CONN_LOG_WRITE_LSN 1306 /*! log: log server thread write LSN walk skipped */ -#define WT_STAT_CONN_LOG_WRITE_LSN_SKIP 1287 +#define WT_STAT_CONN_LOG_WRITE_LSN_SKIP 1307 /*! log: log sync operations */ -#define WT_STAT_CONN_LOG_SYNC 1288 +#define WT_STAT_CONN_LOG_SYNC 1308 /*! log: log sync time duration (usecs) */ -#define WT_STAT_CONN_LOG_SYNC_DURATION 1289 +#define WT_STAT_CONN_LOG_SYNC_DURATION 1309 /*! log: log sync_dir operations */ -#define WT_STAT_CONN_LOG_SYNC_DIR 1290 +#define WT_STAT_CONN_LOG_SYNC_DIR 1310 /*! log: log sync_dir time duration (usecs) */ -#define WT_STAT_CONN_LOG_SYNC_DIR_DURATION 1291 +#define WT_STAT_CONN_LOG_SYNC_DIR_DURATION 1311 /*! log: log write operations */ -#define WT_STAT_CONN_LOG_WRITES 1292 +#define WT_STAT_CONN_LOG_WRITES 1312 /*! log: logging bytes consolidated */ -#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1293 +#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1313 /*! log: maximum log file size */ -#define WT_STAT_CONN_LOG_MAX_FILESIZE 1294 +#define WT_STAT_CONN_LOG_MAX_FILESIZE 1314 /*! log: number of pre-allocated log files to create */ -#define WT_STAT_CONN_LOG_PREALLOC_MAX 1295 +#define WT_STAT_CONN_LOG_PREALLOC_MAX 1315 /*! log: pre-allocated log files not ready and missed */ -#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1296 +#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1316 /*! log: pre-allocated log files prepared */ -#define WT_STAT_CONN_LOG_PREALLOC_FILES 1297 +#define WT_STAT_CONN_LOG_PREALLOC_FILES 1317 /*! log: pre-allocated log files used */ -#define WT_STAT_CONN_LOG_PREALLOC_USED 1298 +#define WT_STAT_CONN_LOG_PREALLOC_USED 1318 /*! log: records processed by log scan */ -#define WT_STAT_CONN_LOG_SCAN_RECORDS 1299 +#define WT_STAT_CONN_LOG_SCAN_RECORDS 1319 /*! log: slot close lost race */ -#define WT_STAT_CONN_LOG_SLOT_CLOSE_RACE 1300 +#define WT_STAT_CONN_LOG_SLOT_CLOSE_RACE 1320 /*! log: slot close unbuffered waits */ -#define WT_STAT_CONN_LOG_SLOT_CLOSE_UNBUF 1301 +#define WT_STAT_CONN_LOG_SLOT_CLOSE_UNBUF 1321 /*! log: slot closures */ -#define WT_STAT_CONN_LOG_SLOT_CLOSES 1302 +#define WT_STAT_CONN_LOG_SLOT_CLOSES 1322 /*! log: slot join atomic update races */ -#define WT_STAT_CONN_LOG_SLOT_RACES 1303 +#define WT_STAT_CONN_LOG_SLOT_RACES 1323 /*! log: slot join calls atomic updates raced */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_RACE 1304 +#define WT_STAT_CONN_LOG_SLOT_YIELD_RACE 1324 /*! log: slot join calls did not yield */ -#define WT_STAT_CONN_LOG_SLOT_IMMEDIATE 1305 +#define WT_STAT_CONN_LOG_SLOT_IMMEDIATE 1325 /*! log: slot join calls found active slot closed */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_CLOSE 1306 +#define WT_STAT_CONN_LOG_SLOT_YIELD_CLOSE 1326 /*! log: slot join calls slept */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_SLEEP 1307 +#define WT_STAT_CONN_LOG_SLOT_YIELD_SLEEP 1327 /*! log: slot join calls yielded */ -#define WT_STAT_CONN_LOG_SLOT_YIELD 1308 +#define WT_STAT_CONN_LOG_SLOT_YIELD 1328 /*! log: slot join found active slot closed */ -#define WT_STAT_CONN_LOG_SLOT_ACTIVE_CLOSED 1309 +#define WT_STAT_CONN_LOG_SLOT_ACTIVE_CLOSED 1329 /*! log: slot joins yield time (usecs) */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_DURATION 1310 +#define WT_STAT_CONN_LOG_SLOT_YIELD_DURATION 1330 /*! log: slot transitions unable to find free slot */ -#define WT_STAT_CONN_LOG_SLOT_NO_FREE_SLOTS 1311 +#define WT_STAT_CONN_LOG_SLOT_NO_FREE_SLOTS 1331 /*! log: slot unbuffered writes */ -#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1312 +#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1332 /*! log: total in-memory size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_MEM 1313 +#define WT_STAT_CONN_LOG_COMPRESS_MEM 1333 /*! log: total log buffer size */ -#define WT_STAT_CONN_LOG_BUFFER_SIZE 1314 +#define WT_STAT_CONN_LOG_BUFFER_SIZE 1334 /*! log: total size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_LEN 1315 +#define WT_STAT_CONN_LOG_COMPRESS_LEN 1335 /*! log: written slots coalesced */ -#define WT_STAT_CONN_LOG_SLOT_COALESCED 1316 +#define WT_STAT_CONN_LOG_SLOT_COALESCED 1336 /*! log: yields waiting for previous log file close */ -#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1317 +#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1337 /*! perf: file system read latency histogram (bucket 1) - 10-49ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT50 1318 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT50 1338 /*! perf: file system read latency histogram (bucket 2) - 50-99ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT100 1319 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT100 1339 /*! perf: file system read latency histogram (bucket 3) - 100-249ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT250 1320 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT250 1340 /*! perf: file system read latency histogram (bucket 4) - 250-499ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT500 1321 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT500 1341 /*! perf: file system read latency histogram (bucket 5) - 500-999ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT1000 1322 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT1000 1342 /*! perf: file system read latency histogram (bucket 6) - 1000ms+ */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_GT1000 1323 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_GT1000 1343 /*! perf: file system write latency histogram (bucket 1) - 10-49ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT50 1324 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT50 1344 /*! perf: file system write latency histogram (bucket 2) - 50-99ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT100 1325 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT100 1345 /*! perf: file system write latency histogram (bucket 3) - 100-249ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT250 1326 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT250 1346 /*! perf: file system write latency histogram (bucket 4) - 250-499ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT500 1327 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT500 1347 /*! perf: file system write latency histogram (bucket 5) - 500-999ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT1000 1328 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT1000 1348 /*! perf: file system write latency histogram (bucket 6) - 1000ms+ */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_GT1000 1329 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_GT1000 1349 /*! perf: operation read latency histogram (bucket 1) - 100-249us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT250 1330 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT250 1350 /*! perf: operation read latency histogram (bucket 2) - 250-499us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT500 1331 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT500 1351 /*! perf: operation read latency histogram (bucket 3) - 500-999us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT1000 1332 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT1000 1352 /*! perf: operation read latency histogram (bucket 4) - 1000-9999us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT10000 1333 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT10000 1353 /*! perf: operation read latency histogram (bucket 5) - 10000us+ */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_GT10000 1334 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_GT10000 1354 /*! perf: operation write latency histogram (bucket 1) - 100-249us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT250 1335 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT250 1355 /*! perf: operation write latency histogram (bucket 2) - 250-499us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT500 1336 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT500 1356 /*! perf: operation write latency histogram (bucket 3) - 500-999us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT1000 1337 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT1000 1357 /*! perf: operation write latency histogram (bucket 4) - 1000-9999us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT10000 1338 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT10000 1358 /*! perf: operation write latency histogram (bucket 5) - 10000us+ */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_GT10000 1339 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_GT10000 1359 /*! reconciliation: approximate byte size of timestamps in pages written */ -#define WT_STAT_CONN_REC_TIME_WINDOW_BYTES_TS 1340 +#define WT_STAT_CONN_REC_TIME_WINDOW_BYTES_TS 1360 /*! * reconciliation: approximate byte size of transaction IDs in pages * written */ -#define WT_STAT_CONN_REC_TIME_WINDOW_BYTES_TXN 1341 +#define WT_STAT_CONN_REC_TIME_WINDOW_BYTES_TXN 1361 /*! reconciliation: fast-path pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1342 -/*! reconciliation: internal-page overflow keys */ -#define WT_STAT_CONN_REC_OVERFLOW_KEY_INTERNAL 1343 +#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1362 /*! reconciliation: leaf-page overflow keys */ -#define WT_STAT_CONN_REC_OVERFLOW_KEY_LEAF 1344 +#define WT_STAT_CONN_REC_OVERFLOW_KEY_LEAF 1363 /*! reconciliation: maximum seconds spent in a reconciliation call */ -#define WT_STAT_CONN_REC_MAXIMUM_SECONDS 1345 +#define WT_STAT_CONN_REC_MAXIMUM_SECONDS 1364 /*! reconciliation: page reconciliation calls */ -#define WT_STAT_CONN_REC_PAGES 1346 +#define WT_STAT_CONN_REC_PAGES 1365 /*! reconciliation: page reconciliation calls for eviction */ -#define WT_STAT_CONN_REC_PAGES_EVICTION 1347 +#define WT_STAT_CONN_REC_PAGES_EVICTION 1366 /*! * reconciliation: page reconciliation calls that resulted in values with * prepared transaction metadata */ -#define WT_STAT_CONN_REC_PAGES_WITH_PREPARE 1348 +#define WT_STAT_CONN_REC_PAGES_WITH_PREPARE 1367 /*! * reconciliation: page reconciliation calls that resulted in values with * timestamps */ -#define WT_STAT_CONN_REC_PAGES_WITH_TS 1349 +#define WT_STAT_CONN_REC_PAGES_WITH_TS 1368 /*! * reconciliation: page reconciliation calls that resulted in values with * transaction ids */ -#define WT_STAT_CONN_REC_PAGES_WITH_TXN 1350 +#define WT_STAT_CONN_REC_PAGES_WITH_TXN 1369 /*! reconciliation: pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE 1351 +#define WT_STAT_CONN_REC_PAGE_DELETE 1370 /*! * reconciliation: pages written including an aggregated newest start * durable timestamp */ -#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_START_DURABLE_TS 1352 +#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_START_DURABLE_TS 1371 /*! * reconciliation: pages written including an aggregated newest stop * durable timestamp */ -#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_DURABLE_TS 1353 +#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_DURABLE_TS 1372 /*! * reconciliation: pages written including an aggregated newest stop * timestamp */ -#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_TS 1354 +#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_TS 1373 /*! * reconciliation: pages written including an aggregated newest stop * transaction ID */ -#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_TXN 1355 +#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_TXN 1374 /*! * reconciliation: pages written including an aggregated newest * transaction ID */ -#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_TXN 1356 +#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_TXN 1375 /*! * reconciliation: pages written including an aggregated oldest start * timestamp */ -#define WT_STAT_CONN_REC_TIME_AGGR_OLDEST_START_TS 1357 +#define WT_STAT_CONN_REC_TIME_AGGR_OLDEST_START_TS 1376 /*! reconciliation: pages written including an aggregated prepare */ -#define WT_STAT_CONN_REC_TIME_AGGR_PREPARED 1358 +#define WT_STAT_CONN_REC_TIME_AGGR_PREPARED 1377 /*! reconciliation: pages written including at least one prepare state */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_PREPARED 1359 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_PREPARED 1378 /*! * reconciliation: pages written including at least one start durable * timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_DURABLE_START_TS 1360 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_DURABLE_START_TS 1379 /*! reconciliation: pages written including at least one start timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_START_TS 1361 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_START_TS 1380 /*! * reconciliation: pages written including at least one start transaction * ID */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_START_TXN 1362 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_START_TXN 1381 /*! * reconciliation: pages written including at least one stop durable * timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_DURABLE_STOP_TS 1363 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_DURABLE_STOP_TS 1382 /*! reconciliation: pages written including at least one stop timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_STOP_TS 1364 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_STOP_TS 1383 /*! * reconciliation: pages written including at least one stop transaction * ID */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_STOP_TXN 1365 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_STOP_TXN 1384 /*! reconciliation: records written including a prepare state */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PREPARED 1366 +#define WT_STAT_CONN_REC_TIME_WINDOW_PREPARED 1385 /*! reconciliation: records written including a start durable timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_DURABLE_START_TS 1367 +#define WT_STAT_CONN_REC_TIME_WINDOW_DURABLE_START_TS 1386 /*! reconciliation: records written including a start timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_START_TS 1368 +#define WT_STAT_CONN_REC_TIME_WINDOW_START_TS 1387 /*! reconciliation: records written including a start transaction ID */ -#define WT_STAT_CONN_REC_TIME_WINDOW_START_TXN 1369 +#define WT_STAT_CONN_REC_TIME_WINDOW_START_TXN 1388 /*! reconciliation: records written including a stop durable timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_DURABLE_STOP_TS 1370 +#define WT_STAT_CONN_REC_TIME_WINDOW_DURABLE_STOP_TS 1389 /*! reconciliation: records written including a stop timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_STOP_TS 1371 +#define WT_STAT_CONN_REC_TIME_WINDOW_STOP_TS 1390 /*! reconciliation: records written including a stop transaction ID */ -#define WT_STAT_CONN_REC_TIME_WINDOW_STOP_TXN 1372 +#define WT_STAT_CONN_REC_TIME_WINDOW_STOP_TXN 1391 /*! reconciliation: split bytes currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1373 +#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1392 /*! reconciliation: split objects currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1374 +#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1393 /*! session: attempts to remove a local object and the object is in use */ -#define WT_STAT_CONN_LOCAL_OBJECTS_INUSE 1375 +#define WT_STAT_CONN_LOCAL_OBJECTS_INUSE 1394 /*! session: flush_tier operation calls */ -#define WT_STAT_CONN_FLUSH_TIER 1376 +#define WT_STAT_CONN_FLUSH_TIER 1395 /*! session: local objects removed */ -#define WT_STAT_CONN_LOCAL_OBJECTS_REMOVED 1377 +#define WT_STAT_CONN_LOCAL_OBJECTS_REMOVED 1396 /*! session: open session count */ -#define WT_STAT_CONN_SESSION_OPEN 1378 +#define WT_STAT_CONN_SESSION_OPEN 1397 /*! session: session query timestamp calls */ -#define WT_STAT_CONN_SESSION_QUERY_TS 1379 +#define WT_STAT_CONN_SESSION_QUERY_TS 1398 /*! session: table alter failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_FAIL 1380 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_FAIL 1399 /*! session: table alter successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_SUCCESS 1381 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_SUCCESS 1400 /*! session: table alter triggering checkpoint calls */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_TRIGGER_CHECKPOINT 1382 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_TRIGGER_CHECKPOINT 1401 /*! session: table alter unchanged and skipped */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_SKIP 1383 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_SKIP 1402 /*! session: table compact failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_FAIL 1384 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_FAIL 1403 /*! session: table compact failed calls due to cache pressure */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_FAIL_CACHE_PRESSURE 1385 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_FAIL_CACHE_PRESSURE 1404 /*! session: table compact running */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_RUNNING 1386 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_RUNNING 1405 /*! session: table compact skipped as process would not reduce file size */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SKIPPED 1387 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SKIPPED 1406 /*! session: table compact successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SUCCESS 1388 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SUCCESS 1407 /*! session: table compact timeout */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_TIMEOUT 1389 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_TIMEOUT 1408 /*! session: table create failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_CREATE_FAIL 1390 +#define WT_STAT_CONN_SESSION_TABLE_CREATE_FAIL 1409 /*! session: table create successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_CREATE_SUCCESS 1391 +#define WT_STAT_CONN_SESSION_TABLE_CREATE_SUCCESS 1410 /*! session: table drop failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_DROP_FAIL 1392 +#define WT_STAT_CONN_SESSION_TABLE_DROP_FAIL 1411 /*! session: table drop successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_DROP_SUCCESS 1393 +#define WT_STAT_CONN_SESSION_TABLE_DROP_SUCCESS 1412 /*! session: table rename failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1394 +#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1413 /*! session: table rename successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1395 +#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1414 /*! session: table salvage failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1396 +#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1415 /*! session: table salvage successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1397 +#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1416 /*! session: table truncate failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1398 +#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1417 /*! session: table truncate successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1399 +#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1418 /*! session: table verify failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1400 +#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1419 /*! session: table verify successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1401 +#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1420 /*! session: tiered operations dequeued and processed */ -#define WT_STAT_CONN_TIERED_WORK_UNITS_DEQUEUED 1402 +#define WT_STAT_CONN_TIERED_WORK_UNITS_DEQUEUED 1421 /*! session: tiered operations scheduled */ -#define WT_STAT_CONN_TIERED_WORK_UNITS_CREATED 1403 +#define WT_STAT_CONN_TIERED_WORK_UNITS_CREATED 1422 /*! session: tiered storage local retention time (secs) */ -#define WT_STAT_CONN_TIERED_RETENTION 1404 +#define WT_STAT_CONN_TIERED_RETENTION 1423 /*! session: tiered storage object size */ -#define WT_STAT_CONN_TIERED_OBJECT_SIZE 1405 +#define WT_STAT_CONN_TIERED_OBJECT_SIZE 1424 /*! thread-state: active filesystem fsync calls */ -#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1406 +#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1425 /*! thread-state: active filesystem read calls */ -#define WT_STAT_CONN_THREAD_READ_ACTIVE 1407 +#define WT_STAT_CONN_THREAD_READ_ACTIVE 1426 /*! thread-state: active filesystem write calls */ -#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1408 +#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1427 /*! thread-yield: application thread time evicting (usecs) */ -#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1409 +#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1428 /*! thread-yield: application thread time waiting for cache (usecs) */ -#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1410 +#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1429 /*! * thread-yield: connection close blocked waiting for transaction state * stabilization */ -#define WT_STAT_CONN_TXN_RELEASE_BLOCKED 1411 +#define WT_STAT_CONN_TXN_RELEASE_BLOCKED 1430 /*! thread-yield: connection close yielded for lsm manager shutdown */ -#define WT_STAT_CONN_CONN_CLOSE_BLOCKED_LSM 1412 +#define WT_STAT_CONN_CONN_CLOSE_BLOCKED_LSM 1431 /*! thread-yield: data handle lock yielded */ -#define WT_STAT_CONN_DHANDLE_LOCK_BLOCKED 1413 +#define WT_STAT_CONN_DHANDLE_LOCK_BLOCKED 1432 /*! * thread-yield: get reference for page index and slot time sleeping * (usecs) */ -#define WT_STAT_CONN_PAGE_INDEX_SLOT_REF_BLOCKED 1414 +#define WT_STAT_CONN_PAGE_INDEX_SLOT_REF_BLOCKED 1433 /*! thread-yield: page access yielded due to prepare state change */ -#define WT_STAT_CONN_PREPARED_TRANSITION_BLOCKED_PAGE 1415 +#define WT_STAT_CONN_PREPARED_TRANSITION_BLOCKED_PAGE 1434 /*! thread-yield: page acquire busy blocked */ -#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1416 +#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1435 /*! thread-yield: page acquire eviction blocked */ -#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1417 +#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1436 /*! thread-yield: page acquire locked blocked */ -#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1418 +#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1437 /*! thread-yield: page acquire read blocked */ -#define WT_STAT_CONN_PAGE_READ_BLOCKED 1419 +#define WT_STAT_CONN_PAGE_READ_BLOCKED 1438 /*! thread-yield: page acquire time sleeping (usecs) */ -#define WT_STAT_CONN_PAGE_SLEEP 1420 +#define WT_STAT_CONN_PAGE_SLEEP 1439 /*! * thread-yield: page delete rollback time sleeping for state change * (usecs) */ -#define WT_STAT_CONN_PAGE_DEL_ROLLBACK_BLOCKED 1421 +#define WT_STAT_CONN_PAGE_DEL_ROLLBACK_BLOCKED 1440 /*! thread-yield: page reconciliation yielded due to child modification */ -#define WT_STAT_CONN_CHILD_MODIFY_BLOCKED_PAGE 1422 +#define WT_STAT_CONN_CHILD_MODIFY_BLOCKED_PAGE 1441 /*! transaction: Number of prepared updates */ -#define WT_STAT_CONN_TXN_PREPARED_UPDATES 1423 +#define WT_STAT_CONN_TXN_PREPARED_UPDATES 1442 /*! transaction: Number of prepared updates committed */ -#define WT_STAT_CONN_TXN_PREPARED_UPDATES_COMMITTED 1424 +#define WT_STAT_CONN_TXN_PREPARED_UPDATES_COMMITTED 1443 /*! transaction: Number of prepared updates repeated on the same key */ -#define WT_STAT_CONN_TXN_PREPARED_UPDATES_KEY_REPEATED 1425 +#define WT_STAT_CONN_TXN_PREPARED_UPDATES_KEY_REPEATED 1444 /*! transaction: Number of prepared updates rolled back */ -#define WT_STAT_CONN_TXN_PREPARED_UPDATES_ROLLEDBACK 1426 +#define WT_STAT_CONN_TXN_PREPARED_UPDATES_ROLLEDBACK 1445 /*! transaction: prepared transactions */ -#define WT_STAT_CONN_TXN_PREPARE 1427 +#define WT_STAT_CONN_TXN_PREPARE 1446 /*! transaction: prepared transactions committed */ -#define WT_STAT_CONN_TXN_PREPARE_COMMIT 1428 +#define WT_STAT_CONN_TXN_PREPARE_COMMIT 1447 /*! transaction: prepared transactions currently active */ -#define WT_STAT_CONN_TXN_PREPARE_ACTIVE 1429 +#define WT_STAT_CONN_TXN_PREPARE_ACTIVE 1448 /*! transaction: prepared transactions rolled back */ -#define WT_STAT_CONN_TXN_PREPARE_ROLLBACK 1430 +#define WT_STAT_CONN_TXN_PREPARE_ROLLBACK 1449 /*! * transaction: prepared transactions rolled back and do not remove the * history store entry */ -#define WT_STAT_CONN_TXN_PREPARE_ROLLBACK_DO_NOT_REMOVE_HS_UPDATE 1431 +#define WT_STAT_CONN_TXN_PREPARE_ROLLBACK_DO_NOT_REMOVE_HS_UPDATE 1450 /*! * transaction: prepared transactions rolled back and fix the history * store entry with checkpoint reserved transaction id */ -#define WT_STAT_CONN_TXN_PREPARE_ROLLBACK_FIX_HS_UPDATE_WITH_CKPT_RESERVED_TXNID 1432 +#define WT_STAT_CONN_TXN_PREPARE_ROLLBACK_FIX_HS_UPDATE_WITH_CKPT_RESERVED_TXNID 1451 /*! transaction: query timestamp calls */ -#define WT_STAT_CONN_TXN_QUERY_TS 1433 +#define WT_STAT_CONN_TXN_QUERY_TS 1452 /*! transaction: race to read prepared update retry */ -#define WT_STAT_CONN_TXN_READ_RACE_PREPARE_UPDATE 1434 +#define WT_STAT_CONN_TXN_READ_RACE_PREPARE_UPDATE 1453 /*! transaction: rollback to stable calls */ -#define WT_STAT_CONN_TXN_RTS 1435 +#define WT_STAT_CONN_TXN_RTS 1454 /*! * transaction: rollback to stable history store records with stop * timestamps older than newer records */ -#define WT_STAT_CONN_TXN_RTS_HS_STOP_OLDER_THAN_NEWER_START 1436 +#define WT_STAT_CONN_TXN_RTS_HS_STOP_OLDER_THAN_NEWER_START 1455 /*! transaction: rollback to stable inconsistent checkpoint */ -#define WT_STAT_CONN_TXN_RTS_INCONSISTENT_CKPT 1437 +#define WT_STAT_CONN_TXN_RTS_INCONSISTENT_CKPT 1456 /*! transaction: rollback to stable keys removed */ -#define WT_STAT_CONN_TXN_RTS_KEYS_REMOVED 1438 +#define WT_STAT_CONN_TXN_RTS_KEYS_REMOVED 1457 /*! transaction: rollback to stable keys restored */ -#define WT_STAT_CONN_TXN_RTS_KEYS_RESTORED 1439 +#define WT_STAT_CONN_TXN_RTS_KEYS_RESTORED 1458 /*! transaction: rollback to stable pages visited */ -#define WT_STAT_CONN_TXN_RTS_PAGES_VISITED 1440 +#define WT_STAT_CONN_TXN_RTS_PAGES_VISITED 1459 /*! transaction: rollback to stable restored tombstones from history store */ -#define WT_STAT_CONN_TXN_RTS_HS_RESTORE_TOMBSTONES 1441 +#define WT_STAT_CONN_TXN_RTS_HS_RESTORE_TOMBSTONES 1460 /*! transaction: rollback to stable restored updates from history store */ -#define WT_STAT_CONN_TXN_RTS_HS_RESTORE_UPDATES 1442 +#define WT_STAT_CONN_TXN_RTS_HS_RESTORE_UPDATES 1461 /*! transaction: rollback to stable skipping delete rle */ -#define WT_STAT_CONN_TXN_RTS_DELETE_RLE_SKIPPED 1443 +#define WT_STAT_CONN_TXN_RTS_DELETE_RLE_SKIPPED 1462 /*! transaction: rollback to stable skipping stable rle */ -#define WT_STAT_CONN_TXN_RTS_STABLE_RLE_SKIPPED 1444 +#define WT_STAT_CONN_TXN_RTS_STABLE_RLE_SKIPPED 1463 /*! transaction: rollback to stable sweeping history store keys */ -#define WT_STAT_CONN_TXN_RTS_SWEEP_HS_KEYS 1445 +#define WT_STAT_CONN_TXN_RTS_SWEEP_HS_KEYS 1464 /*! transaction: rollback to stable tree walk skipping pages */ -#define WT_STAT_CONN_TXN_RTS_TREE_WALK_SKIP_PAGES 1446 +#define WT_STAT_CONN_TXN_RTS_TREE_WALK_SKIP_PAGES 1465 /*! transaction: rollback to stable updates aborted */ -#define WT_STAT_CONN_TXN_RTS_UPD_ABORTED 1447 +#define WT_STAT_CONN_TXN_RTS_UPD_ABORTED 1466 /*! transaction: rollback to stable updates removed from history store */ -#define WT_STAT_CONN_TXN_RTS_HS_REMOVED 1448 +#define WT_STAT_CONN_TXN_RTS_HS_REMOVED 1467 /*! transaction: sessions scanned in each walk of concurrent sessions */ -#define WT_STAT_CONN_TXN_SESSIONS_WALKED 1449 +#define WT_STAT_CONN_TXN_SESSIONS_WALKED 1468 /*! transaction: set timestamp calls */ -#define WT_STAT_CONN_TXN_SET_TS 1450 +#define WT_STAT_CONN_TXN_SET_TS 1469 /*! transaction: set timestamp durable calls */ -#define WT_STAT_CONN_TXN_SET_TS_DURABLE 1451 +#define WT_STAT_CONN_TXN_SET_TS_DURABLE 1470 /*! transaction: set timestamp durable updates */ -#define WT_STAT_CONN_TXN_SET_TS_DURABLE_UPD 1452 +#define WT_STAT_CONN_TXN_SET_TS_DURABLE_UPD 1471 /*! transaction: set timestamp oldest calls */ -#define WT_STAT_CONN_TXN_SET_TS_OLDEST 1453 +#define WT_STAT_CONN_TXN_SET_TS_OLDEST 1472 /*! transaction: set timestamp oldest updates */ -#define WT_STAT_CONN_TXN_SET_TS_OLDEST_UPD 1454 +#define WT_STAT_CONN_TXN_SET_TS_OLDEST_UPD 1473 /*! transaction: set timestamp stable calls */ -#define WT_STAT_CONN_TXN_SET_TS_STABLE 1455 +#define WT_STAT_CONN_TXN_SET_TS_STABLE 1474 /*! transaction: set timestamp stable updates */ -#define WT_STAT_CONN_TXN_SET_TS_STABLE_UPD 1456 +#define WT_STAT_CONN_TXN_SET_TS_STABLE_UPD 1475 /*! transaction: transaction begins */ -#define WT_STAT_CONN_TXN_BEGIN 1457 +#define WT_STAT_CONN_TXN_BEGIN 1476 /*! transaction: transaction checkpoint currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1458 +#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1477 /*! * transaction: transaction checkpoint currently running for history * store file */ -#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING_HS 1459 +#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING_HS 1478 /*! transaction: transaction checkpoint generation */ -#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1460 +#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1479 /*! * transaction: transaction checkpoint history store file duration * (usecs) */ -#define WT_STAT_CONN_TXN_HS_CKPT_DURATION 1461 +#define WT_STAT_CONN_TXN_HS_CKPT_DURATION 1480 /*! transaction: transaction checkpoint max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1462 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1481 /*! transaction: transaction checkpoint min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1463 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1482 /*! * transaction: transaction checkpoint most recent duration for gathering * all handles (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION 1464 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION 1483 /*! * transaction: transaction checkpoint most recent duration for gathering * applied handles (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_APPLY 1465 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_APPLY 1484 /*! * transaction: transaction checkpoint most recent duration for gathering * skipped handles (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_SKIP 1466 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_SKIP 1485 /*! transaction: transaction checkpoint most recent handles applied */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_APPLIED 1467 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_APPLIED 1486 /*! transaction: transaction checkpoint most recent handles skipped */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_SKIPPED 1468 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_SKIPPED 1487 /*! transaction: transaction checkpoint most recent handles walked */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_WALKED 1469 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_WALKED 1488 /*! transaction: transaction checkpoint most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1470 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1489 /*! transaction: transaction checkpoint prepare currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RUNNING 1471 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RUNNING 1490 /*! transaction: transaction checkpoint prepare max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MAX 1472 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MAX 1491 /*! transaction: transaction checkpoint prepare min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MIN 1473 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MIN 1492 /*! transaction: transaction checkpoint prepare most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RECENT 1474 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RECENT 1493 /*! transaction: transaction checkpoint prepare total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_TOTAL 1475 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_TOTAL 1494 /*! transaction: transaction checkpoint scrub dirty target */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1476 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1495 /*! transaction: transaction checkpoint scrub time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1477 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1496 /*! transaction: transaction checkpoint total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1478 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1497 /*! transaction: transaction checkpoints */ -#define WT_STAT_CONN_TXN_CHECKPOINT 1479 +#define WT_STAT_CONN_TXN_CHECKPOINT 1498 /*! transaction: transaction checkpoints due to obsolete pages */ -#define WT_STAT_CONN_TXN_CHECKPOINT_OBSOLETE_APPLIED 1480 +#define WT_STAT_CONN_TXN_CHECKPOINT_OBSOLETE_APPLIED 1499 /*! * transaction: transaction checkpoints skipped because database was * clean */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1481 +#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1500 /*! transaction: transaction failures due to history store */ -#define WT_STAT_CONN_TXN_FAIL_CACHE 1482 +#define WT_STAT_CONN_TXN_FAIL_CACHE 1501 /*! * transaction: transaction fsync calls for checkpoint after allocating * the transaction ID */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1483 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1502 /*! * transaction: transaction fsync duration for checkpoint after * allocating the transaction ID (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1484 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1503 /*! transaction: transaction range of IDs currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_RANGE 1485 +#define WT_STAT_CONN_TXN_PINNED_RANGE 1504 /*! transaction: transaction range of IDs currently pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1486 +#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1505 /*! transaction: transaction range of timestamps currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP 1487 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP 1506 /*! transaction: transaction range of timestamps pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_CHECKPOINT 1488 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_CHECKPOINT 1507 /*! * transaction: transaction range of timestamps pinned by the oldest * active read timestamp */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_READER 1489 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_READER 1508 /*! * transaction: transaction range of timestamps pinned by the oldest * timestamp */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_OLDEST 1490 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_OLDEST 1509 /*! transaction: transaction read timestamp of the oldest active reader */ -#define WT_STAT_CONN_TXN_TIMESTAMP_OLDEST_ACTIVE_READ 1491 +#define WT_STAT_CONN_TXN_TIMESTAMP_OLDEST_ACTIVE_READ 1510 /*! transaction: transaction rollback to stable currently running */ -#define WT_STAT_CONN_TXN_ROLLBACK_TO_STABLE_RUNNING 1492 +#define WT_STAT_CONN_TXN_ROLLBACK_TO_STABLE_RUNNING 1511 /*! transaction: transaction walk of concurrent sessions */ -#define WT_STAT_CONN_TXN_WALK_SESSIONS 1493 +#define WT_STAT_CONN_TXN_WALK_SESSIONS 1512 /*! transaction: transactions committed */ -#define WT_STAT_CONN_TXN_COMMIT 1494 +#define WT_STAT_CONN_TXN_COMMIT 1513 /*! transaction: transactions rolled back */ -#define WT_STAT_CONN_TXN_ROLLBACK 1495 +#define WT_STAT_CONN_TXN_ROLLBACK 1514 /*! transaction: update conflicts */ -#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1496 +#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1515 /*! * @} @@ -6268,8 +6400,8 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_DSRC_BTREE_CLEAN_CHECKPOINT_TIMER 2023 /*! btree: btree compact pages reviewed */ #define WT_STAT_DSRC_BTREE_COMPACT_PAGES_REVIEWED 2024 -/*! btree: btree compact pages selected to be rewritten */ -#define WT_STAT_DSRC_BTREE_COMPACT_PAGES_WRITE_SELECTED 2025 +/*! btree: btree compact pages rewritten */ +#define WT_STAT_DSRC_BTREE_COMPACT_PAGES_REWRITTEN 2025 /*! btree: btree compact pages skipped */ #define WT_STAT_DSRC_BTREE_COMPACT_PAGES_SKIPPED 2026 /*! btree: btree skipped by compaction as process would not reduce size */ @@ -6301,577 +6433,585 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_DSRC_BTREE_COLUMN_VARIABLE 2032 /*! btree: fixed-record size */ #define WT_STAT_DSRC_BTREE_FIXED_LEN 2033 -/*! btree: maximum internal page key size */ -#define WT_STAT_DSRC_BTREE_MAXINTLKEY 2034 /*! btree: maximum internal page size */ -#define WT_STAT_DSRC_BTREE_MAXINTLPAGE 2035 +#define WT_STAT_DSRC_BTREE_MAXINTLPAGE 2034 /*! btree: maximum leaf page key size */ -#define WT_STAT_DSRC_BTREE_MAXLEAFKEY 2036 +#define WT_STAT_DSRC_BTREE_MAXLEAFKEY 2035 /*! btree: maximum leaf page size */ -#define WT_STAT_DSRC_BTREE_MAXLEAFPAGE 2037 +#define WT_STAT_DSRC_BTREE_MAXLEAFPAGE 2036 /*! btree: maximum leaf page value size */ -#define WT_STAT_DSRC_BTREE_MAXLEAFVALUE 2038 +#define WT_STAT_DSRC_BTREE_MAXLEAFVALUE 2037 /*! btree: maximum tree depth */ -#define WT_STAT_DSRC_BTREE_MAXIMUM_DEPTH 2039 +#define WT_STAT_DSRC_BTREE_MAXIMUM_DEPTH 2038 /*! * btree: number of key/value pairs, only reported if tree_walk or all * statistics are enabled */ -#define WT_STAT_DSRC_BTREE_ENTRIES 2040 +#define WT_STAT_DSRC_BTREE_ENTRIES 2039 /*! * btree: overflow pages, only reported if tree_walk or all statistics * are enabled */ -#define WT_STAT_DSRC_BTREE_OVERFLOW 2041 -/*! btree: pages rewritten by compaction */ -#define WT_STAT_DSRC_BTREE_COMPACT_PAGES_REWRITTEN 2042 +#define WT_STAT_DSRC_BTREE_OVERFLOW 2040 /*! * btree: row-store empty values, only reported if tree_walk or all * statistics are enabled */ -#define WT_STAT_DSRC_BTREE_ROW_EMPTY_VALUES 2043 +#define WT_STAT_DSRC_BTREE_ROW_EMPTY_VALUES 2041 /*! * btree: row-store internal pages, only reported if tree_walk or all * statistics are enabled */ -#define WT_STAT_DSRC_BTREE_ROW_INTERNAL 2044 +#define WT_STAT_DSRC_BTREE_ROW_INTERNAL 2042 /*! * btree: row-store leaf pages, only reported if tree_walk or all * statistics are enabled */ -#define WT_STAT_DSRC_BTREE_ROW_LEAF 2045 +#define WT_STAT_DSRC_BTREE_ROW_LEAF 2043 /*! cache: bytes currently in the cache */ -#define WT_STAT_DSRC_CACHE_BYTES_INUSE 2046 +#define WT_STAT_DSRC_CACHE_BYTES_INUSE 2044 /*! cache: bytes dirty in the cache cumulative */ -#define WT_STAT_DSRC_CACHE_BYTES_DIRTY_TOTAL 2047 +#define WT_STAT_DSRC_CACHE_BYTES_DIRTY_TOTAL 2045 /*! cache: bytes read into cache */ -#define WT_STAT_DSRC_CACHE_BYTES_READ 2048 +#define WT_STAT_DSRC_CACHE_BYTES_READ 2046 /*! cache: bytes written from cache */ -#define WT_STAT_DSRC_CACHE_BYTES_WRITE 2049 +#define WT_STAT_DSRC_CACHE_BYTES_WRITE 2047 /*! cache: checkpoint blocked page eviction */ -#define WT_STAT_DSRC_CACHE_EVICTION_CHECKPOINT 2050 +#define WT_STAT_DSRC_CACHE_EVICTION_CHECKPOINT 2048 /*! * cache: checkpoint of history store file blocked non-history store page * eviction */ -#define WT_STAT_DSRC_CACHE_EVICTION_BLOCKED_CHECKPOINT_HS 2051 +#define WT_STAT_DSRC_CACHE_EVICTION_BLOCKED_CHECKPOINT_HS 2049 /*! cache: data source pages selected for eviction unable to be evicted */ -#define WT_STAT_DSRC_CACHE_EVICTION_FAIL 2052 +#define WT_STAT_DSRC_CACHE_EVICTION_FAIL 2050 /*! * cache: eviction gave up due to detecting an out of order on disk value * behind the last update on the chain */ -#define WT_STAT_DSRC_CACHE_EVICTION_BLOCKED_OOO_CHECKPOINT_RACE_1 2053 +#define WT_STAT_DSRC_CACHE_EVICTION_BLOCKED_OOO_CHECKPOINT_RACE_1 2051 /*! * cache: eviction gave up due to detecting an out of order tombstone * ahead of the selected on disk update */ -#define WT_STAT_DSRC_CACHE_EVICTION_BLOCKED_OOO_CHECKPOINT_RACE_2 2054 +#define WT_STAT_DSRC_CACHE_EVICTION_BLOCKED_OOO_CHECKPOINT_RACE_2 2052 /*! * cache: eviction gave up due to detecting an out of order tombstone * ahead of the selected on disk update after validating the update chain */ -#define WT_STAT_DSRC_CACHE_EVICTION_BLOCKED_OOO_CHECKPOINT_RACE_3 2055 +#define WT_STAT_DSRC_CACHE_EVICTION_BLOCKED_OOO_CHECKPOINT_RACE_3 2053 /*! * cache: eviction gave up due to detecting out of order timestamps on * the update chain after the selected on disk update */ -#define WT_STAT_DSRC_CACHE_EVICTION_BLOCKED_OOO_CHECKPOINT_RACE_4 2056 +#define WT_STAT_DSRC_CACHE_EVICTION_BLOCKED_OOO_CHECKPOINT_RACE_4 2054 /*! cache: eviction walk passes of a file */ -#define WT_STAT_DSRC_CACHE_EVICTION_WALK_PASSES 2057 +#define WT_STAT_DSRC_CACHE_EVICTION_WALK_PASSES 2055 /*! cache: eviction walk target pages histogram - 0-9 */ -#define WT_STAT_DSRC_CACHE_EVICTION_TARGET_PAGE_LT10 2058 +#define WT_STAT_DSRC_CACHE_EVICTION_TARGET_PAGE_LT10 2056 /*! cache: eviction walk target pages histogram - 10-31 */ -#define WT_STAT_DSRC_CACHE_EVICTION_TARGET_PAGE_LT32 2059 +#define WT_STAT_DSRC_CACHE_EVICTION_TARGET_PAGE_LT32 2057 /*! cache: eviction walk target pages histogram - 128 and higher */ -#define WT_STAT_DSRC_CACHE_EVICTION_TARGET_PAGE_GE128 2060 +#define WT_STAT_DSRC_CACHE_EVICTION_TARGET_PAGE_GE128 2058 /*! cache: eviction walk target pages histogram - 32-63 */ -#define WT_STAT_DSRC_CACHE_EVICTION_TARGET_PAGE_LT64 2061 +#define WT_STAT_DSRC_CACHE_EVICTION_TARGET_PAGE_LT64 2059 /*! cache: eviction walk target pages histogram - 64-128 */ -#define WT_STAT_DSRC_CACHE_EVICTION_TARGET_PAGE_LT128 2062 +#define WT_STAT_DSRC_CACHE_EVICTION_TARGET_PAGE_LT128 2060 /*! * cache: eviction walk target pages reduced due to history store cache * pressure */ -#define WT_STAT_DSRC_CACHE_EVICTION_TARGET_PAGE_REDUCED 2063 +#define WT_STAT_DSRC_CACHE_EVICTION_TARGET_PAGE_REDUCED 2061 /*! cache: eviction walks abandoned */ -#define WT_STAT_DSRC_CACHE_EVICTION_WALKS_ABANDONED 2064 +#define WT_STAT_DSRC_CACHE_EVICTION_WALKS_ABANDONED 2062 /*! cache: eviction walks gave up because they restarted their walk twice */ -#define WT_STAT_DSRC_CACHE_EVICTION_WALKS_STOPPED 2065 +#define WT_STAT_DSRC_CACHE_EVICTION_WALKS_STOPPED 2063 /*! * cache: eviction walks gave up because they saw too many pages and * found no candidates */ -#define WT_STAT_DSRC_CACHE_EVICTION_WALKS_GAVE_UP_NO_TARGETS 2066 +#define WT_STAT_DSRC_CACHE_EVICTION_WALKS_GAVE_UP_NO_TARGETS 2064 /*! * cache: eviction walks gave up because they saw too many pages and * found too few candidates */ -#define WT_STAT_DSRC_CACHE_EVICTION_WALKS_GAVE_UP_RATIO 2067 +#define WT_STAT_DSRC_CACHE_EVICTION_WALKS_GAVE_UP_RATIO 2065 /*! cache: eviction walks reached end of tree */ -#define WT_STAT_DSRC_CACHE_EVICTION_WALKS_ENDED 2068 +#define WT_STAT_DSRC_CACHE_EVICTION_WALKS_ENDED 2066 /*! cache: eviction walks restarted */ -#define WT_STAT_DSRC_CACHE_EVICTION_WALK_RESTART 2069 +#define WT_STAT_DSRC_CACHE_EVICTION_WALK_RESTART 2067 /*! cache: eviction walks started from root of tree */ -#define WT_STAT_DSRC_CACHE_EVICTION_WALK_FROM_ROOT 2070 +#define WT_STAT_DSRC_CACHE_EVICTION_WALK_FROM_ROOT 2068 /*! cache: eviction walks started from saved location in tree */ -#define WT_STAT_DSRC_CACHE_EVICTION_WALK_SAVED_POS 2071 +#define WT_STAT_DSRC_CACHE_EVICTION_WALK_SAVED_POS 2069 /*! cache: hazard pointer blocked page eviction */ -#define WT_STAT_DSRC_CACHE_EVICTION_HAZARD 2072 +#define WT_STAT_DSRC_CACHE_EVICTION_HAZARD 2070 /*! cache: history store table insert calls */ -#define WT_STAT_DSRC_CACHE_HS_INSERT 2073 +#define WT_STAT_DSRC_CACHE_HS_INSERT 2071 /*! cache: history store table insert calls that returned restart */ -#define WT_STAT_DSRC_CACHE_HS_INSERT_RESTART 2074 +#define WT_STAT_DSRC_CACHE_HS_INSERT_RESTART 2072 /*! * cache: history store table out-of-order resolved updates that lose * their durable timestamp */ -#define WT_STAT_DSRC_CACHE_HS_ORDER_LOSE_DURABLE_TIMESTAMP 2075 +#define WT_STAT_DSRC_CACHE_HS_ORDER_LOSE_DURABLE_TIMESTAMP 2073 /*! * cache: history store table out-of-order updates that were fixed up by * reinserting with the fixed timestamp */ -#define WT_STAT_DSRC_CACHE_HS_ORDER_REINSERT 2076 +#define WT_STAT_DSRC_CACHE_HS_ORDER_REINSERT 2074 /*! cache: history store table reads */ -#define WT_STAT_DSRC_CACHE_HS_READ 2077 +#define WT_STAT_DSRC_CACHE_HS_READ 2075 /*! cache: history store table reads missed */ -#define WT_STAT_DSRC_CACHE_HS_READ_MISS 2078 +#define WT_STAT_DSRC_CACHE_HS_READ_MISS 2076 /*! cache: history store table reads requiring squashed modifies */ -#define WT_STAT_DSRC_CACHE_HS_READ_SQUASH 2079 +#define WT_STAT_DSRC_CACHE_HS_READ_SQUASH 2077 /*! * cache: history store table truncation by rollback to stable to remove * an unstable update */ -#define WT_STAT_DSRC_CACHE_HS_KEY_TRUNCATE_RTS_UNSTABLE 2080 +#define WT_STAT_DSRC_CACHE_HS_KEY_TRUNCATE_RTS_UNSTABLE 2078 /*! * cache: history store table truncation by rollback to stable to remove * an update */ -#define WT_STAT_DSRC_CACHE_HS_KEY_TRUNCATE_RTS 2081 +#define WT_STAT_DSRC_CACHE_HS_KEY_TRUNCATE_RTS 2079 /*! cache: history store table truncation to remove an update */ -#define WT_STAT_DSRC_CACHE_HS_KEY_TRUNCATE 2082 +#define WT_STAT_DSRC_CACHE_HS_KEY_TRUNCATE 2080 /*! * cache: history store table truncation to remove range of updates due * to key being removed from the data page during reconciliation */ -#define WT_STAT_DSRC_CACHE_HS_KEY_TRUNCATE_ONPAGE_REMOVAL 2083 +#define WT_STAT_DSRC_CACHE_HS_KEY_TRUNCATE_ONPAGE_REMOVAL 2081 /*! * cache: history store table truncation to remove range of updates due * to out-of-order timestamp update on data page */ -#define WT_STAT_DSRC_CACHE_HS_ORDER_REMOVE 2084 +#define WT_STAT_DSRC_CACHE_HS_ORDER_REMOVE 2082 /*! cache: history store table writes requiring squashed modifies */ -#define WT_STAT_DSRC_CACHE_HS_WRITE_SQUASH 2085 +#define WT_STAT_DSRC_CACHE_HS_WRITE_SQUASH 2083 /*! cache: in-memory page passed criteria to be split */ -#define WT_STAT_DSRC_CACHE_INMEM_SPLITTABLE 2086 +#define WT_STAT_DSRC_CACHE_INMEM_SPLITTABLE 2084 /*! cache: in-memory page splits */ -#define WT_STAT_DSRC_CACHE_INMEM_SPLIT 2087 +#define WT_STAT_DSRC_CACHE_INMEM_SPLIT 2085 /*! cache: internal pages evicted */ -#define WT_STAT_DSRC_CACHE_EVICTION_INTERNAL 2088 +#define WT_STAT_DSRC_CACHE_EVICTION_INTERNAL 2086 /*! cache: internal pages split during eviction */ -#define WT_STAT_DSRC_CACHE_EVICTION_SPLIT_INTERNAL 2089 +#define WT_STAT_DSRC_CACHE_EVICTION_SPLIT_INTERNAL 2087 /*! cache: leaf pages split during eviction */ -#define WT_STAT_DSRC_CACHE_EVICTION_SPLIT_LEAF 2090 +#define WT_STAT_DSRC_CACHE_EVICTION_SPLIT_LEAF 2088 /*! cache: modified pages evicted */ -#define WT_STAT_DSRC_CACHE_EVICTION_DIRTY 2091 +#define WT_STAT_DSRC_CACHE_EVICTION_DIRTY 2089 /*! cache: overflow pages read into cache */ -#define WT_STAT_DSRC_CACHE_READ_OVERFLOW 2092 +#define WT_STAT_DSRC_CACHE_READ_OVERFLOW 2090 /*! cache: page split during eviction deepened the tree */ -#define WT_STAT_DSRC_CACHE_EVICTION_DEEPEN 2093 +#define WT_STAT_DSRC_CACHE_EVICTION_DEEPEN 2091 /*! cache: page written requiring history store records */ -#define WT_STAT_DSRC_CACHE_WRITE_HS 2094 +#define WT_STAT_DSRC_CACHE_WRITE_HS 2092 /*! cache: pages read into cache */ -#define WT_STAT_DSRC_CACHE_READ 2095 +#define WT_STAT_DSRC_CACHE_READ 2093 /*! cache: pages read into cache after truncate */ -#define WT_STAT_DSRC_CACHE_READ_DELETED 2096 +#define WT_STAT_DSRC_CACHE_READ_DELETED 2094 /*! cache: pages read into cache after truncate in prepare state */ -#define WT_STAT_DSRC_CACHE_READ_DELETED_PREPARED 2097 +#define WT_STAT_DSRC_CACHE_READ_DELETED_PREPARED 2095 /*! cache: pages requested from the cache */ -#define WT_STAT_DSRC_CACHE_PAGES_REQUESTED 2098 +#define WT_STAT_DSRC_CACHE_PAGES_REQUESTED 2096 /*! cache: pages seen by eviction walk */ -#define WT_STAT_DSRC_CACHE_EVICTION_PAGES_SEEN 2099 +#define WT_STAT_DSRC_CACHE_EVICTION_PAGES_SEEN 2097 /*! cache: pages written from cache */ -#define WT_STAT_DSRC_CACHE_WRITE 2100 +#define WT_STAT_DSRC_CACHE_WRITE 2098 /*! cache: pages written requiring in-memory restoration */ -#define WT_STAT_DSRC_CACHE_WRITE_RESTORE 2101 +#define WT_STAT_DSRC_CACHE_WRITE_RESTORE 2099 /*! cache: the number of times full update inserted to history store */ -#define WT_STAT_DSRC_CACHE_HS_INSERT_FULL_UPDATE 2102 +#define WT_STAT_DSRC_CACHE_HS_INSERT_FULL_UPDATE 2100 /*! cache: the number of times reverse modify inserted to history store */ -#define WT_STAT_DSRC_CACHE_HS_INSERT_REVERSE_MODIFY 2103 +#define WT_STAT_DSRC_CACHE_HS_INSERT_REVERSE_MODIFY 2101 /*! cache: tracked dirty bytes in the cache */ -#define WT_STAT_DSRC_CACHE_BYTES_DIRTY 2104 +#define WT_STAT_DSRC_CACHE_BYTES_DIRTY 2102 /*! cache: unmodified pages evicted */ -#define WT_STAT_DSRC_CACHE_EVICTION_CLEAN 2105 +#define WT_STAT_DSRC_CACHE_EVICTION_CLEAN 2103 /*! * cache_walk: Average difference between current eviction generation * when the page was last considered, only reported if cache_walk or all * statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_GEN_AVG_GAP 2106 +#define WT_STAT_DSRC_CACHE_STATE_GEN_AVG_GAP 2104 /*! * cache_walk: Average on-disk page image size seen, only reported if * cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_AVG_WRITTEN_SIZE 2107 +#define WT_STAT_DSRC_CACHE_STATE_AVG_WRITTEN_SIZE 2105 /*! * cache_walk: Average time in cache for pages that have been visited by * the eviction server, only reported if cache_walk or all statistics are * enabled */ -#define WT_STAT_DSRC_CACHE_STATE_AVG_VISITED_AGE 2108 +#define WT_STAT_DSRC_CACHE_STATE_AVG_VISITED_AGE 2106 /*! * cache_walk: Average time in cache for pages that have not been visited * by the eviction server, only reported if cache_walk or all statistics * are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_AVG_UNVISITED_AGE 2109 +#define WT_STAT_DSRC_CACHE_STATE_AVG_UNVISITED_AGE 2107 /*! * cache_walk: Clean pages currently in cache, only reported if * cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_PAGES_CLEAN 2110 +#define WT_STAT_DSRC_CACHE_STATE_PAGES_CLEAN 2108 /*! * cache_walk: Current eviction generation, only reported if cache_walk * or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_GEN_CURRENT 2111 +#define WT_STAT_DSRC_CACHE_STATE_GEN_CURRENT 2109 /*! * cache_walk: Dirty pages currently in cache, only reported if * cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_PAGES_DIRTY 2112 +#define WT_STAT_DSRC_CACHE_STATE_PAGES_DIRTY 2110 /*! * cache_walk: Entries in the root page, only reported if cache_walk or * all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_ROOT_ENTRIES 2113 +#define WT_STAT_DSRC_CACHE_STATE_ROOT_ENTRIES 2111 /*! * cache_walk: Internal pages currently in cache, only reported if * cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_PAGES_INTERNAL 2114 +#define WT_STAT_DSRC_CACHE_STATE_PAGES_INTERNAL 2112 /*! * cache_walk: Leaf pages currently in cache, only reported if cache_walk * or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_PAGES_LEAF 2115 +#define WT_STAT_DSRC_CACHE_STATE_PAGES_LEAF 2113 /*! * cache_walk: Maximum difference between current eviction generation * when the page was last considered, only reported if cache_walk or all * statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_GEN_MAX_GAP 2116 +#define WT_STAT_DSRC_CACHE_STATE_GEN_MAX_GAP 2114 /*! * cache_walk: Maximum page size seen, only reported if cache_walk or all * statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_MAX_PAGESIZE 2117 +#define WT_STAT_DSRC_CACHE_STATE_MAX_PAGESIZE 2115 /*! * cache_walk: Minimum on-disk page image size seen, only reported if * cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_MIN_WRITTEN_SIZE 2118 +#define WT_STAT_DSRC_CACHE_STATE_MIN_WRITTEN_SIZE 2116 /*! * cache_walk: Number of pages never visited by eviction server, only * reported if cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_UNVISITED_COUNT 2119 +#define WT_STAT_DSRC_CACHE_STATE_UNVISITED_COUNT 2117 /*! * cache_walk: On-disk page image sizes smaller than a single allocation * unit, only reported if cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_SMALLER_ALLOC_SIZE 2120 +#define WT_STAT_DSRC_CACHE_STATE_SMALLER_ALLOC_SIZE 2118 /*! * cache_walk: Pages created in memory and never written, only reported * if cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_MEMORY 2121 +#define WT_STAT_DSRC_CACHE_STATE_MEMORY 2119 /*! * cache_walk: Pages currently queued for eviction, only reported if * cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_QUEUED 2122 +#define WT_STAT_DSRC_CACHE_STATE_QUEUED 2120 /*! * cache_walk: Pages that could not be queued for eviction, only reported * if cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_NOT_QUEUEABLE 2123 +#define WT_STAT_DSRC_CACHE_STATE_NOT_QUEUEABLE 2121 /*! * cache_walk: Refs skipped during cache traversal, only reported if * cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_REFS_SKIPPED 2124 +#define WT_STAT_DSRC_CACHE_STATE_REFS_SKIPPED 2122 /*! * cache_walk: Size of the root page, only reported if cache_walk or all * statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_ROOT_SIZE 2125 +#define WT_STAT_DSRC_CACHE_STATE_ROOT_SIZE 2123 /*! * cache_walk: Total number of pages currently in cache, only reported if * cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_PAGES 2126 +#define WT_STAT_DSRC_CACHE_STATE_PAGES 2124 /*! checkpoint-cleanup: pages added for eviction */ -#define WT_STAT_DSRC_CC_PAGES_EVICT 2127 +#define WT_STAT_DSRC_CC_PAGES_EVICT 2125 /*! checkpoint-cleanup: pages removed */ -#define WT_STAT_DSRC_CC_PAGES_REMOVED 2128 +#define WT_STAT_DSRC_CC_PAGES_REMOVED 2126 /*! checkpoint-cleanup: pages skipped during tree walk */ -#define WT_STAT_DSRC_CC_PAGES_WALK_SKIPPED 2129 +#define WT_STAT_DSRC_CC_PAGES_WALK_SKIPPED 2127 /*! checkpoint-cleanup: pages visited */ -#define WT_STAT_DSRC_CC_PAGES_VISITED 2130 +#define WT_STAT_DSRC_CC_PAGES_VISITED 2128 /*! * compression: compressed page maximum internal page size prior to * compression */ -#define WT_STAT_DSRC_COMPRESS_PRECOMP_INTL_MAX_PAGE_SIZE 2131 +#define WT_STAT_DSRC_COMPRESS_PRECOMP_INTL_MAX_PAGE_SIZE 2129 /*! * compression: compressed page maximum leaf page size prior to * compression */ -#define WT_STAT_DSRC_COMPRESS_PRECOMP_LEAF_MAX_PAGE_SIZE 2132 +#define WT_STAT_DSRC_COMPRESS_PRECOMP_LEAF_MAX_PAGE_SIZE 2130 /*! compression: compressed pages read */ -#define WT_STAT_DSRC_COMPRESS_READ 2133 +#define WT_STAT_DSRC_COMPRESS_READ 2131 /*! compression: compressed pages written */ -#define WT_STAT_DSRC_COMPRESS_WRITE 2134 +#define WT_STAT_DSRC_COMPRESS_WRITE 2132 +/*! compression: number of blocks with compress ratio greater than 64 */ +#define WT_STAT_DSRC_COMPRESS_HIST_RATIO_MAX 2133 +/*! compression: number of blocks with compress ratio smaller than 16 */ +#define WT_STAT_DSRC_COMPRESS_HIST_RATIO_16 2134 +/*! compression: number of blocks with compress ratio smaller than 2 */ +#define WT_STAT_DSRC_COMPRESS_HIST_RATIO_2 2135 +/*! compression: number of blocks with compress ratio smaller than 32 */ +#define WT_STAT_DSRC_COMPRESS_HIST_RATIO_32 2136 +/*! compression: number of blocks with compress ratio smaller than 4 */ +#define WT_STAT_DSRC_COMPRESS_HIST_RATIO_4 2137 +/*! compression: number of blocks with compress ratio smaller than 64 */ +#define WT_STAT_DSRC_COMPRESS_HIST_RATIO_64 2138 +/*! compression: number of blocks with compress ratio smaller than 8 */ +#define WT_STAT_DSRC_COMPRESS_HIST_RATIO_8 2139 /*! compression: page written failed to compress */ -#define WT_STAT_DSRC_COMPRESS_WRITE_FAIL 2135 +#define WT_STAT_DSRC_COMPRESS_WRITE_FAIL 2140 /*! compression: page written was too small to compress */ -#define WT_STAT_DSRC_COMPRESS_WRITE_TOO_SMALL 2136 +#define WT_STAT_DSRC_COMPRESS_WRITE_TOO_SMALL 2141 /*! cursor: Total number of entries skipped by cursor next calls */ -#define WT_STAT_DSRC_CURSOR_NEXT_SKIP_TOTAL 2137 +#define WT_STAT_DSRC_CURSOR_NEXT_SKIP_TOTAL 2142 /*! cursor: Total number of entries skipped by cursor prev calls */ -#define WT_STAT_DSRC_CURSOR_PREV_SKIP_TOTAL 2138 +#define WT_STAT_DSRC_CURSOR_PREV_SKIP_TOTAL 2143 /*! * cursor: Total number of entries skipped to position the history store * cursor */ -#define WT_STAT_DSRC_CURSOR_SKIP_HS_CUR_POSITION 2139 +#define WT_STAT_DSRC_CURSOR_SKIP_HS_CUR_POSITION 2144 /*! * cursor: Total number of times a search near has exited due to prefix * config */ -#define WT_STAT_DSRC_CURSOR_SEARCH_NEAR_PREFIX_FAST_PATHS 2140 +#define WT_STAT_DSRC_CURSOR_SEARCH_NEAR_PREFIX_FAST_PATHS 2145 /*! cursor: bulk loaded cursor insert calls */ -#define WT_STAT_DSRC_CURSOR_INSERT_BULK 2141 +#define WT_STAT_DSRC_CURSOR_INSERT_BULK 2146 /*! cursor: cache cursors reuse count */ -#define WT_STAT_DSRC_CURSOR_REOPEN 2142 +#define WT_STAT_DSRC_CURSOR_REOPEN 2147 /*! cursor: close calls that result in cache */ -#define WT_STAT_DSRC_CURSOR_CACHE 2143 +#define WT_STAT_DSRC_CURSOR_CACHE 2148 /*! cursor: create calls */ -#define WT_STAT_DSRC_CURSOR_CREATE 2144 +#define WT_STAT_DSRC_CURSOR_CREATE 2149 /*! * cursor: cursor next calls that skip due to a globally visible history * store tombstone */ -#define WT_STAT_DSRC_CURSOR_NEXT_HS_TOMBSTONE 2145 +#define WT_STAT_DSRC_CURSOR_NEXT_HS_TOMBSTONE 2150 /*! * cursor: cursor next calls that skip greater than or equal to 100 * entries */ -#define WT_STAT_DSRC_CURSOR_NEXT_SKIP_GE_100 2146 +#define WT_STAT_DSRC_CURSOR_NEXT_SKIP_GE_100 2151 /*! cursor: cursor next calls that skip less than 100 entries */ -#define WT_STAT_DSRC_CURSOR_NEXT_SKIP_LT_100 2147 +#define WT_STAT_DSRC_CURSOR_NEXT_SKIP_LT_100 2152 /*! * cursor: cursor prev calls that skip due to a globally visible history * store tombstone */ -#define WT_STAT_DSRC_CURSOR_PREV_HS_TOMBSTONE 2148 +#define WT_STAT_DSRC_CURSOR_PREV_HS_TOMBSTONE 2153 /*! * cursor: cursor prev calls that skip greater than or equal to 100 * entries */ -#define WT_STAT_DSRC_CURSOR_PREV_SKIP_GE_100 2149 +#define WT_STAT_DSRC_CURSOR_PREV_SKIP_GE_100 2154 /*! cursor: cursor prev calls that skip less than 100 entries */ -#define WT_STAT_DSRC_CURSOR_PREV_SKIP_LT_100 2150 +#define WT_STAT_DSRC_CURSOR_PREV_SKIP_LT_100 2155 /*! cursor: insert calls */ -#define WT_STAT_DSRC_CURSOR_INSERT 2151 +#define WT_STAT_DSRC_CURSOR_INSERT 2156 /*! cursor: insert key and value bytes */ -#define WT_STAT_DSRC_CURSOR_INSERT_BYTES 2152 +#define WT_STAT_DSRC_CURSOR_INSERT_BYTES 2157 /*! cursor: modify */ -#define WT_STAT_DSRC_CURSOR_MODIFY 2153 +#define WT_STAT_DSRC_CURSOR_MODIFY 2158 /*! cursor: modify key and value bytes affected */ -#define WT_STAT_DSRC_CURSOR_MODIFY_BYTES 2154 +#define WT_STAT_DSRC_CURSOR_MODIFY_BYTES 2159 /*! cursor: modify value bytes modified */ -#define WT_STAT_DSRC_CURSOR_MODIFY_BYTES_TOUCH 2155 +#define WT_STAT_DSRC_CURSOR_MODIFY_BYTES_TOUCH 2160 /*! cursor: next calls */ -#define WT_STAT_DSRC_CURSOR_NEXT 2156 +#define WT_STAT_DSRC_CURSOR_NEXT 2161 /*! cursor: open cursor count */ -#define WT_STAT_DSRC_CURSOR_OPEN_COUNT 2157 +#define WT_STAT_DSRC_CURSOR_OPEN_COUNT 2162 /*! cursor: operation restarted */ -#define WT_STAT_DSRC_CURSOR_RESTART 2158 +#define WT_STAT_DSRC_CURSOR_RESTART 2163 /*! cursor: prev calls */ -#define WT_STAT_DSRC_CURSOR_PREV 2159 +#define WT_STAT_DSRC_CURSOR_PREV 2164 /*! cursor: remove calls */ -#define WT_STAT_DSRC_CURSOR_REMOVE 2160 +#define WT_STAT_DSRC_CURSOR_REMOVE 2165 /*! cursor: remove key bytes removed */ -#define WT_STAT_DSRC_CURSOR_REMOVE_BYTES 2161 +#define WT_STAT_DSRC_CURSOR_REMOVE_BYTES 2166 /*! cursor: reserve calls */ -#define WT_STAT_DSRC_CURSOR_RESERVE 2162 +#define WT_STAT_DSRC_CURSOR_RESERVE 2167 /*! cursor: reset calls */ -#define WT_STAT_DSRC_CURSOR_RESET 2163 +#define WT_STAT_DSRC_CURSOR_RESET 2168 /*! cursor: search calls */ -#define WT_STAT_DSRC_CURSOR_SEARCH 2164 +#define WT_STAT_DSRC_CURSOR_SEARCH 2169 /*! cursor: search history store calls */ -#define WT_STAT_DSRC_CURSOR_SEARCH_HS 2165 +#define WT_STAT_DSRC_CURSOR_SEARCH_HS 2170 /*! cursor: search near calls */ -#define WT_STAT_DSRC_CURSOR_SEARCH_NEAR 2166 +#define WT_STAT_DSRC_CURSOR_SEARCH_NEAR 2171 /*! cursor: truncate calls */ -#define WT_STAT_DSRC_CURSOR_TRUNCATE 2167 +#define WT_STAT_DSRC_CURSOR_TRUNCATE 2172 /*! cursor: update calls */ -#define WT_STAT_DSRC_CURSOR_UPDATE 2168 +#define WT_STAT_DSRC_CURSOR_UPDATE 2173 /*! cursor: update key and value bytes */ -#define WT_STAT_DSRC_CURSOR_UPDATE_BYTES 2169 +#define WT_STAT_DSRC_CURSOR_UPDATE_BYTES 2174 /*! cursor: update value size change */ -#define WT_STAT_DSRC_CURSOR_UPDATE_BYTES_CHANGED 2170 +#define WT_STAT_DSRC_CURSOR_UPDATE_BYTES_CHANGED 2175 /*! reconciliation: approximate byte size of timestamps in pages written */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_BYTES_TS 2171 +#define WT_STAT_DSRC_REC_TIME_WINDOW_BYTES_TS 2176 /*! * reconciliation: approximate byte size of transaction IDs in pages * written */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_BYTES_TXN 2172 +#define WT_STAT_DSRC_REC_TIME_WINDOW_BYTES_TXN 2177 /*! reconciliation: dictionary matches */ -#define WT_STAT_DSRC_REC_DICTIONARY 2173 +#define WT_STAT_DSRC_REC_DICTIONARY 2178 /*! reconciliation: fast-path pages deleted */ -#define WT_STAT_DSRC_REC_PAGE_DELETE_FAST 2174 +#define WT_STAT_DSRC_REC_PAGE_DELETE_FAST 2179 /*! * reconciliation: internal page key bytes discarded using suffix * compression */ -#define WT_STAT_DSRC_REC_SUFFIX_COMPRESSION 2175 +#define WT_STAT_DSRC_REC_SUFFIX_COMPRESSION 2180 /*! reconciliation: internal page multi-block writes */ -#define WT_STAT_DSRC_REC_MULTIBLOCK_INTERNAL 2176 -/*! reconciliation: internal-page overflow keys */ -#define WT_STAT_DSRC_REC_OVERFLOW_KEY_INTERNAL 2177 +#define WT_STAT_DSRC_REC_MULTIBLOCK_INTERNAL 2181 /*! reconciliation: leaf page key bytes discarded using prefix compression */ -#define WT_STAT_DSRC_REC_PREFIX_COMPRESSION 2178 +#define WT_STAT_DSRC_REC_PREFIX_COMPRESSION 2182 /*! reconciliation: leaf page multi-block writes */ -#define WT_STAT_DSRC_REC_MULTIBLOCK_LEAF 2179 +#define WT_STAT_DSRC_REC_MULTIBLOCK_LEAF 2183 /*! reconciliation: leaf-page overflow keys */ -#define WT_STAT_DSRC_REC_OVERFLOW_KEY_LEAF 2180 +#define WT_STAT_DSRC_REC_OVERFLOW_KEY_LEAF 2184 /*! reconciliation: maximum blocks required for a page */ -#define WT_STAT_DSRC_REC_MULTIBLOCK_MAX 2181 +#define WT_STAT_DSRC_REC_MULTIBLOCK_MAX 2185 /*! reconciliation: overflow values written */ -#define WT_STAT_DSRC_REC_OVERFLOW_VALUE 2182 +#define WT_STAT_DSRC_REC_OVERFLOW_VALUE 2186 /*! reconciliation: page checksum matches */ -#define WT_STAT_DSRC_REC_PAGE_MATCH 2183 +#define WT_STAT_DSRC_REC_PAGE_MATCH 2187 /*! reconciliation: page reconciliation calls */ -#define WT_STAT_DSRC_REC_PAGES 2184 +#define WT_STAT_DSRC_REC_PAGES 2188 /*! reconciliation: page reconciliation calls for eviction */ -#define WT_STAT_DSRC_REC_PAGES_EVICTION 2185 +#define WT_STAT_DSRC_REC_PAGES_EVICTION 2189 /*! reconciliation: pages deleted */ -#define WT_STAT_DSRC_REC_PAGE_DELETE 2186 +#define WT_STAT_DSRC_REC_PAGE_DELETE 2190 /*! * reconciliation: pages written including an aggregated newest start * durable timestamp */ -#define WT_STAT_DSRC_REC_TIME_AGGR_NEWEST_START_DURABLE_TS 2187 +#define WT_STAT_DSRC_REC_TIME_AGGR_NEWEST_START_DURABLE_TS 2191 /*! * reconciliation: pages written including an aggregated newest stop * durable timestamp */ -#define WT_STAT_DSRC_REC_TIME_AGGR_NEWEST_STOP_DURABLE_TS 2188 +#define WT_STAT_DSRC_REC_TIME_AGGR_NEWEST_STOP_DURABLE_TS 2192 /*! * reconciliation: pages written including an aggregated newest stop * timestamp */ -#define WT_STAT_DSRC_REC_TIME_AGGR_NEWEST_STOP_TS 2189 +#define WT_STAT_DSRC_REC_TIME_AGGR_NEWEST_STOP_TS 2193 /*! * reconciliation: pages written including an aggregated newest stop * transaction ID */ -#define WT_STAT_DSRC_REC_TIME_AGGR_NEWEST_STOP_TXN 2190 +#define WT_STAT_DSRC_REC_TIME_AGGR_NEWEST_STOP_TXN 2194 /*! * reconciliation: pages written including an aggregated newest * transaction ID */ -#define WT_STAT_DSRC_REC_TIME_AGGR_NEWEST_TXN 2191 +#define WT_STAT_DSRC_REC_TIME_AGGR_NEWEST_TXN 2195 /*! * reconciliation: pages written including an aggregated oldest start * timestamp */ -#define WT_STAT_DSRC_REC_TIME_AGGR_OLDEST_START_TS 2192 +#define WT_STAT_DSRC_REC_TIME_AGGR_OLDEST_START_TS 2196 /*! reconciliation: pages written including an aggregated prepare */ -#define WT_STAT_DSRC_REC_TIME_AGGR_PREPARED 2193 +#define WT_STAT_DSRC_REC_TIME_AGGR_PREPARED 2197 /*! reconciliation: pages written including at least one prepare */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_PREPARED 2194 +#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_PREPARED 2198 /*! * reconciliation: pages written including at least one start durable * timestamp */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_DURABLE_START_TS 2195 +#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_DURABLE_START_TS 2199 /*! reconciliation: pages written including at least one start timestamp */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_START_TS 2196 +#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_START_TS 2200 /*! * reconciliation: pages written including at least one start transaction * ID */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_START_TXN 2197 +#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_START_TXN 2201 /*! * reconciliation: pages written including at least one stop durable * timestamp */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_DURABLE_STOP_TS 2198 +#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_DURABLE_STOP_TS 2202 /*! reconciliation: pages written including at least one stop timestamp */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_STOP_TS 2199 +#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_STOP_TS 2203 /*! * reconciliation: pages written including at least one stop transaction * ID */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_STOP_TXN 2200 +#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_STOP_TXN 2204 /*! reconciliation: records written including a prepare */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_PREPARED 2201 +#define WT_STAT_DSRC_REC_TIME_WINDOW_PREPARED 2205 /*! reconciliation: records written including a start durable timestamp */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_DURABLE_START_TS 2202 +#define WT_STAT_DSRC_REC_TIME_WINDOW_DURABLE_START_TS 2206 /*! reconciliation: records written including a start timestamp */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_START_TS 2203 +#define WT_STAT_DSRC_REC_TIME_WINDOW_START_TS 2207 /*! reconciliation: records written including a start transaction ID */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_START_TXN 2204 +#define WT_STAT_DSRC_REC_TIME_WINDOW_START_TXN 2208 /*! reconciliation: records written including a stop durable timestamp */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_DURABLE_STOP_TS 2205 +#define WT_STAT_DSRC_REC_TIME_WINDOW_DURABLE_STOP_TS 2209 /*! reconciliation: records written including a stop timestamp */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_STOP_TS 2206 +#define WT_STAT_DSRC_REC_TIME_WINDOW_STOP_TS 2210 /*! reconciliation: records written including a stop transaction ID */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_STOP_TXN 2207 +#define WT_STAT_DSRC_REC_TIME_WINDOW_STOP_TXN 2211 /*! session: object compaction */ -#define WT_STAT_DSRC_SESSION_COMPACT 2208 +#define WT_STAT_DSRC_SESSION_COMPACT 2212 /*! session: tiered operations dequeued and processed */ -#define WT_STAT_DSRC_TIERED_WORK_UNITS_DEQUEUED 2209 +#define WT_STAT_DSRC_TIERED_WORK_UNITS_DEQUEUED 2213 /*! session: tiered operations scheduled */ -#define WT_STAT_DSRC_TIERED_WORK_UNITS_CREATED 2210 +#define WT_STAT_DSRC_TIERED_WORK_UNITS_CREATED 2214 /*! session: tiered storage local retention time (secs) */ -#define WT_STAT_DSRC_TIERED_RETENTION 2211 +#define WT_STAT_DSRC_TIERED_RETENTION 2215 /*! session: tiered storage object size */ -#define WT_STAT_DSRC_TIERED_OBJECT_SIZE 2212 +#define WT_STAT_DSRC_TIERED_OBJECT_SIZE 2216 /*! transaction: race to read prepared update retry */ -#define WT_STAT_DSRC_TXN_READ_RACE_PREPARE_UPDATE 2213 +#define WT_STAT_DSRC_TXN_READ_RACE_PREPARE_UPDATE 2217 /*! * transaction: rollback to stable history store records with stop * timestamps older than newer records */ -#define WT_STAT_DSRC_TXN_RTS_HS_STOP_OLDER_THAN_NEWER_START 2214 +#define WT_STAT_DSRC_TXN_RTS_HS_STOP_OLDER_THAN_NEWER_START 2218 /*! transaction: rollback to stable inconsistent checkpoint */ -#define WT_STAT_DSRC_TXN_RTS_INCONSISTENT_CKPT 2215 +#define WT_STAT_DSRC_TXN_RTS_INCONSISTENT_CKPT 2219 /*! transaction: rollback to stable keys removed */ -#define WT_STAT_DSRC_TXN_RTS_KEYS_REMOVED 2216 +#define WT_STAT_DSRC_TXN_RTS_KEYS_REMOVED 2220 /*! transaction: rollback to stable keys restored */ -#define WT_STAT_DSRC_TXN_RTS_KEYS_RESTORED 2217 +#define WT_STAT_DSRC_TXN_RTS_KEYS_RESTORED 2221 /*! transaction: rollback to stable restored tombstones from history store */ -#define WT_STAT_DSRC_TXN_RTS_HS_RESTORE_TOMBSTONES 2218 +#define WT_STAT_DSRC_TXN_RTS_HS_RESTORE_TOMBSTONES 2222 /*! transaction: rollback to stable restored updates from history store */ -#define WT_STAT_DSRC_TXN_RTS_HS_RESTORE_UPDATES 2219 +#define WT_STAT_DSRC_TXN_RTS_HS_RESTORE_UPDATES 2223 /*! transaction: rollback to stable skipping delete rle */ -#define WT_STAT_DSRC_TXN_RTS_DELETE_RLE_SKIPPED 2220 +#define WT_STAT_DSRC_TXN_RTS_DELETE_RLE_SKIPPED 2224 /*! transaction: rollback to stable skipping stable rle */ -#define WT_STAT_DSRC_TXN_RTS_STABLE_RLE_SKIPPED 2221 +#define WT_STAT_DSRC_TXN_RTS_STABLE_RLE_SKIPPED 2225 /*! transaction: rollback to stable sweeping history store keys */ -#define WT_STAT_DSRC_TXN_RTS_SWEEP_HS_KEYS 2222 +#define WT_STAT_DSRC_TXN_RTS_SWEEP_HS_KEYS 2226 /*! transaction: rollback to stable updates removed from history store */ -#define WT_STAT_DSRC_TXN_RTS_HS_REMOVED 2223 +#define WT_STAT_DSRC_TXN_RTS_HS_REMOVED 2227 /*! transaction: transaction checkpoints due to obsolete pages */ -#define WT_STAT_DSRC_TXN_CHECKPOINT_OBSOLETE_APPLIED 2224 +#define WT_STAT_DSRC_TXN_CHECKPOINT_OBSOLETE_APPLIED 2228 /*! transaction: update conflicts */ -#define WT_STAT_DSRC_TXN_UPDATE_CONFLICT 2225 +#define WT_STAT_DSRC_TXN_UPDATE_CONFLICT 2229 /*! * @} diff --git a/src/third_party/wiredtiger/src/include/wt_internal.h b/src/third_party/wiredtiger/src/include/wt_internal.h index 4487554a7f3..d1666d35a9c 100644 --- a/src/third_party/wiredtiger/src/include/wt_internal.h +++ b/src/third_party/wiredtiger/src/include/wt_internal.h @@ -69,6 +69,14 @@ struct __wt_addr; typedef struct __wt_addr WT_ADDR; struct __wt_addr_copy; typedef struct __wt_addr_copy WT_ADDR_COPY; +struct __wt_blkcache; +typedef struct __wt_blkcache WT_BLKCACHE; +struct __wt_blkcache_bucket_metadata; +typedef struct __wt_blkcache_bucket_metadata WT_BLKCACHE_BUCKET_METADATA; +struct __wt_blkcache_id; +typedef struct __wt_blkcache_id WT_BLKCACHE_ID; +struct __wt_blkcache_item; +typedef struct __wt_blkcache_item WT_BLKCACHE_ITEM; struct __wt_blkincr; typedef struct __wt_blkincr WT_BLKINCR; struct __wt_block; @@ -349,6 +357,8 @@ struct __wt_update_value; typedef struct __wt_update_value WT_UPDATE_VALUE; struct __wt_update_vector; typedef struct __wt_update_vector WT_UPDATE_VECTOR; +struct __wt_verbose_multi_category; +typedef struct __wt_verbose_multi_category WT_VERBOSE_MULTI_CATEGORY; union __wt_lsn; typedef union __wt_lsn WT_LSN; union __wt_rand_state; @@ -401,6 +411,7 @@ typedef uint64_t wt_timestamp_t; #include "api.h" #include "block.h" +#include "block_cache.h" #include "bloom.h" #include "btmem.h" #include "btree.h" @@ -422,6 +433,7 @@ typedef uint64_t wt_timestamp_t; #include "thread_group.h" #include "tiered.h" #include "txn.h" +#include "verbose.h" #include "session.h" /* required by connection.h */ #include "connection.h" diff --git a/src/third_party/wiredtiger/src/os_win/os_once.c b/src/third_party/wiredtiger/src/os_win/os_once.c index da02db7ae85..5a8da364885 100644 --- a/src/third_party/wiredtiger/src/os_win/os_once.c +++ b/src/third_party/wiredtiger/src/os_win/os_once.c @@ -36,5 +36,5 @@ __wt_once(void (*init_routine)(void)) INIT_ONCE once_control = INIT_ONCE_STATIC_INIT; PVOID lpContext = NULL; - return !InitOnceExecuteOnce(&once_control, &__wt_init_once_callback, init_routine, lpContext); + return (!InitOnceExecuteOnce(&once_control, &__wt_init_once_callback, init_routine, lpContext)); } diff --git a/src/third_party/wiredtiger/src/reconcile/rec_col.c b/src/third_party/wiredtiger/src/reconcile/rec_col.c index 56390a6c50d..136d6c4589b 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_col.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_col.c @@ -35,7 +35,7 @@ __rec_col_fix_bulk_insert_split_check(WT_CURSOR_BULK *cbulk) */ __wt_rec_incr( session, r, cbulk->entry, __bitstr_size((size_t)cbulk->entry * btree->bitcnt)); - WT_RET(__wt_rec_split(session, r, 0, false)); + WT_RET(__wt_rec_split(session, r, 0)); } cbulk->entry = 0; cbulk->nrecs = WT_FIX_BYTES_TO_ENTRIES(btree, r->space_avail); @@ -132,7 +132,7 @@ __wt_bulk_insert_var(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk, bool delet /* Boundary: split or write the page. */ if (WT_CROSSING_SPLIT_BND(r, val->len)) - WT_RET(__wt_rec_split_crossing_bnd(session, r, val->len, false)); + WT_RET(__wt_rec_split_crossing_bnd(session, r, val->len)); /* Copy the value onto the page. */ if (btree->dictionary) @@ -174,7 +174,7 @@ __rec_col_merge(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) /* Boundary: split or write the page. */ if (__wt_rec_need_split(r, val->len)) - WT_RET(__wt_rec_split_crossing_bnd(session, r, val->len, false)); + WT_RET(__wt_rec_split_crossing_bnd(session, r, val->len)); /* Copy the value onto the page. */ __wt_rec_image_copy(session, r, val); @@ -294,7 +294,7 @@ __wt_rec_col_int(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_REF *pageref) /* Boundary: split or write the page. */ if (__wt_rec_need_split(r, val->len)) - WT_ERR(__wt_rec_split_crossing_bnd(session, r, val->len, false)); + WT_ERR(__wt_rec_split_crossing_bnd(session, r, val->len)); /* Copy the value (which is in val, val == r->v) onto the page. */ __wt_rec_image_copy(session, r, val); @@ -408,7 +408,7 @@ __wt_rec_col_fix(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_REF *pageref) * last, allowing it to grow in the future. */ __wt_rec_incr(session, r, entry, __bitstr_size((size_t)entry * btree->bitcnt)); - WT_RET(__wt_rec_split(session, r, 0, false)); + WT_RET(__wt_rec_split(session, r, 0)); /* Calculate the number of entries per page. */ entry = 0; @@ -548,7 +548,7 @@ __rec_col_var_helper(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_SALVAGE_COOKI /* Boundary: split or write the page. */ if (__wt_rec_need_split(r, val->len)) - WT_RET(__wt_rec_split_crossing_bnd(session, r, val->len, false)); + WT_RET(__wt_rec_split_crossing_bnd(session, r, val->len)); /* Copy the value onto the page. */ if (!deleted && !overflow_type && btree->dictionary) diff --git a/src/third_party/wiredtiger/src/reconcile/rec_row.c b/src/third_party/wiredtiger/src/reconcile/rec_row.c index 6817ee53899..6f68749b614 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_row.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_row.c @@ -55,29 +55,16 @@ __rec_key_state_update(WT_RECONCILE *r, bool ovfl_key) * internal page. */ static int -__rec_cell_build_int_key( - WT_SESSION_IMPL *session, WT_RECONCILE *r, const void *data, size_t size, bool *is_ovflp) +__rec_cell_build_int_key(WT_SESSION_IMPL *session, WT_RECONCILE *r, const void *data, size_t size) { - WT_BTREE *btree; WT_REC_KV *key; - *is_ovflp = false; - - btree = S2BT(session); key = &r->k; /* Copy the bytes into the "current" and key buffers. */ WT_RET(__wt_buf_set(session, r->cur, data, size)); WT_RET(__wt_buf_set(session, &key->buf, data, size)); - /* Create an overflow object if the data won't fit. */ - if (size > btree->maxintlkey) { - WT_STAT_CONN_DATA_INCR(session, rec_overflow_key_internal); - - *is_ovflp = true; - return (__wt_rec_cell_build_ovfl(session, r, key, WT_CELL_KEY_OVFL, NULL, 0)); - } - key->cell_len = __wt_cell_pack_int_key(&key->cell, key->buf.size); key->len = key->cell_len + key->buf.size; @@ -222,7 +209,7 @@ __wt_bulk_insert_row(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk) if (!ovfl_key) WT_RET(__rec_cell_build_leaf_key(session, r, NULL, 0, &ovfl_key)); } - WT_RET(__wt_rec_split_crossing_bnd(session, r, key->len + val->len, false)); + WT_RET(__wt_rec_split_crossing_bnd(session, r, key->len + val->len)); } /* Copy the key/value pair onto the page. */ @@ -255,7 +242,6 @@ __rec_row_merge(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) WT_PAGE_MODIFY *mod; WT_REC_KV *key, *val; uint32_t i; - bool ovfl_key; mod = page->modify; @@ -265,8 +251,8 @@ __rec_row_merge(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) /* For each entry in the split array... */ for (multi = mod->mod_multi, i = 0; i < mod->mod_multi_entries; ++multi, ++i) { /* Build the key and value cells. */ - WT_RET(__rec_cell_build_int_key(session, r, WT_IKEY_DATA(multi->key.ikey), - r->cell_zero ? 1 : multi->key.ikey->size, &ovfl_key)); + WT_RET(__rec_cell_build_int_key( + session, r, WT_IKEY_DATA(multi->key.ikey), r->cell_zero ? 1 : multi->key.ikey->size)); r->cell_zero = false; addr = &multi->addr; @@ -274,7 +260,7 @@ __rec_row_merge(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) /* Boundary: split or write the page. */ if (__wt_rec_need_split(r, key->len + val->len)) - WT_RET(__wt_rec_split_crossing_bnd(session, r, key->len + val->len, false)); + WT_RET(__wt_rec_split_crossing_bnd(session, r, key->len + val->len)); /* Copy the key and value onto the page. */ __wt_rec_image_copy(session, r, key); @@ -282,7 +268,7 @@ __rec_row_merge(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) WT_TIME_AGGREGATE_MERGE(session, &r->cur_ptr->ta, &addr->ta); /* Update compression state. */ - __rec_key_state_update(r, ovfl_key); + __rec_key_state_update(r, false); } return (0); } @@ -305,8 +291,8 @@ __wt_rec_row_int(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) WT_REC_KV *key, *val; WT_REF *ref; WT_TIME_AGGREGATE ta; - size_t key_overflow_size, size; - bool force, hazard, key_onpage_ovfl, ovfl_key; + size_t size; + bool hazard, key_onpage_ovfl; const void *p; btree = S2BT(session); @@ -339,8 +325,6 @@ __wt_rec_row_int(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) */ r->cell_zero = true; - key_overflow_size = 0; - /* For each entry in the in-memory page... */ WT_INTL_FOREACH_BEGIN (session, page, ref) { /* @@ -459,52 +443,16 @@ __wt_rec_row_int(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) } WT_CHILD_RELEASE_ERR(session, hazard, ref); - /* - * Build key cell. Truncate any 0th key, internal pages don't need 0th keys. - */ - if (key_onpage_ovfl) { - key->buf.data = cell; - key->buf.size = __wt_cell_total_len(kpack); - key->cell_len = 0; - key->len = key->buf.size; - ovfl_key = true; - key_overflow_size += ikey->size; - } else { - __wt_ref_key(page, ref, &p, &size); - if (r->cell_zero) - size = 1; - WT_ERR(__rec_cell_build_int_key(session, r, p, size, &ovfl_key)); - if (ovfl_key) - key_overflow_size += size; - } + /* Build key cell. Truncate any 0th key, internal pages don't need 0th keys. */ + __wt_ref_key(page, ref, &p, &size); + if (r->cell_zero) + size = 1; + WT_ERR(__rec_cell_build_int_key(session, r, p, size)); r->cell_zero = false; - /* - * Boundary: split or write the page. - * - * We always instantiate row-store internal page keys in order to avoid special casing the - * B+tree search code to handle keys that may not exist (and I/O in a search path). Because - * root pages are pinned, overflow keys on the root page can cause follow-on cache effects - * where huge root pages tie down lots of cache space and eviction frantically attempts to - * evict objects that can't be evicted. If the in-memory image is too large, force a split. - * Potentially, limiting ourselves to 10 items per page is going to result in deep trees - * which will impact search performance, but at some point, the application's configuration - * is too stupid to survive. - */ - force = r->entries > 20 && key_overflow_size > btree->maxmempage_image; - if (force || __wt_rec_need_split(r, key->len + val->len)) { - key_overflow_size = 0; - - /* - * In one path above, we copied address blocks from the page rather than building the - * actual key. In that case, we have to build the key now because we are about to - * promote it. - */ - if (key_onpage_ovfl) - WT_ERR(__wt_buf_set(session, r->cur, WT_IKEY_DATA(ikey), ikey->size)); - - WT_ERR(__wt_rec_split_crossing_bnd(session, r, key->len + val->len, force)); - } + /* Boundary: split or write the page. */ + if (__wt_rec_need_split(r, key->len + val->len)) + WT_ERR(__wt_rec_split_crossing_bnd(session, r, key->len + val->len)); /* Copy the key and value onto the page. */ __wt_rec_image_copy(session, r, key); @@ -512,7 +460,7 @@ __wt_rec_row_int(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) WT_TIME_AGGREGATE_MERGE(session, &r->cur_ptr->ta, &ta); /* Update compression state. */ - __rec_key_state_update(r, ovfl_key); + __rec_key_state_update(r, false); } WT_INTL_FOREACH_END; @@ -581,7 +529,7 @@ __rec_row_leaf_insert(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins) continue; WT_RET(__wt_buf_set(session, r->cur, WT_INSERT_KEY(ins), WT_INSERT_KEY_SIZE(ins))); - WT_RET(__wt_rec_split_crossing_bnd(session, r, 0, false)); + WT_RET(__wt_rec_split_crossing_bnd(session, r, 0)); /* * Turn off prefix and suffix compression until a full key is written into the new page. @@ -644,7 +592,7 @@ __rec_row_leaf_insert(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins) WT_RET(__rec_cell_build_leaf_key(session, r, NULL, 0, &ovfl_key)); } - WT_RET(__wt_rec_split_crossing_bnd(session, r, key->len + val->len, false)); + WT_RET(__wt_rec_split_crossing_bnd(session, r, key->len + val->len)); } /* Copy the key/value pair onto the page. */ @@ -1046,7 +994,7 @@ build: WT_ERR(__rec_cell_build_leaf_key(session, r, NULL, 0, &ovfl_key)); } - WT_ERR(__wt_rec_split_crossing_bnd(session, r, key->len + val->len, false)); + WT_ERR(__wt_rec_split_crossing_bnd(session, r, key->len + val->len)); } /* Copy the key/value pair onto the page. */ diff --git a/src/third_party/wiredtiger/src/reconcile/rec_visibility.c b/src/third_party/wiredtiger/src/reconcile/rec_visibility.c index 24b8e862dd2..cce2ca357d4 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_visibility.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_visibility.c @@ -255,6 +255,20 @@ __timestamp_out_of_order_fix(WT_SESSION_IMPL *session, WT_TIME_WINDOW *select_tw select_tw->start_ts = select_tw->stop_ts; return (true); } + + /* + * As per the time window validation the durable_start_ts must not be greater than the stop_ts. + * Hence, if the stop_ts is less than durable_start_ts and greater than start_ts, make + * durable_start_ts equal to stop_ts. + */ + if ((select_tw->start_ts != select_tw->stop_ts) && + (select_tw->stop_ts < select_tw->durable_start_ts)) { + __wt_verbose(session, WT_VERB_TIMESTAMP, + "Warning: fixing out-of-order timestamps remove earlier than value; time window %s", + __wt_time_window_to_string(select_tw, time_string)); + + select_tw->durable_start_ts = select_tw->stop_ts; + } return (false); } @@ -776,8 +790,7 @@ __wt_rec_upd_select(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins, W !vpack->tw.prepare && (upd_saved || F_ISSET(vpack, WT_CELL_UNPACK_OVERFLOW))) WT_ERR(__rec_append_orig_value(session, page, upd_select->upd, vpack)); - __wt_time_window_clear_obsolete( - session, &upd_select->tw, r->rec_start_oldest_id, r->rec_start_pinned_ts); + __wt_rec_time_window_clear_obsolete(session, &upd_select->tw, r); err: __wt_scr_free(session, &tmp); return (ret); diff --git a/src/third_party/wiredtiger/src/reconcile/rec_write.c b/src/third_party/wiredtiger/src/reconcile/rec_write.c index a4f006b82c3..934de78f516 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_write.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_write.c @@ -73,11 +73,9 @@ __wt_reconcile(WT_SESSION_IMPL *session, WT_REF *ref, WT_SALVAGE_COOKIE *salvage F_SET(session, WT_SESSION_NO_RECONCILE); /* - * Reconciliation locks the page for three reasons: + * Reconciliation locks the page for two reasons: * Reconciliation reads the lists of page updates, obsolete updates * cannot be discarded while reconciliation is in progress; - * The compaction process reads page modification information, which - * reconciliation modifies; * In-memory splits: reconciliation of an internal page cannot handle * a child page splitting during the reconciliation. */ @@ -98,6 +96,9 @@ __wt_reconcile(WT_SESSION_IMPL *session, WT_REF *ref, WT_SALVAGE_COOKIE *salvage */ ret = __reconcile(session, ref, salvage, flags, &page_locked); + /* If writing a page in service of compaction, we're done, clear the flag. */ + F_CLR_ATOMIC(ref->page, WT_PAGE_COMPACTION_WRITE); + err: if (page_locked) WT_PAGE_UNLOCK(session, page); @@ -392,14 +393,18 @@ __rec_root_write(WT_SESSION_IMPL *session, WT_PAGE *page, uint32_t flags) /* * If a single root page was written (either an empty page or there was a 1-for-1 page swap), - * we've written root and checkpoint, we're done. If the root page split, write the resulting - * WT_REF array. We already have an infrastructure for writing pages, create a fake root page - * and write it instead of adding code to write blocks based on the list of blocks resulting - * from a multiblock reconciliation. + * we've written root and checkpoint, we're done. Clear the result of the reconciliation, a root + * page never has the structures that would normally be associated with (at least), the + * replaced-object flag. If the root page split, write the resulting WT_REF array. We already + * have an infrastructure for writing pages, create a fake root page and write it instead of + * adding code to write blocks based on the list of blocks resulting from a multiblock + * reconciliation. + * */ switch (mod->rec_result) { case WT_PM_REC_EMPTY: /* Page is empty */ case WT_PM_REC_REPLACE: /* 1-for-1 page swap */ + mod->rec_result = 0; return (0); case WT_PM_REC_MULTIBLOCK: /* Multiple blocks */ break; @@ -1145,13 +1150,16 @@ __rec_split_grow(WT_SESSION_IMPL *session, WT_RECONCILE *r, size_t add_len) return (0); } +/* The minimum number of entries before we'll split a row-store internal page. */ +#define WT_PAGE_INTL_MINIMUM_ENTRIES 20 + /* * __wt_rec_split -- * Handle the page reconciliation bookkeeping. (Did you know "bookkeeper" has 3 doubled letters * in a row? Sweet-tooth does, too.) */ int -__wt_rec_split(WT_SESSION_IMPL *session, WT_RECONCILE *r, size_t next_len, bool forced) +__wt_rec_split(WT_SESSION_IMPL *session, WT_RECONCILE *r, size_t next_len) { WT_BTREE *btree; WT_REC_CHUNK *tmp; @@ -1170,9 +1178,16 @@ __wt_rec_split(WT_SESSION_IMPL *session, WT_RECONCILE *r, size_t next_len, bool /* * We can get here if the first key/value pair won't fit. Grow the buffer to contain the current * item if we haven't already consumed a reasonable portion of a split chunk. + * + * If we're promoting huge keys into an internal page, we might be about to write an internal + * page with too few items, which isn't good for tree depth or search. Grow the buffer to + * contain the current item if we don't have enough items to split an internal page. */ inuse = WT_PTRDIFF(r->first_free, r->cur_ptr->image.mem); - if (!forced && inuse < r->split_size / 2 && !__wt_rec_need_split(r, 0)) + if (inuse < r->split_size / 2 && !__wt_rec_need_split(r, 0)) + goto done; + + if (r->page->type == WT_PAGE_ROW_INT && r->entries < WT_PAGE_INTL_MINIMUM_ENTRIES) goto done; /* All page boundaries reset the dictionary. */ @@ -1185,7 +1200,7 @@ __wt_rec_split(WT_SESSION_IMPL *session, WT_RECONCILE *r, size_t next_len, bool /* * Normally we keep two chunks in memory at a given time, and we write the previous chunk at * each boundary, switching the previous and current check references. The exception is when - * doing a bulk load or a forced split, where we write out the chunks as we get them. + * doing a bulk load. */ if (r->is_bulk_load) WT_RET(__rec_split_write(session, r, r->cur_ptr, NULL, false)); @@ -1193,19 +1208,13 @@ __wt_rec_split(WT_SESSION_IMPL *session, WT_RECONCILE *r, size_t next_len, bool if (r->prev_ptr != NULL) WT_RET(__rec_split_write(session, r, r->prev_ptr, NULL, false)); - if (forced) { - WT_RET(__rec_split_write(session, r, r->cur_ptr, NULL, false)); - r->prev_ptr = NULL; - r->cur_ptr = &r->chunk_A; - } else { - if (r->prev_ptr == NULL) { - WT_RET(__rec_split_chunk_init(session, r, &r->chunk_B)); - r->prev_ptr = &r->chunk_B; - } - tmp = r->prev_ptr; - r->prev_ptr = r->cur_ptr; - r->cur_ptr = tmp; + if (r->prev_ptr == NULL) { + WT_RET(__rec_split_chunk_init(session, r, &r->chunk_B)); + r->prev_ptr = &r->chunk_B; } + tmp = r->prev_ptr; + r->prev_ptr = r->cur_ptr; + r->cur_ptr = tmp; } /* Initialize the next chunk, including the key. */ @@ -1226,13 +1235,19 @@ __wt_rec_split(WT_SESSION_IMPL *session, WT_RECONCILE *r, size_t next_len, bool done: /* - * Overflow values can be larger than the maximum page size but still be "on-page". If the next - * key/value pair is larger than space available after a split has happened (in other words, - * larger than the maximum page size), create a page sized to hold that one key/value pair. This - * generally splits the page into key/value pairs before a large object, the object, and - * key/value pairs after the object. It's possible other key/value pairs will also be aggregated - * onto the bigger page before or after, if the page happens to hold them, but it won't - * necessarily happen that way. + * We may have declined the split as described above, in which case grow the buffer based on the + * next key/value pair's length. In the internal page minimum-key case, we could grow more than + * a single key/value pair's length to avoid repeatedly calling this function, but we'd prefer + * not to have internal pages that are larger than they need to be, and repeatedly trying to + * split means we will split as soon as we can. + * + * Also, overflow values can be larger than the maximum page size but still be "on-page". If the + * next key/value pair is larger than space available after a split has happened (in other + * words, larger than the maximum page size), create a page sized to hold that one key/value + * pair. This generally splits the page into key/value pairs before a large object, the object, + * and key/value pairs after the object. It's possible other key/value pairs will also be + * aggregated onto the bigger page before or after, if the page happens to hold them, but it + * won't necessarily happen that way. */ if (r->space_avail < next_len) WT_RET(__rec_split_grow(session, r, next_len)); @@ -1245,14 +1260,14 @@ done: * Save the details for the minimum split size boundary or call for a split. */ int -__wt_rec_split_crossing_bnd(WT_SESSION_IMPL *session, WT_RECONCILE *r, size_t next_len, bool forced) +__wt_rec_split_crossing_bnd(WT_SESSION_IMPL *session, WT_RECONCILE *r, size_t next_len) { /* * If crossing the minimum split size boundary, store the boundary details at the current * location in the buffer. If we are crossing the split boundary at the same time, possible when * the next record is large enough, just split at this point. */ - if (!forced && WT_CROSSING_MIN_BND(r, next_len) && !WT_CROSSING_SPLIT_BND(r, next_len) && + if (WT_CROSSING_MIN_BND(r, next_len) && !WT_CROSSING_SPLIT_BND(r, next_len) && !__wt_rec_need_split(r, 0)) { /* * If the first record doesn't fit into the minimum split size, we end up here. Write the @@ -1279,7 +1294,7 @@ __wt_rec_split_crossing_bnd(WT_SESSION_IMPL *session, WT_RECONCILE *r, size_t ne } /* We are crossing a split boundary */ - return (__wt_rec_split(session, r, next_len, forced)); + return (__wt_rec_split(session, r, next_len)); } /* @@ -1635,12 +1650,11 @@ __rec_split_write_reuse( multi->checksum = __wt_checksum(image->data, image->size); /* - * Don't check for a block match when writing blocks during compaction, the whole idea is to - * move those blocks. Check after calculating the checksum, we don't distinguish between pages - * written solely as part of the compaction and pages written at around the same time, and so - * there's a possibility the calculated checksum will be useful in the future. + * Don't check for a block match when writing a page for compaction, the whole idea is to move + * those blocks. Check after calculating the checksum, there's a possibility the calculated + * checksum will be useful in the future. */ - if (session->compact_state != WT_COMPACT_NONE) + if (F_ISSET_ATOMIC(r->page, WT_PAGE_COMPACTION_WRITE)) return (false); /* diff --git a/src/third_party/wiredtiger/src/session/session_api.c b/src/third_party/wiredtiger/src/session/session_api.c index 8353a6e9d5a..4fa42bd9b3f 100644 --- a/src/third_party/wiredtiger/src/session/session_api.c +++ b/src/third_party/wiredtiger/src/session/session_api.c @@ -417,6 +417,11 @@ __session_reconfigure(WT_SESSION *wt_session, const char *config) } WT_ERR_NOTFOUND_OK(ret, false); + ret = __wt_config_getones(session, config, "cache_max_wait_ms", &cval); + if (ret == 0 && cval.val) + session->cache_max_wait_us = (uint64_t)(cval.val * WT_THOUSAND); + WT_ERR_NOTFOUND_OK(ret, false); + err: API_END_RET_NOTFOUND_MAP(session, ret); } diff --git a/src/third_party/wiredtiger/src/session/session_compact.c b/src/third_party/wiredtiger/src/session/session_compact.c index b57a734b29b..0fc929ac94f 100644 --- a/src/third_party/wiredtiger/src/session/session_compact.c +++ b/src/third_party/wiredtiger/src/session/session_compact.c @@ -189,10 +189,17 @@ __wt_session_compact_check_timeout(WT_SESSION_IMPL *session) return (0); __wt_epoch(session, &end); + ret = session->compact->max_time > WT_TIMEDIFF_SEC(end, session->compact->begin) ? 0 : ETIMEDOUT; - if (ret != 0) + if (ret != 0) { WT_STAT_CONN_INCR(session, session_table_compact_timeout); + + __wt_verbose_info(session, WT_VERB_COMPACT, + "Compact has timed out! The operation has been running for %" PRIu64 + " second(s). Configured timeout is %" PRIu64 " second(s).", + WT_TIMEDIFF_SEC(end, session->compact->begin), session->compact->max_time); + } return (ret); } @@ -279,6 +286,10 @@ __compact_worker(WT_SESSION_IMPL *session) } ret = 0; another_pass = true; + + __wt_verbose_info(session, WT_VERB_COMPACT, "%s", + "Data handle compaction failed with EBUSY but the cache is not stuck. " + "Will give it another go."); } } if (!another_pass) @@ -329,8 +340,11 @@ __wt_session_compact(WT_SESSION *wt_session, const char *uri, const char *config } /* In-memory ignores compaction operations. */ - if (F_ISSET(S2C(session), WT_CONN_IN_MEMORY)) + if (F_ISSET(S2C(session), WT_CONN_IN_MEMORY)) { + __wt_verbose_warning( + session, WT_VERB_COMPACT, "%s", "Compact does not work for in-memory databases."); goto err; + } /* * Non-LSM object compaction requires checkpoints, which are impossible in transactional diff --git a/src/third_party/wiredtiger/src/support/lock_ext.c b/src/third_party/wiredtiger/src/support/lock_ext.c index 38d338892c1..e9d99aefe13 100644 --- a/src/third_party/wiredtiger/src/support/lock_ext.c +++ b/src/third_party/wiredtiger/src/support/lock_ext.c @@ -22,10 +22,10 @@ __wt_ext_spin_init(WT_EXTENSION_API *wt_api, WT_EXTENSION_SPINLOCK *ext_spinlock ext_spinlock->spinlock = NULL; default_session = ((WT_CONNECTION_IMPL *)wt_api->conn)->default_session; if ((ret = __wt_calloc_one(default_session, &lock)) != 0) - return ret; + return (ret); if ((ret = __wt_spin_init(default_session, lock, name)) != 0) { __wt_free(default_session, lock); - return ret; + return (ret); } ext_spinlock->spinlock = lock; return (0); @@ -44,7 +44,6 @@ __wt_ext_spin_lock( WT_UNUSED(wt_api); /* Unused parameters */ lock = ((WT_SPINLOCK *)ext_spinlock->spinlock); __wt_spin_lock((WT_SESSION_IMPL *)session, lock); - return; } /* @@ -60,7 +59,6 @@ __wt_ext_spin_unlock( WT_UNUSED(wt_api); /* Unused parameters */ lock = ((WT_SPINLOCK *)ext_spinlock->spinlock); __wt_spin_unlock((WT_SESSION_IMPL *)session, lock); - return; } /* @@ -80,5 +78,4 @@ __wt_ext_spin_destroy(WT_EXTENSION_API *wt_api, WT_EXTENSION_SPINLOCK *ext_spinl __wt_spin_destroy(default_session, lock); __wt_free(default_session, lock); ext_spinlock->spinlock = NULL; - return; } diff --git a/src/third_party/wiredtiger/src/support/stat.c b/src/third_party/wiredtiger/src/support/stat.c index 43527b9092a..fe442ab7465 100644 --- a/src/third_party/wiredtiger/src/support/stat.c +++ b/src/third_party/wiredtiger/src/support/stat.c @@ -28,7 +28,7 @@ static const char *const __stats_dsrc_desc[] = { "btree: btree checkpoint generation", "btree: btree clean tree checkpoint expiration time", "btree: btree compact pages reviewed", - "btree: btree compact pages selected to be rewritten", + "btree: btree compact pages rewritten", "btree: btree compact pages skipped", "btree: btree skipped by compaction as process would not reduce size", "btree: column-store fixed-size leaf pages", @@ -37,7 +37,6 @@ static const char *const __stats_dsrc_desc[] = { "btree: column-store variable-size deleted values", "btree: column-store variable-size leaf pages", "btree: fixed-record size", - "btree: maximum internal page key size", "btree: maximum internal page size", "btree: maximum leaf page key size", "btree: maximum leaf page size", @@ -45,7 +44,6 @@ static const char *const __stats_dsrc_desc[] = { "btree: maximum tree depth", "btree: number of key/value pairs", "btree: overflow pages", - "btree: pages rewritten by compaction", "btree: row-store empty values", "btree: row-store internal pages", "btree: row-store leaf pages", @@ -147,6 +145,13 @@ static const char *const __stats_dsrc_desc[] = { "compression: compressed page maximum leaf page size prior to compression ", "compression: compressed pages read", "compression: compressed pages written", + "compression: number of blocks with compress ratio greater than 64", + "compression: number of blocks with compress ratio smaller than 16", + "compression: number of blocks with compress ratio smaller than 2", + "compression: number of blocks with compress ratio smaller than 32", + "compression: number of blocks with compress ratio smaller than 4", + "compression: number of blocks with compress ratio smaller than 64", + "compression: number of blocks with compress ratio smaller than 8", "compression: page written failed to compress", "compression: page written was too small to compress", "cursor: Total number of entries skipped by cursor next calls", @@ -189,7 +194,6 @@ static const char *const __stats_dsrc_desc[] = { "reconciliation: fast-path pages deleted", "reconciliation: internal page key bytes discarded using suffix compression", "reconciliation: internal page multi-block writes", - "reconciliation: internal-page overflow keys", "reconciliation: leaf page key bytes discarded using prefix compression", "reconciliation: leaf page multi-block writes", "reconciliation: leaf-page overflow keys", @@ -304,16 +308,15 @@ __wt_stat_dsrc_clear_single(WT_DSRC_STATS *stats) /* not clearing btree_checkpoint_generation */ /* not clearing btree_clean_checkpoint_timer */ /* not clearing btree_compact_pages_reviewed */ - /* not clearing btree_compact_pages_write_selected */ + /* not clearing btree_compact_pages_rewritten */ /* not clearing btree_compact_pages_skipped */ - stats->btree_compact_skipped = 0; + /* not clearing btree_compact_skipped */ stats->btree_column_fix = 0; stats->btree_column_internal = 0; stats->btree_column_rle = 0; stats->btree_column_deleted = 0; stats->btree_column_variable = 0; stats->btree_fixed_len = 0; - stats->btree_maxintlkey = 0; stats->btree_maxintlpage = 0; stats->btree_maxleafkey = 0; stats->btree_maxleafpage = 0; @@ -321,7 +324,6 @@ __wt_stat_dsrc_clear_single(WT_DSRC_STATS *stats) stats->btree_maximum_depth = 0; stats->btree_entries = 0; stats->btree_overflow = 0; - stats->btree_compact_pages_rewritten = 0; stats->btree_row_empty_values = 0; stats->btree_row_internal = 0; stats->btree_row_leaf = 0; @@ -414,6 +416,13 @@ __wt_stat_dsrc_clear_single(WT_DSRC_STATS *stats) /* not clearing compress_precomp_leaf_max_page_size */ stats->compress_read = 0; stats->compress_write = 0; + stats->compress_hist_ratio_max = 0; + stats->compress_hist_ratio_16 = 0; + stats->compress_hist_ratio_2 = 0; + stats->compress_hist_ratio_32 = 0; + stats->compress_hist_ratio_4 = 0; + stats->compress_hist_ratio_64 = 0; + stats->compress_hist_ratio_8 = 0; stats->compress_write_fail = 0; stats->compress_write_too_small = 0; stats->cursor_next_skip_total = 0; @@ -456,7 +465,6 @@ __wt_stat_dsrc_clear_single(WT_DSRC_STATS *stats) stats->rec_page_delete_fast = 0; stats->rec_suffix_compression = 0; stats->rec_multiblock_internal = 0; - stats->rec_overflow_key_internal = 0; stats->rec_prefix_compression = 0; stats->rec_multiblock_leaf = 0; stats->rec_overflow_key_leaf = 0; @@ -549,7 +557,7 @@ __wt_stat_dsrc_aggregate_single(WT_DSRC_STATS *from, WT_DSRC_STATS *to) to->btree_checkpoint_generation += from->btree_checkpoint_generation; to->btree_clean_checkpoint_timer += from->btree_clean_checkpoint_timer; to->btree_compact_pages_reviewed += from->btree_compact_pages_reviewed; - to->btree_compact_pages_write_selected += from->btree_compact_pages_write_selected; + to->btree_compact_pages_rewritten += from->btree_compact_pages_rewritten; to->btree_compact_pages_skipped += from->btree_compact_pages_skipped; to->btree_compact_skipped += from->btree_compact_skipped; to->btree_column_fix += from->btree_column_fix; @@ -559,8 +567,6 @@ __wt_stat_dsrc_aggregate_single(WT_DSRC_STATS *from, WT_DSRC_STATS *to) to->btree_column_variable += from->btree_column_variable; if (from->btree_fixed_len > to->btree_fixed_len) to->btree_fixed_len = from->btree_fixed_len; - if (from->btree_maxintlkey > to->btree_maxintlkey) - to->btree_maxintlkey = from->btree_maxintlkey; if (from->btree_maxintlpage > to->btree_maxintlpage) to->btree_maxintlpage = from->btree_maxintlpage; if (from->btree_maxleafkey > to->btree_maxleafkey) @@ -573,7 +579,6 @@ __wt_stat_dsrc_aggregate_single(WT_DSRC_STATS *from, WT_DSRC_STATS *to) to->btree_maximum_depth = from->btree_maximum_depth; to->btree_entries += from->btree_entries; to->btree_overflow += from->btree_overflow; - to->btree_compact_pages_rewritten += from->btree_compact_pages_rewritten; to->btree_row_empty_values += from->btree_row_empty_values; to->btree_row_internal += from->btree_row_internal; to->btree_row_leaf += from->btree_row_leaf; @@ -670,6 +675,13 @@ __wt_stat_dsrc_aggregate_single(WT_DSRC_STATS *from, WT_DSRC_STATS *to) to->compress_precomp_leaf_max_page_size += from->compress_precomp_leaf_max_page_size; to->compress_read += from->compress_read; to->compress_write += from->compress_write; + to->compress_hist_ratio_max += from->compress_hist_ratio_max; + to->compress_hist_ratio_16 += from->compress_hist_ratio_16; + to->compress_hist_ratio_2 += from->compress_hist_ratio_2; + to->compress_hist_ratio_32 += from->compress_hist_ratio_32; + to->compress_hist_ratio_4 += from->compress_hist_ratio_4; + to->compress_hist_ratio_64 += from->compress_hist_ratio_64; + to->compress_hist_ratio_8 += from->compress_hist_ratio_8; to->compress_write_fail += from->compress_write_fail; to->compress_write_too_small += from->compress_write_too_small; to->cursor_next_skip_total += from->cursor_next_skip_total; @@ -712,7 +724,6 @@ __wt_stat_dsrc_aggregate_single(WT_DSRC_STATS *from, WT_DSRC_STATS *to) to->rec_page_delete_fast += from->rec_page_delete_fast; to->rec_suffix_compression += from->rec_suffix_compression; to->rec_multiblock_internal += from->rec_multiblock_internal; - to->rec_overflow_key_internal += from->rec_overflow_key_internal; to->rec_prefix_compression += from->rec_prefix_compression; to->rec_multiblock_leaf += from->rec_multiblock_leaf; to->rec_overflow_key_leaf += from->rec_overflow_key_leaf; @@ -799,8 +810,7 @@ __wt_stat_dsrc_aggregate(WT_DSRC_STATS **from, WT_DSRC_STATS *to) to->btree_checkpoint_generation += WT_STAT_READ(from, btree_checkpoint_generation); to->btree_clean_checkpoint_timer += WT_STAT_READ(from, btree_clean_checkpoint_timer); to->btree_compact_pages_reviewed += WT_STAT_READ(from, btree_compact_pages_reviewed); - to->btree_compact_pages_write_selected += - WT_STAT_READ(from, btree_compact_pages_write_selected); + to->btree_compact_pages_rewritten += WT_STAT_READ(from, btree_compact_pages_rewritten); to->btree_compact_pages_skipped += WT_STAT_READ(from, btree_compact_pages_skipped); to->btree_compact_skipped += WT_STAT_READ(from, btree_compact_skipped); to->btree_column_fix += WT_STAT_READ(from, btree_column_fix); @@ -810,8 +820,6 @@ __wt_stat_dsrc_aggregate(WT_DSRC_STATS **from, WT_DSRC_STATS *to) to->btree_column_variable += WT_STAT_READ(from, btree_column_variable); if ((v = WT_STAT_READ(from, btree_fixed_len)) > to->btree_fixed_len) to->btree_fixed_len = v; - if ((v = WT_STAT_READ(from, btree_maxintlkey)) > to->btree_maxintlkey) - to->btree_maxintlkey = v; if ((v = WT_STAT_READ(from, btree_maxintlpage)) > to->btree_maxintlpage) to->btree_maxintlpage = v; if ((v = WT_STAT_READ(from, btree_maxleafkey)) > to->btree_maxleafkey) @@ -824,7 +832,6 @@ __wt_stat_dsrc_aggregate(WT_DSRC_STATS **from, WT_DSRC_STATS *to) to->btree_maximum_depth = v; to->btree_entries += WT_STAT_READ(from, btree_entries); to->btree_overflow += WT_STAT_READ(from, btree_overflow); - to->btree_compact_pages_rewritten += WT_STAT_READ(from, btree_compact_pages_rewritten); to->btree_row_empty_values += WT_STAT_READ(from, btree_row_empty_values); to->btree_row_internal += WT_STAT_READ(from, btree_row_internal); to->btree_row_leaf += WT_STAT_READ(from, btree_row_leaf); @@ -930,6 +937,13 @@ __wt_stat_dsrc_aggregate(WT_DSRC_STATS **from, WT_DSRC_STATS *to) WT_STAT_READ(from, compress_precomp_leaf_max_page_size); to->compress_read += WT_STAT_READ(from, compress_read); to->compress_write += WT_STAT_READ(from, compress_write); + to->compress_hist_ratio_max += WT_STAT_READ(from, compress_hist_ratio_max); + to->compress_hist_ratio_16 += WT_STAT_READ(from, compress_hist_ratio_16); + to->compress_hist_ratio_2 += WT_STAT_READ(from, compress_hist_ratio_2); + to->compress_hist_ratio_32 += WT_STAT_READ(from, compress_hist_ratio_32); + to->compress_hist_ratio_4 += WT_STAT_READ(from, compress_hist_ratio_4); + to->compress_hist_ratio_64 += WT_STAT_READ(from, compress_hist_ratio_64); + to->compress_hist_ratio_8 += WT_STAT_READ(from, compress_hist_ratio_8); to->compress_write_fail += WT_STAT_READ(from, compress_write_fail); to->compress_write_too_small += WT_STAT_READ(from, compress_write_too_small); to->cursor_next_skip_total += WT_STAT_READ(from, cursor_next_skip_total); @@ -973,7 +987,6 @@ __wt_stat_dsrc_aggregate(WT_DSRC_STATS **from, WT_DSRC_STATS *to) to->rec_page_delete_fast += WT_STAT_READ(from, rec_page_delete_fast); to->rec_suffix_compression += WT_STAT_READ(from, rec_suffix_compression); to->rec_multiblock_internal += WT_STAT_READ(from, rec_multiblock_internal); - to->rec_overflow_key_internal += WT_STAT_READ(from, rec_overflow_key_internal); to->rec_prefix_compression += WT_STAT_READ(from, rec_prefix_compression); to->rec_multiblock_leaf += WT_STAT_READ(from, rec_multiblock_leaf); to->rec_overflow_key_leaf += WT_STAT_READ(from, rec_overflow_key_leaf); @@ -1041,6 +1054,27 @@ static const char *const __stats_connection_desc[] = { "LSM: tree maintenance operations executed", "LSM: tree maintenance operations scheduled", "LSM: tree queue hit maximum", + "block-manager: block cache cached blocks updated", + "block-manager: block cache cached bytes updated", + "block-manager: block cache evicted blocks", + "block-manager: block cache file size causing bypass", + "block-manager: block cache lookups", + "block-manager: block cache number of blocks not evicted due to overhead", + "block-manager: block cache number of bypasses because no-write-allocate setting was on", + "block-manager: block cache number of bypasses due to overhead on put", + "block-manager: block cache number of bypasses on get", + "block-manager: block cache number of bypasses on put because file is too small", + "block-manager: block cache number of eviction passes", + "block-manager: block cache number of hits including existence checks", + "block-manager: block cache number of misses including existence checks", + "block-manager: block cache number of put bypasses on checkpoint I/O", + "block-manager: block cache removed blocks", + "block-manager: block cache total blocks", + "block-manager: block cache total blocks inserted on read path", + "block-manager: block cache total blocks inserted on write path", + "block-manager: block cache total bytes", + "block-manager: block cache total bytes inserted on read path", + "block-manager: block cache total bytes inserted on write path", "block-manager: blocks pre-loaded", "block-manager: blocks read", "block-manager: blocks written", @@ -1187,7 +1221,6 @@ static const char *const __stats_connection_desc[] = { "cache: pages seen by eviction walk", "cache: pages seen by eviction walk that are already queued", "cache: pages selected for eviction unable to be evicted", - "cache: pages selected for eviction unable to be evicted as the parent page has overflow items", "cache: pages selected for eviction unable to be evicted because of active children on an " "internal page", "cache: pages selected for eviction unable to be evicted because of failure in reconciliation", @@ -1387,7 +1420,6 @@ static const char *const __stats_connection_desc[] = { "reconciliation: approximate byte size of timestamps in pages written", "reconciliation: approximate byte size of transaction IDs in pages written", "reconciliation: fast-path pages deleted", - "reconciliation: internal-page overflow keys", "reconciliation: leaf-page overflow keys", "reconciliation: maximum seconds spent in a reconciliation call", "reconciliation: page reconciliation calls", @@ -1595,6 +1627,27 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->lsm_work_units_done = 0; stats->lsm_work_units_created = 0; stats->lsm_work_queue_max = 0; + stats->block_cache_blocks_update = 0; + stats->block_cache_bytes_update = 0; + stats->block_cache_blocks_evicted = 0; + stats->block_cache_bypass_filesize = 0; + stats->block_cache_data_refs = 0; + stats->block_cache_not_evicted_overhead = 0; + stats->block_cache_bypass_writealloc = 0; + stats->block_cache_bypass_overhead_put = 0; + stats->block_cache_bypass_get = 0; + stats->block_cache_bypass_put = 0; + stats->block_cache_eviction_passes = 0; + stats->block_cache_hits = 0; + stats->block_cache_misses = 0; + stats->block_cache_bypass_chkpt = 0; + stats->block_cache_blocks_removed = 0; + stats->block_cache_blocks = 0; + stats->block_cache_blocks_insert_read = 0; + stats->block_cache_blocks_insert_write = 0; + stats->block_cache_bytes = 0; + stats->block_cache_bytes_insert_read = 0; + stats->block_cache_bytes_insert_write = 0; stats->block_preload = 0; stats->block_read = 0; stats->block_write = 0; @@ -1730,7 +1783,6 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->cache_eviction_pages_seen = 0; stats->cache_eviction_pages_already_queued = 0; stats->cache_eviction_fail = 0; - stats->cache_eviction_fail_parent_has_overflow_items = 0; stats->cache_eviction_fail_active_children_on_an_internal_page = 0; stats->cache_eviction_fail_in_reconciliation = 0; stats->cache_eviction_fail_checkpoint_out_of_order_ts = 0; @@ -1928,7 +1980,6 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->rec_time_window_bytes_ts = 0; stats->rec_time_window_bytes_txn = 0; stats->rec_page_delete_fast = 0; - stats->rec_overflow_key_internal = 0; stats->rec_overflow_key_leaf = 0; /* not clearing rec_maximum_seconds */ stats->rec_pages = 0; @@ -2108,6 +2159,27 @@ __wt_stat_connection_aggregate(WT_CONNECTION_STATS **from, WT_CONNECTION_STATS * to->lsm_work_units_done += WT_STAT_READ(from, lsm_work_units_done); to->lsm_work_units_created += WT_STAT_READ(from, lsm_work_units_created); to->lsm_work_queue_max += WT_STAT_READ(from, lsm_work_queue_max); + to->block_cache_blocks_update += WT_STAT_READ(from, block_cache_blocks_update); + to->block_cache_bytes_update += WT_STAT_READ(from, block_cache_bytes_update); + to->block_cache_blocks_evicted += WT_STAT_READ(from, block_cache_blocks_evicted); + to->block_cache_bypass_filesize += WT_STAT_READ(from, block_cache_bypass_filesize); + to->block_cache_data_refs += WT_STAT_READ(from, block_cache_data_refs); + to->block_cache_not_evicted_overhead += WT_STAT_READ(from, block_cache_not_evicted_overhead); + to->block_cache_bypass_writealloc += WT_STAT_READ(from, block_cache_bypass_writealloc); + to->block_cache_bypass_overhead_put += WT_STAT_READ(from, block_cache_bypass_overhead_put); + to->block_cache_bypass_get += WT_STAT_READ(from, block_cache_bypass_get); + to->block_cache_bypass_put += WT_STAT_READ(from, block_cache_bypass_put); + to->block_cache_eviction_passes += WT_STAT_READ(from, block_cache_eviction_passes); + to->block_cache_hits += WT_STAT_READ(from, block_cache_hits); + to->block_cache_misses += WT_STAT_READ(from, block_cache_misses); + to->block_cache_bypass_chkpt += WT_STAT_READ(from, block_cache_bypass_chkpt); + to->block_cache_blocks_removed += WT_STAT_READ(from, block_cache_blocks_removed); + to->block_cache_blocks += WT_STAT_READ(from, block_cache_blocks); + to->block_cache_blocks_insert_read += WT_STAT_READ(from, block_cache_blocks_insert_read); + to->block_cache_blocks_insert_write += WT_STAT_READ(from, block_cache_blocks_insert_write); + to->block_cache_bytes += WT_STAT_READ(from, block_cache_bytes); + to->block_cache_bytes_insert_read += WT_STAT_READ(from, block_cache_bytes_insert_read); + to->block_cache_bytes_insert_write += WT_STAT_READ(from, block_cache_bytes_insert_write); to->block_preload += WT_STAT_READ(from, block_preload); to->block_read += WT_STAT_READ(from, block_read); to->block_write += WT_STAT_READ(from, block_write); @@ -2269,8 +2341,6 @@ __wt_stat_connection_aggregate(WT_CONNECTION_STATS **from, WT_CONNECTION_STATS * to->cache_eviction_pages_already_queued += WT_STAT_READ(from, cache_eviction_pages_already_queued); to->cache_eviction_fail += WT_STAT_READ(from, cache_eviction_fail); - to->cache_eviction_fail_parent_has_overflow_items += - WT_STAT_READ(from, cache_eviction_fail_parent_has_overflow_items); to->cache_eviction_fail_active_children_on_an_internal_page += WT_STAT_READ(from, cache_eviction_fail_active_children_on_an_internal_page); to->cache_eviction_fail_in_reconciliation += @@ -2476,7 +2546,6 @@ __wt_stat_connection_aggregate(WT_CONNECTION_STATS **from, WT_CONNECTION_STATS * to->rec_time_window_bytes_ts += WT_STAT_READ(from, rec_time_window_bytes_ts); to->rec_time_window_bytes_txn += WT_STAT_READ(from, rec_time_window_bytes_txn); to->rec_page_delete_fast += WT_STAT_READ(from, rec_page_delete_fast); - to->rec_overflow_key_internal += WT_STAT_READ(from, rec_overflow_key_internal); to->rec_overflow_key_leaf += WT_STAT_READ(from, rec_overflow_key_leaf); to->rec_maximum_seconds += WT_STAT_READ(from, rec_maximum_seconds); to->rec_pages += WT_STAT_READ(from, rec_pages); diff --git a/src/third_party/wiredtiger/src/tiered/tiered_handle.c b/src/third_party/wiredtiger/src/tiered/tiered_handle.c index 5be43f315c6..4a44cd90443 100644 --- a/src/third_party/wiredtiger/src/tiered/tiered_handle.c +++ b/src/third_party/wiredtiger/src/tiered/tiered_handle.c @@ -301,7 +301,8 @@ __wt_tiered_set_metadata(WT_SESSION_IMPL *session, WT_TIERED *tiered, WT_ITEM *b { uint32_t i; - WT_RET(__wt_buf_catfmt(session, buf, ",last=%" PRIu32 ",tiers=(", tiered->current_id)); + WT_RET(__wt_buf_catfmt(session, buf, ",last=%" PRIu32 ",oldest=%" PRIu32 ",tiers=(", + tiered->current_id, tiered->oldest_id)); for (i = 0; i < WT_TIERED_MAX_TIERS; ++i) { if (tiered->tiers[i].name == NULL) { __wt_verbose(session, WT_VERB_TIERED, "TIER_SET_META: names[%" PRIu32 "] NULL", i); @@ -543,8 +544,18 @@ __tiered_open(WT_SESSION_IMPL *session, const char *cfg[]) WT_ERR(__wt_config_getones(session, config, "last", &cval)); tiered->current_id = (uint32_t)cval.val; tiered->next_id = tiered->current_id + 1; - __wt_verbose(session, WT_VERB_TIERED, "TIERED_OPEN: current %d, next %d", - (int)tiered->current_id, (int)tiered->next_id); + /* + * For now this is always one. When garbage collection gets implemented then it will be updated + * to reflect the first object number that exists. Knowing this information will be helpful for + * other tasks such as tiered backup. + */ + WT_ERR(__wt_config_getones(session, config, "oldest", &cval)); + tiered->oldest_id = (uint32_t)cval.val; + WT_ASSERT(session, tiered->oldest_id == 1); + + __wt_verbose(session, WT_VERB_TIERED, + "TIERED_OPEN: current %" PRIu32 ", next %" PRIu32 ", oldest %" PRIu32, tiered->current_id, + tiered->next_id, tiered->oldest_id); ret = __wt_config_getones(session, config, "tiers", &tierconf); WT_ERR_NOTFOUND_OK(ret, true); diff --git a/src/third_party/wiredtiger/src/tiered/tiered_opener.c b/src/third_party/wiredtiger/src/tiered/tiered_opener.c index b6f3df90fa7..f786ef30eff 100644 --- a/src/third_party/wiredtiger/src/tiered/tiered_opener.c +++ b/src/third_party/wiredtiger/src/tiered/tiered_opener.c @@ -17,13 +17,18 @@ __tiered_opener_open(WT_BLOCK_FILE_OPENER *opener, WT_SESSION_IMPL *session, uin WT_FS_OPEN_FILE_TYPE type, u_int flags, WT_FH **fhp) { WT_BUCKET_STORAGE *bstorage; + WT_CONFIG_ITEM pfx; WT_DECL_RET; WT_TIERED *tiered; - const char *object_name, *object_uri; + size_t len; + char *tmp; + const char *cfg[2], *object_name, *object_uri, *object_val; bool local_only; tiered = opener->cookie; object_uri = NULL; + object_val = NULL; + tmp = NULL; local_only = false; WT_ASSERT(session, @@ -54,13 +59,25 @@ __tiered_opener_open(WT_BLOCK_FILE_OPENER *opener, WT_SESSION_IMPL *session, uin * This can be called at any time, because we are opening the objects lazily. */ if (!local_only && ret != 0) { + /* Get the prefix from the object's metadata, not the connection. */ + WT_ERR(__wt_metadata_search(session, object_uri, (char **)&object_val)); + cfg[0] = object_val; + cfg[1] = NULL; + WT_ERR(__wt_config_gets(session, cfg, "tiered_storage.bucket_prefix", &pfx)); + /* We expect a prefix. */ + WT_ASSERT(session, pfx.len != 0); + len = strlen(object_name) + pfx.len + 1; + WT_ERR(__wt_calloc_def(session, len, &tmp)); + WT_ERR(__wt_snprintf(tmp, len, "%.*s%s", (int)pfx.len, pfx.str, object_name)); bstorage = tiered->bstorage; - LF_SET(WT_FS_OPEN_READONLY); + LF_SET(WT_FS_OPEN_FIXED | WT_FS_OPEN_READONLY); WT_WITH_BUCKET_STORAGE( - bstorage, session, { ret = __wt_open(session, object_name, type, flags, fhp); }); + bstorage, session, { ret = __wt_open(session, tmp, type, flags, fhp); }); } err: __wt_free(session, object_uri); + __wt_free(session, object_val); + __wt_free(session, tmp); return (ret); } diff --git a/src/third_party/wiredtiger/src/txn/txn.c b/src/third_party/wiredtiger/src/txn/txn.c index f113463a1ef..bddfbe26ac7 100644 --- a/src/third_party/wiredtiger/src/txn/txn.c +++ b/src/third_party/wiredtiger/src/txn/txn.c @@ -893,6 +893,13 @@ __txn_commit_timestamps_usage_check(WT_SESSION_IMPL *session, WT_TXN_OP *op, WT_ char ts_string[2][WT_TS_INT_STRING_SIZE]; bool txn_has_ts; + /* + * Do not check for timestamp usage in recovery as it is possible that timestamps may be out of + * order due to WiredTiger log replay in recovery doesn't use any timestamps. + */ + if (F_ISSET(S2C(session), WT_CONN_RECOVERING)) + return (0); + txn = session->txn; txn_has_ts = F_ISSET(txn, WT_TXN_HAS_TS_COMMIT | WT_TXN_HAS_TS_DURABLE); @@ -2205,6 +2212,7 @@ int __wt_txn_rollback_required(WT_SESSION_IMPL *session, const char *reason) { session->txn->rollback_reason = reason; + __wt_verbose_debug(session, WT_VERB_TRANSACTION, "Rollback reason: %s", reason); return (WT_ROLLBACK); } diff --git a/src/third_party/wiredtiger/src/txn/txn_recover.c b/src/third_party/wiredtiger/src/txn/txn_recover.c index c76e2af3597..4d23db24ac4 100644 --- a/src/third_party/wiredtiger/src/txn/txn_recover.c +++ b/src/third_party/wiredtiger/src/txn/txn_recover.c @@ -8,6 +8,11 @@ #include "wt_internal.h" +/* Enable all recovery-related verbose messaging events. */ +#define WT_VERB_RECOVERY_ALL \ + WT_DECL_VERBOSE_MULTI_CATEGORY( \ + ((WT_VERBOSE_CATEGORY[]){WT_VERB_RECOVERY, WT_VERB_RECOVERY_PROGRESS})) + /* State maintained during recovery. */ typedef struct { const char *uri; /* File URI. */ @@ -413,8 +418,7 @@ __recovery_set_checkpoint_timestamp(WT_RECOVERY *r) */ conn->txn_global.meta_ckpt_timestamp = conn->txn_global.recovery_timestamp = ckpt_timestamp; - __wt_verbose(session, WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS, - "Set global recovery timestamp: %s", + __wt_verbose_multi(session, WT_VERB_RECOVERY_ALL, "Set global recovery timestamp: %s", __wt_timestamp_to_string(conn->txn_global.recovery_timestamp, ts_string)); return (0); @@ -445,8 +449,7 @@ __recovery_set_oldest_timestamp(WT_RECOVERY *r) conn->txn_global.oldest_timestamp = oldest_timestamp; conn->txn_global.has_oldest_timestamp = oldest_timestamp != WT_TS_NONE; - __wt_verbose(session, WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS, - "Set global oldest timestamp: %s", + __wt_verbose_multi(session, WT_VERB_RECOVERY_ALL, "Set global oldest timestamp: %s", __wt_timestamp_to_string(conn->txn_global.oldest_timestamp, ts_string)); return (0); @@ -901,7 +904,7 @@ __wt_txn_recover(WT_SESSION_IMPL *session, const char *cfg[]) * get truncated. */ r.metadata_only = false; - __wt_verbose(session, WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS, + __wt_verbose_multi(session, WT_VERB_RECOVERY_ALL, "Main recovery loop: starting at %" PRIu32 "/%" PRIu32 " to %" PRIu32 "/%" PRIu32, r.ckpt_lsn.l.file, r.ckpt_lsn.l.offset, r.max_rec_lsn.l.file, r.max_rec_lsn.l.offset); WT_ERR(__wt_log_needs_recovery(session, &r.ckpt_lsn, &needs_rec)); @@ -922,7 +925,7 @@ __wt_txn_recover(WT_SESSION_IMPL *session, const char *cfg[]) } if (!hs_exists) { - __wt_verbose(session, WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS, "%s", + __wt_verbose_multi(session, WT_VERB_RECOVERY_ALL, "%s", "Creating the history store before applying log records. Likely recovering after an" "unclean shutdown on an earlier version"); /* @@ -997,7 +1000,8 @@ done: if (conn->txn_global.recovery_timestamp != WT_TS_NONE) conn->txn_global.has_stable_timestamp = true; - __wt_verbose(session, WT_VERB_RECOVERY | WT_VERB_RTS, + __wt_verbose_multi(session, + WT_DECL_VERBOSE_MULTI_CATEGORY(((WT_VERBOSE_CATEGORY[]){WT_VERB_RECOVERY, WT_VERB_RTS})), "performing recovery rollback_to_stable with stable timestamp: %s and oldest timestamp: " "%s", __wt_timestamp_to_string(conn->txn_global.stable_timestamp, ts_string[0]), diff --git a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c index 6aee858e94b..6639f019709 100644 --- a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c +++ b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c @@ -12,8 +12,10 @@ (F_ISSET(S2C(session), WT_CONN_RECOVERING) && (txnid) >= S2C(session)->recovery_ckpt_snap_min) /* Enable rollback to stable verbose messaging during recovery. */ -#define WT_VERB_RECOVERY_RTS(session) \ - (F_ISSET(S2C(session), WT_CONN_RECOVERING) ? WT_VERB_RECOVERY | WT_VERB_RTS : WT_VERB_RTS) +#define WT_VERB_RECOVERY_RTS(session) \ + (F_ISSET(S2C(session), WT_CONN_RECOVERING) ? \ + WT_DECL_VERBOSE_MULTI_CATEGORY(((WT_VERBOSE_CATEGORY[]){WT_VERB_RECOVERY, WT_VERB_RTS})) : \ + WT_DECL_VERBOSE_MULTI_CATEGORY(((WT_VERBOSE_CATEGORY[]){WT_VERB_RTS}))) /* * __rollback_delete_hs -- @@ -89,7 +91,7 @@ __rollback_abort_update(WT_SESSION_IMPL *session, WT_ITEM *key, WT_UPDATE *first continue; if (rollback_timestamp < upd->durable_ts || upd->prepare_state == WT_PREPARE_INPROGRESS) { - __wt_verbose(session, WT_VERB_RECOVERY_RTS(session), + __wt_verbose_multi(session, WT_VERB_RECOVERY_RTS(session), "rollback to stable update aborted with txnid: %" PRIu64 " durable timestamp: %s and stable timestamp: %s, prepared: %s", upd->txnid, __wt_timestamp_to_string(upd->durable_ts, ts_string[0]), @@ -204,7 +206,7 @@ __rollback_has_stable_update(WT_UPDATE *upd) { while (upd != NULL && (upd->type == WT_UPDATE_INVALID || upd->txnid == WT_TXN_ABORTED)) upd = upd->next; - return upd != NULL; + return (upd != NULL); } /* @@ -399,7 +401,7 @@ __rollback_ondisk_fixup_key(WT_SESSION_IMPL *session, WT_REF *ref, WT_ROW *rip, */ if (__rollback_txn_visible_id(session, hs_tw->stop_txn) && hs_stop_durable_ts <= pinned_ts) { - __wt_verbose(session, WT_VERB_RECOVERY_RTS(session), + __wt_verbose_multi(session, WT_VERB_RECOVERY_RTS(session), "history store stop is obsolete with time window: %s and pinned timestamp: %s", __wt_time_window_to_string(hs_tw, tw_string), __wt_timestamp_to_string(pinned_ts, ts_string[0])); @@ -429,7 +431,7 @@ __rollback_ondisk_fixup_key(WT_SESSION_IMPL *session, WT_REF *ref, WT_ROW *rip, WT_ERR(__wt_buf_set(session, full_value, hs_value->data, hs_value->size)); } } else - __wt_verbose(session, WT_VERB_RECOVERY_RTS(session), + __wt_verbose_multi(session, WT_VERB_RECOVERY_RTS(session), "history store update more recent than on-disk update with start timestamp: %s," " durable timestamp: %s, stop timestamp: %s and type: %" PRIu8, __wt_timestamp_to_string(hs_start_ts, ts_string[0]), @@ -471,7 +473,7 @@ __rollback_ondisk_fixup_key(WT_SESSION_IMPL *session, WT_REF *ref, WT_ROW *rip, */ if (__rollback_txn_visible_id(session, hs_tw->start_txn) && hs_durable_ts <= rollback_timestamp) { - __wt_verbose(session, WT_VERB_RECOVERY_RTS(session), + __wt_verbose_multi(session, WT_VERB_RECOVERY_RTS(session), "history store update valid with start timestamp: %s, durable timestamp: %s, stop " "timestamp: %s, stable timestamp: %s, txnid: %" PRIu64 " and type: %" PRIu8, __wt_timestamp_to_string(hs_start_ts, ts_string[0]), @@ -483,7 +485,7 @@ __rollback_ondisk_fixup_key(WT_SESSION_IMPL *session, WT_REF *ref, WT_ROW *rip, break; } - __wt_verbose(session, WT_VERB_RECOVERY_RTS(session), + __wt_verbose_multi(session, WT_VERB_RECOVERY_RTS(session), "history store update aborted with start timestamp: %s, durable timestamp: %s, stop " "timestamp: %s, stable timestamp: %s, start txnid: %" PRIu64 ", stop txnid: %" PRIu64 " and type: %" PRIu8, @@ -531,7 +533,7 @@ __rollback_ondisk_fixup_key(WT_SESSION_IMPL *session, WT_REF *ref, WT_ROW *rip, upd->txnid = hs_tw->start_txn; upd->durable_ts = hs_tw->durable_start_ts; upd->start_ts = hs_tw->start_ts; - __wt_verbose(session, WT_VERB_RECOVERY_RTS(session), + __wt_verbose_multi(session, WT_VERB_RECOVERY_RTS(session), "update restored from history store txnid: %" PRIu64 ", start_ts: %s and durable_ts: %s", upd->txnid, __wt_timestamp_to_string(upd->start_ts, ts_string[0]), __wt_timestamp_to_string(upd->durable_ts, ts_string[1])); @@ -570,7 +572,7 @@ __rollback_ondisk_fixup_key(WT_SESSION_IMPL *session, WT_REF *ref, WT_ROW *rip, tombstone->txnid = hs_tw->stop_txn; tombstone->durable_ts = hs_tw->durable_stop_ts; tombstone->start_ts = hs_tw->stop_ts; - __wt_verbose(session, WT_VERB_RECOVERY_RTS(session), + __wt_verbose_multi(session, WT_VERB_RECOVERY_RTS(session), "tombstone restored from history store txnid: %" PRIu64 ", start_ts: %s, durable_ts: %s", tombstone->txnid, __wt_timestamp_to_string(tombstone->start_ts, ts_string[0]), @@ -589,7 +591,7 @@ __rollback_ondisk_fixup_key(WT_SESSION_IMPL *session, WT_REF *ref, WT_ROW *rip, } else { WT_ERR(__wt_upd_alloc_tombstone(session, &upd, NULL)); WT_STAT_CONN_DATA_INCR(session, txn_rts_keys_removed); - __wt_verbose(session, WT_VERB_RECOVERY_RTS(session), "%p: key removed", (void *)key); + __wt_verbose_multi(session, WT_VERB_RECOVERY_RTS(session), "%p: key removed", (void *)key); } if (rip != NULL) @@ -654,7 +656,7 @@ __rollback_abort_ondisk_kv(WT_SESSION_IMPL *session, WT_REF *ref, WT_ROW *rip, u * with prepared transactions. */ if (vpack->tw.durable_stop_ts > rollback_timestamp || vpack->tw.stop_ts == WT_TS_MAX) { - __wt_verbose(session, WT_VERB_RECOVERY_RTS(session), + __wt_verbose_multi(session, WT_VERB_RECOVERY_RTS(session), "hs update aborted with start durable/commit timestamp: %s, %s, stop durable/commit " "timestamp: %s, %s and stable timestamp: %s", __wt_timestamp_to_string(vpack->tw.durable_start_ts, ts_string[0]), @@ -669,7 +671,7 @@ __rollback_abort_ondisk_kv(WT_SESSION_IMPL *session, WT_REF *ref, WT_ROW *rip, u } else if (vpack->tw.durable_start_ts > rollback_timestamp || !__rollback_txn_visible_id(session, vpack->tw.start_txn) || (!WT_TIME_WINDOW_HAS_STOP(&vpack->tw) && prepared)) { - __wt_verbose(session, WT_VERB_RECOVERY_RTS(session), + __wt_verbose_multi(session, WT_VERB_RECOVERY_RTS(session), "on-disk update aborted with start durable timestamp: %s, commit timestamp: %s, " "prepared: %s, stable timestamp: %s and txnid : %" PRIu64, __wt_timestamp_to_string(vpack->tw.durable_start_ts, ts_string[0]), @@ -734,7 +736,7 @@ __rollback_abort_ondisk_kv(WT_SESSION_IMPL *session, WT_REF *ref, WT_ROW *rip, u upd->start_ts = vpack->tw.start_ts; F_SET(upd, WT_UPDATE_RESTORED_FROM_DS); WT_STAT_CONN_DATA_INCR(session, txn_rts_keys_restored); - __wt_verbose(session, WT_VERB_RECOVERY_RTS(session), + __wt_verbose_multi(session, WT_VERB_RECOVERY_RTS(session), "key restored with commit timestamp: %s, durable timestamp: %s, stable timestamp: " "%s, " "txnid: %" PRIu64 @@ -996,9 +998,8 @@ static wt_timestamp_t __rollback_get_ref_max_durable_timestamp(WT_SESSION_IMPL *session, WT_TIME_AGGREGATE *ta) { if (WT_IS_HS(session->dhandle)) - return WT_MAX(ta->newest_stop_durable_ts, ta->newest_stop_ts); - else - return WT_MAX(ta->newest_start_durable_ts, ta->newest_stop_durable_ts); + return (WT_MAX(ta->newest_stop_durable_ts, ta->newest_stop_ts)); + return (WT_MAX(ta->newest_start_durable_ts, ta->newest_stop_durable_ts)); } /* @@ -1070,7 +1071,7 @@ __rollback_page_needs_abort( WT_CHECK_RECOVERY_FLAG_TXNID(session, newest_txn); } - __wt_verbose(session, WT_VERB_RECOVERY_RTS(session), + __wt_verbose_multi(session, WT_VERB_RECOVERY_RTS(session), "%p: page with %s durable timestamp: %s, newest txn: %" PRIu64 " and prepared updates: %s", (void *)ref, tag, __wt_timestamp_to_string(durable_ts, ts_string), newest_txn, prepared ? "true" : "false"); @@ -1095,12 +1096,12 @@ __rollback_abort_updates(WT_SESSION_IMPL *session, WT_REF *ref, wt_timestamp_t r page = ref->page; if (!__wt_page_is_modified(page) && !__rollback_page_needs_abort(session, ref, rollback_timestamp)) { - __wt_verbose(session, WT_VERB_RECOVERY_RTS(session), "%p: page skipped", (void *)ref); + __wt_verbose_multi(session, WT_VERB_RECOVERY_RTS(session), "%p: page skipped", (void *)ref); return (0); } WT_STAT_CONN_INCR(session, txn_rts_pages_visited); - __wt_verbose(session, WT_VERB_RECOVERY_RTS(session), + __wt_verbose_multi(session, WT_VERB_RECOVERY_RTS(session), "%p: page rolled back when page is modified: %s", (void *)ref, __wt_page_is_modified(page) ? "true" : "false"); @@ -1149,8 +1150,8 @@ __rollback_abort_fast_truncate( */ if (child_ref->state == WT_REF_DELETED && child_ref->ft_info.del != NULL && rollback_timestamp < child_ref->ft_info.del->durable_timestamp) { - __wt_verbose(session, WT_VERB_RECOVERY_RTS(session), "%p: deleted page rolled back", - (void *)child_ref); + __wt_verbose_multi(session, WT_VERB_RECOVERY_RTS(session), + "%p: deleted page rolled back", (void *)child_ref); WT_RET(__wt_delete_page_rollback(session, child_ref)); } } @@ -1177,7 +1178,8 @@ __wt_rts_page_skip(WT_SESSION_IMPL *session, WT_REF *ref, void *context, bool *s /* Check whether this ref has any possible updates to be aborted. */ if (!__rollback_page_needs_abort(session, ref, rollback_timestamp)) { *skipp = true; - __wt_verbose(session, WT_VERB_RECOVERY_RTS(session), "%p: page walk skipped", (void *)ref); + __wt_verbose_multi( + session, WT_VERB_RECOVERY_RTS(session), "%p: page walk skipped", (void *)ref); WT_STAT_CONN_INCR(session, txn_rts_tree_walk_skip_pages); } @@ -1221,7 +1223,7 @@ __rollback_to_stable_btree(WT_SESSION_IMPL *session, wt_timestamp_t rollback_tim btree = S2BT(session); conn = S2C(session); - __wt_verbose(session, WT_VERB_RECOVERY_RTS(session), + __wt_verbose_multi(session, WT_VERB_RECOVERY_RTS(session), "rollback to stable connection logging enabled: %s and btree logging enabled: %s", FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED) ? "true" : "false", !F_ISSET(btree, WT_BTREE_NO_LOGGING) ? "true" : "false"); @@ -1304,7 +1306,7 @@ __rollback_to_stable_btree_hs_truncate(WT_SESSION_IMPL *session, uint32_t btree_ /* We shouldn't cross the btree search space. */ WT_ASSERT(session, btree_id == hs_btree_id); - __wt_verbose(session, WT_VERB_RECOVERY_RTS(session), + __wt_verbose_multi(session, WT_VERB_RECOVERY_RTS(session), "rollback to stable history store cleanup of update with start timestamp: %s", __wt_timestamp_to_string(hs_start_ts, ts_string)); @@ -1369,12 +1371,12 @@ __rollback_to_stable_hs_final_pass(WT_SESSION_IMPL *session, wt_timestamp_t roll * durable start/stop timestamp is greater than the rollback timestamp. */ if (max_durable_ts > rollback_timestamp) { - __wt_verbose(session, WT_VERB_RECOVERY_RTS(session), + __wt_verbose_multi(session, WT_VERB_RECOVERY_RTS(session), "tree rolled back with durable timestamp: %s", __wt_timestamp_to_string(max_durable_ts, ts_string[0])); WT_TRET(__rollback_to_stable_btree(session, rollback_timestamp)); } else - __wt_verbose(session, WT_VERB_RECOVERY_RTS(session), + __wt_verbose_multi(session, WT_VERB_RECOVERY_RTS(session), "tree skipped with durable timestamp: %s and stable timestamp: %s", __wt_timestamp_to_string(max_durable_ts, ts_string[0]), __wt_timestamp_to_string(rollback_timestamp, ts_string[1])); @@ -1425,8 +1427,8 @@ __rollback_to_stable_btree_apply( WT_TXN_GLOBAL *txn_global; wt_timestamp_t max_durable_ts, newest_start_durable_ts, newest_stop_durable_ts; size_t addr_size; - uint64_t rollback_txnid; - uint32_t btree_id, handle_open_flags; + uint64_t rollback_txnid, write_gen; + uint32_t btree_id; char ts_string[2][WT_TS_INT_STRING_SIZE]; bool dhandle_allocated, durable_ts_found, has_txn_updates_gt_than_ckpt_snap, perform_rts; bool prepared_updates; @@ -1436,8 +1438,9 @@ __rollback_to_stable_btree_apply( return (0); txn_global = &S2C(session)->txn_global; - rollback_txnid = 0; addr_size = 0; + rollback_txnid = 0; + write_gen = 0; dhandle_allocated = false; /* Find out the max durable timestamp of the object from checkpoint. */ @@ -1473,13 +1476,25 @@ __rollback_to_stable_btree_apply( if (ret == 0) addr_size = value.len; WT_RET_NOTFOUND_OK(ret); + ret = __wt_config_subgets(session, &cval, "write_gen", &value); + if (ret == 0) + write_gen = (uint64_t)value.val; + WT_RET_NOTFOUND_OK(ret); } max_durable_ts = WT_MAX(newest_start_durable_ts, newest_stop_durable_ts); - has_txn_updates_gt_than_ckpt_snap = WT_CHECK_RECOVERY_FLAG_TXNID(session, rollback_txnid); - /* Increment the inconsistent checkpoint stats counter. */ - if (has_txn_updates_gt_than_ckpt_snap) + /* + * Perform rollback to stable when the newest written transaction of the btree is greater than + * or equal to the checkpoint snapshot. The snapshot comparison is valid only when the btree + * write generation number is greater than the last checkpoint connection base write generation + * to confirm that the btree is modified in the previous restart cycle. + */ + if (WT_CHECK_RECOVERY_FLAG_TXNID(session, rollback_txnid) && + (write_gen >= S2C(session)->last_ckpt_base_write_gen)) { + has_txn_updates_gt_than_ckpt_snap = true; + /* Increment the inconsistent checkpoint stats counter. */ WT_STAT_CONN_DATA_INCR(session, txn_rts_inconsistent_ckpt); + } /* * The rollback to stable will skip the tables during recovery and shutdown in the following @@ -1491,7 +1506,7 @@ __rollback_to_stable_btree_apply( F_ISSET(S2C(session), WT_CONN_CLOSING_TIMESTAMP)) && (addr_size == 0 || (txn_global->stable_timestamp == WT_TS_NONE && max_durable_ts != WT_TS_NONE))) { - __wt_verbose(session, WT_VERB_RECOVERY_RTS(session), + __wt_verbose_multi(session, WT_VERB_RECOVERY_RTS(session), "skip rollback to stable on file %s because %s", uri, addr_size == 0 ? "its checkpoint address length is 0" : "it has timestamped updates and the stable timestamp is 0"); @@ -1514,22 +1529,13 @@ __rollback_to_stable_btree_apply( if (perform_rts || max_durable_ts > rollback_timestamp || prepared_updates || !durable_ts_found || has_txn_updates_gt_than_ckpt_snap) { - /* - * MongoDB does not close all open handles before calling rollback-to-stable; otherwise, - * don't permit that behavior, the application is likely making a mistake. - */ -#ifdef WT_STANDALONE_BUILD - handle_open_flags = WT_DHANDLE_DISCARD | WT_DHANDLE_EXCLUSIVE; -#else - handle_open_flags = 0; -#endif - ret = __wt_session_get_dhandle(session, uri, NULL, NULL, handle_open_flags); + ret = __wt_session_get_dhandle(session, uri, NULL, NULL, 0); if (ret != 0) WT_ERR_MSG(session, ret, "%s: unable to open handle%s", uri, ret == EBUSY ? ", error indicates handle is unavailable due to concurrent use" : ""); dhandle_allocated = true; - __wt_verbose(session, WT_VERB_RECOVERY_RTS(session), + __wt_verbose_multi(session, WT_VERB_RECOVERY_RTS(session), "tree rolled back with durable timestamp: %s, or when tree is modified: %s or " "prepared updates: %s or when durable time is not found: %s or txnid: %" PRIu64 " is greater than recovery checkpoint snap min: %s", @@ -1539,7 +1545,7 @@ __rollback_to_stable_btree_apply( has_txn_updates_gt_than_ckpt_snap ? "true" : "false"); WT_ERR(__rollback_to_stable_btree(session, rollback_timestamp)); } else - __wt_verbose(session, WT_VERB_RECOVERY_RTS(session), + __wt_verbose_multi(session, WT_VERB_RECOVERY_RTS(session), "%s: tree skipped with durable timestamp: %s and stable timestamp: %s or txnid: %" PRIu64, uri, __wt_timestamp_to_string(max_durable_ts, ts_string[0]), __wt_timestamp_to_string(rollback_timestamp, ts_string[1]), rollback_txnid); @@ -1639,7 +1645,7 @@ __rollback_to_stable_btree_apply_all(WT_SESSION_IMPL *session, wt_timestamp_t ro * detected. */ if (ret == ENOENT || (ret == WT_ERROR && F_ISSET(S2C(session), WT_CONN_DATA_CORRUPTION))) { - __wt_verbose(session, WT_VERB_RECOVERY_RTS(session), + __wt_verbose_multi(session, WT_VERB_RECOVERY_RTS(session), "%s: skipped performing rollback to stable because the file %s", uri, ret == ENOENT ? "does not exist" : "is corrupted."); continue; @@ -1686,13 +1692,13 @@ __rollback_to_stable(WT_SESSION_IMPL *session, bool no_ckpt) * without a lock would violate protocol. */ WT_ORDERED_READ(rollback_timestamp, txn_global->stable_timestamp); - __wt_verbose(session, WT_VERB_RECOVERY_RTS(session), + __wt_verbose_multi(session, WT_VERB_RECOVERY_RTS(session), "performing rollback to stable with stable timestamp: %s and oldest timestamp: %s", __wt_timestamp_to_string(rollback_timestamp, ts_string[0]), __wt_timestamp_to_string(txn_global->oldest_timestamp, ts_string[1])); if (F_ISSET(conn, WT_CONN_RECOVERING)) - __wt_verbose(session, WT_VERB_RECOVERY_RTS(session), + __wt_verbose_multi(session, WT_VERB_RECOVERY_RTS(session), "recovered checkpoint snapshot min: %" PRIu64 ", snapshot max: %" PRIu64 ", snapshot count: %" PRIu32, conn->recovery_ckpt_snap_min, conn->recovery_ckpt_snap_max, diff --git a/src/third_party/wiredtiger/src/txn/txn_timestamp.c b/src/third_party/wiredtiger/src/txn/txn_timestamp.c index c138f23fe57..784c2cacfed 100644 --- a/src/third_party/wiredtiger/src/txn/txn_timestamp.c +++ b/src/third_party/wiredtiger/src/txn/txn_timestamp.c @@ -715,19 +715,7 @@ __wt_txn_set_prepare_timestamp(WT_SESSION_IMPL *session, wt_timestamp_t prepare_ WT_RET_MSG(session, EINVAL, "commit timestamp should not have been set before the prepare timestamp"); - /* - * Allows setting the prepared timestamp smaller than or equal to the latest active read - * timestamp. This feature is necessary to find the largest key in the table even if that - * key has been deleted. Set this flag cautiously as it breaks repeated reads. - */ -#ifdef HAVE_DIAGNOSTIC - if (!F_ISSET(txn, WT_TXN_TS_ROUND_PREPARED)) - WT_RET(__txn_assert_after_reads(session, "prepare", prepare_ts)); - else - WT_RET(__wt_msg(session, - "Skip checking prepare timestamp %s against the latest active read timestamp.", - __wt_timestamp_to_string(prepare_ts, ts_string[0]))); -#endif + WT_RET(__txn_assert_after_reads(session, "prepare", prepare_ts)); /* * Check whether the prepare timestamp is less than the oldest timestamp. diff --git a/src/third_party/wiredtiger/src/utilities/util_dump.c b/src/third_party/wiredtiger/src/utilities/util_dump.c index 727cf132ffc..77aa5697370 100755 --- a/src/third_party/wiredtiger/src/utilities/util_dump.c +++ b/src/third_party/wiredtiger/src/utilities/util_dump.c @@ -590,7 +590,7 @@ get_dump_type(bool pretty, bool hex, bool json) else result = "print"; - return result; + return (result); } /* diff --git a/src/third_party/wiredtiger/src/utilities/util_main.c b/src/third_party/wiredtiger/src/utilities/util_main.c index 1d2d86e94ec..c4d1fa2fd67 100644 --- a/src/third_party/wiredtiger/src/utilities/util_main.c +++ b/src/third_party/wiredtiger/src/utilities/util_main.c @@ -306,7 +306,7 @@ open: } free(p); free(secretkey); - config = secretkey = p = NULL; + secretkey = p = NULL; /* If we only want to verify the metadata, that is done in wiredtiger_open. We're done. */ if (func == NULL && meta_verify) diff --git a/src/third_party/wiredtiger/test/checkpoint/workers.c b/src/third_party/wiredtiger/test/checkpoint/workers.c index 3c9313c4c99..babc8731b0a 100644 --- a/src/third_party/wiredtiger/test/checkpoint/workers.c +++ b/src/third_party/wiredtiger/test/checkpoint/workers.c @@ -239,6 +239,7 @@ worker_op(WT_CURSOR *cursor, uint64_t keyno, u_int new_val) return (WT_ROLLBACK); return (log_print_err("cursor.modify", ret, 1)); } + return (0); } else if (ret != WT_NOTFOUND) { if (ret == WT_ROLLBACK || ret == WT_PREPARE_CONFLICT) return (WT_ROLLBACK); diff --git a/src/third_party/wiredtiger/test/cppsuite/Makefile.am b/src/third_party/wiredtiger/test/cppsuite/Makefile.am index 8221f882014..31c9e0797a9 100644 --- a/src/third_party/wiredtiger/test/cppsuite/Makefile.am +++ b/src/third_party/wiredtiger/test/cppsuite/Makefile.am @@ -14,6 +14,7 @@ noinst_PROGRAMS= test_harness = test_harness/core/component.cxx \ test_harness/core/configuration.cxx \ test_harness/core/throttle.cxx \ + test_harness/util/api_const.cxx \ test_harness/util/logger.cxx \ test_harness/util/scoped_connection.cxx \ test_harness/util/scoped_types.cxx \ diff --git a/src/third_party/wiredtiger/test/cppsuite/configs/search_near_01_default.txt b/src/third_party/wiredtiger/test/cppsuite/configs/search_near_01_default.txt index b3b3d6c668a..7d4ed2c2bdb 100644 --- a/src/third_party/wiredtiger/test/cppsuite/configs/search_near_01_default.txt +++ b/src/third_party/wiredtiger/test/cppsuite/configs/search_near_01_default.txt @@ -19,4 +19,4 @@ workload_generator= ( thread_count=10 ) -),
\ No newline at end of file +), diff --git a/src/third_party/wiredtiger/test/cppsuite/configs/search_near_01_stress.txt b/src/third_party/wiredtiger/test/cppsuite/configs/search_near_01_stress.txt new file mode 100644 index 00000000000..31c8101861e --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/configs/search_near_01_stress.txt @@ -0,0 +1,22 @@ +# Configuration for search_near_01 stress test. +# The test will generate key_count_per_collection number of keys for each prefix in aaa -> zzz. +# This config will have a 30 minute duration, with 3 tables of an entry for each prefix. +duration_seconds=1800, +cache_size_mb=1000, +timestamp_manager= +( + enabled=false, +), +workload_generator= +( + populate_config= + ( + collection_count=3, + key_count_per_collection=100, + ), + read_config= + ( + op_rate=10ms, + thread_count=10 + ) +), diff --git a/src/third_party/wiredtiger/test/cppsuite/configs/search_near_02_stress.txt b/src/third_party/wiredtiger/test/cppsuite/configs/search_near_02_stress.txt new file mode 100644 index 00000000000..42ea7ab420d --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/configs/search_near_02_stress.txt @@ -0,0 +1,34 @@ +# Configuration for search_near_02 stress test. +# The configuration creates: +# - threads that continuously insert random keys and values. +# - threads that continuously perform prefix search_near calls on random keys. +duration_seconds=1800, +cache_size_mb=500, +timestamp_manager= +( + # This will let us randomly pick a read timestamp in a bigger range to trigger visibility + # checks. + oldest_lag=50, +), +workload_generator= +( + populate_config= + ( + collection_count=10, + ), + insert_config= + ( + key_size=5, + op_rate=10ms, + thread_count=10 + ), + read_config= + ( + # The key size parameter is used to generate the prefix + # used in search_near. Use a smaller size to increase + # the chances to match an existing key. + key_size=3, + op_rate=10ms, + thread_count=10 + ) +) diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/core/component.cxx b/src/third_party/wiredtiger/test/cppsuite/test_harness/core/component.cxx index b607f8a821d..05d6976ec4b 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/core/component.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/core/component.cxx @@ -30,7 +30,7 @@ #include "test_harness/util/api_const.h" namespace test_harness { -component::component(const std::string &name, configuration *config) : _name(name), _config(config) +component::component(const std::string &name, configuration *config) : _config(config), _name(name) { } diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/core/configuration.cxx b/src/third_party/wiredtiger/test/cppsuite/test_harness/core/configuration.cxx index 3a785f395ad..7bc5e6f5e2c 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/core/configuration.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/core/configuration.cxx @@ -101,12 +101,6 @@ configuration::~configuration() } } -const std::string & -configuration::get_config() const -{ - return (_config); -} - std::string configuration::get_string(const std::string &key) { @@ -163,12 +157,6 @@ configuration::get_list(const std::string &key) return get<std::vector<std::string>>(key, false, types::LIST, {}, config_item_to_list); } -std::vector<std::string> -configuration::get_optional_list(const std::string &key) -{ - return get<std::vector<std::string>>(key, true, types::LIST, {}, config_item_to_list); -} - template <typename T> T configuration::get( diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/test.cxx b/src/third_party/wiredtiger/test/cppsuite/test_harness/test.cxx index 8cdea3170df..774fcba7e6d 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/test.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/test.cxx @@ -107,7 +107,7 @@ test::run() statistics_config = _config->get_subconfig(STATISTICS_CONFIG); statistics_type = statistics_config->get_string(TYPE); statistics_logging = statistics_config->get_bool(ENABLE_LOGGING); - db_create_config += statistics_logging ? "," + std::string(STATISTICS_LOG) : ""; + db_create_config += statistics_logging ? "," + STATISTICS_LOG : ""; db_create_config += ",statistics=(" + statistics_type + ")"; /* Don't forget to delete. */ delete statistics_config; @@ -164,28 +164,4 @@ test::run() logger::log_msg(LOG_INFO, "SUCCESS"); } - -workload_generator * -test::get_workload_generator() -{ - return (_workload_generator); -} - -runtime_monitor * -test::get_runtime_monitor() -{ - return (_runtime_monitor); -} - -timestamp_manager * -test::get_timestamp_manager() -{ - return (_timestamp_manager); -} - -thread_manager * -test::get_thread_manager() -{ - return (_thread_manager); -} } // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/timestamp_manager.cxx b/src/third_party/wiredtiger/test/cppsuite/test_harness/timestamp_manager.cxx index 3b7d2d312aa..33ad4b80bac 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/timestamp_manager.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/timestamp_manager.cxx @@ -84,7 +84,7 @@ timestamp_manager::do_work() if ((latest_ts_s - _stable_ts) > _stable_lag) { log_msg = "Timestamp_manager: Stable timestamp expired."; _stable_ts = latest_ts_s; - config += std::string(STABLE_TS) + "=" + decimal_to_hex(_stable_ts); + config += STABLE_TS + "=" + decimal_to_hex(_stable_ts); } /* @@ -101,7 +101,7 @@ timestamp_manager::do_work() new_oldest_ts = _stable_ts - _oldest_lag; if (!config.empty()) config += ","; - config += std::string(OLDEST_TS) + "=" + decimal_to_hex(new_oldest_ts); + config += OLDEST_TS + "=" + decimal_to_hex(new_oldest_ts); } if (!log_msg.empty()) diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/util/api_const.cxx b/src/third_party/wiredtiger/test/cppsuite/test_harness/util/api_const.cxx new file mode 100644 index 00000000000..469885c280c --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/util/api_const.cxx @@ -0,0 +1,82 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "api_const.h" + +/* Define all constants related to WiredTiger APIs and testing. */ +namespace test_harness { + +/* Component names. */ +const std::string CHECKPOINT_MANAGER = "checkpoint_manager"; +const std::string RUNTIME_MONITOR = "runtime_monitor"; +const std::string TIMESTAMP_MANAGER = "timestamp_manager"; +const std::string WORKLOAD_GENERATOR = "workload_generator"; +const std::string WORKLOAD_TRACKING = "workload_tracking"; + +/* Configuration API consts. */ +const std::string CACHE_SIZE_MB = "cache_size_mb"; +const std::string COLLECTION_COUNT = "collection_count"; +const std::string COMPRESSION_ENABLED = "compression_enabled"; +const std::string DURATION_SECONDS = "duration_seconds"; +const std::string ENABLED = "enabled"; +const std::string ENABLE_LOGGING = "enable_logging"; +const std::string INSERT_CONFIG = "insert_config"; +const std::string KEY_COUNT_PER_COLLECTION = "key_count_per_collection"; +const std::string KEY_SIZE = "key_size"; +const std::string LIMIT = "limit"; +const std::string MAX = "max"; +const std::string MIN = "min"; +const std::string OLDEST_LAG = "oldest_lag"; +const std::string OP_RATE = "op_rate"; +const std::string OPS_PER_TRANSACTION = "ops_per_transaction"; +const std::string POPULATE_CONFIG = "populate_config"; +const std::string POSTRUN_STATISTICS = "postrun_statistics"; +const std::string READ_CONFIG = "read_config"; +const std::string STABLE_LAG = "stable_lag"; +const std::string STAT_CACHE_SIZE = "stat_cache_size"; +const std::string STAT_DB_SIZE = "stat_db_size"; +const std::string STATISTICS_CONFIG = "statistics_config"; +const std::string THREAD_COUNT = "thread_count"; +const std::string TYPE = "type"; +const std::string UPDATE_CONFIG = "update_config"; +const std::string VALUE_SIZE = "value_size"; + +/* WiredTiger API consts. */ +const std::string COMMIT_TS = "commit_timestamp"; +const std::string CONNECTION_CREATE = "create"; +const std::string OLDEST_TS = "oldest_timestamp"; +const std::string STABLE_TS = "stable_timestamp"; +const std::string STATISTICS_LOG = "statistics_log=(json,wait=1)"; + +/* Test harness consts. */ +const std::string DEFAULT_FRAMEWORK_SCHEMA = "key_format=S,value_format=S,"; +const std::string TABLE_OPERATION_TRACKING = "table:operation_tracking"; +const std::string TABLE_SCHEMA_TRACKING = "table:schema_tracking"; +const std::string STATISTICS_URI = "statistics:"; + +} // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/util/api_const.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/util/api_const.h index 1eef096aa29..ba7c8166ed7 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/util/api_const.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/util/api_const.h @@ -29,50 +29,52 @@ #ifndef API_CONST_H #define API_CONST_H +#include <string> + /* Define all constants related to WiredTiger APIs and testing. */ namespace test_harness { /* Component names. */ -static const char *CHECKPOINT_MANAGER = "checkpoint_manager"; -static const char *RUNTIME_MONITOR = "runtime_monitor"; -static const char *TIMESTAMP_MANAGER = "timestamp_manager"; -static const char *WORKLOAD_GENERATOR = "workload_generator"; -static const char *WORKLOAD_TRACKING = "workload_tracking"; +extern const std::string CHECKPOINT_MANAGER; +extern const std::string RUNTIME_MONITOR; +extern const std::string TIMESTAMP_MANAGER; +extern const std::string WORKLOAD_GENERATOR; +extern const std::string WORKLOAD_TRACKING; /* Configuration API consts. */ -static const char *CACHE_SIZE_MB = "cache_size_mb"; -static const char *COLLECTION_COUNT = "collection_count"; -static const char *COMPRESSION_ENABLED = "compression_enabled"; -static const char *DURATION_SECONDS = "duration_seconds"; -static const char *ENABLED = "enabled"; -static const char *ENABLE_LOGGING = "enable_logging"; -static const char *INSERT_CONFIG = "insert_config"; -static const char *KEY_COUNT_PER_COLLECTION = "key_count_per_collection"; -static const char *KEY_SIZE = "key_size"; -static const char *LIMIT = "limit"; -static const char *MAX = "max"; -static const char *MIN = "min"; -static const char *OLDEST_LAG = "oldest_lag"; -static const char *OP_RATE = "op_rate"; -static const char *OPS_PER_TRANSACTION = "ops_per_transaction"; -static const char *POPULATE_CONFIG = "populate_config"; -static const char *POSTRUN_STATISTICS = "postrun_statistics"; -static const char *READ_CONFIG = "read_config"; -static const char *STABLE_LAG = "stable_lag"; -static const char *STAT_CACHE_SIZE = "stat_cache_size"; -static const char *STAT_DB_SIZE = "stat_db_size"; -static const char *STATISTICS_CONFIG = "statistics_config"; -static const char *THREAD_COUNT = "thread_count"; -static const char *TYPE = "type"; -static const char *UPDATE_CONFIG = "update_config"; -static const char *VALUE_SIZE = "value_size"; +extern const std::string CACHE_SIZE_MB; +extern const std::string COLLECTION_COUNT; +extern const std::string COMPRESSION_ENABLED; +extern const std::string DURATION_SECONDS; +extern const std::string ENABLED; +extern const std::string ENABLE_LOGGING; +extern const std::string INSERT_CONFIG; +extern const std::string KEY_COUNT_PER_COLLECTION; +extern const std::string KEY_SIZE; +extern const std::string LIMIT; +extern const std::string MAX; +extern const std::string MIN; +extern const std::string OLDEST_LAG; +extern const std::string OP_RATE; +extern const std::string OPS_PER_TRANSACTION; +extern const std::string POPULATE_CONFIG; +extern const std::string POSTRUN_STATISTICS; +extern const std::string READ_CONFIG; +extern const std::string STABLE_LAG; +extern const std::string STAT_CACHE_SIZE; +extern const std::string STAT_DB_SIZE; +extern const std::string STATISTICS_CONFIG; +extern const std::string THREAD_COUNT; +extern const std::string TYPE; +extern const std::string UPDATE_CONFIG; +extern const std::string VALUE_SIZE; /* WiredTiger API consts. */ -static const char *COMMIT_TS = "commit_timestamp"; -static const char *CONNECTION_CREATE = "create"; -static const char *OLDEST_TS = "oldest_timestamp"; -static const char *STABLE_TS = "stable_timestamp"; -static const char *STATISTICS_LOG = "statistics_log=(json,wait=1)"; +extern const std::string COMMIT_TS; +extern const std::string CONNECTION_CREATE; +extern const std::string OLDEST_TS; +extern const std::string STABLE_TS; +extern const std::string STATISTICS_LOG; /* * Use the Snappy compressor for stress testing to avoid excessive disk space usage. Our CMake @@ -88,10 +90,10 @@ static const char *STATISTICS_LOG = "statistics_log=(json,wait=1)"; #define SNAPPY_EXT ",extensions=(" SNAPPY_PATH ")" /* Test harness consts. */ -static const char *DEFAULT_FRAMEWORK_SCHEMA = "key_format=S,value_format=S,"; -static const char *TABLE_OPERATION_TRACKING = "table:operation_tracking"; -static const char *TABLE_SCHEMA_TRACKING = "table:schema_tracking"; -static const char *STATISTICS_URI = "statistics:"; +extern const std::string DEFAULT_FRAMEWORK_SCHEMA; +extern const std::string TABLE_OPERATION_TRACKING; +extern const std::string TABLE_SCHEMA_TRACKING; +extern const std::string STATISTICS_URI; } // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/util/logger.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/util/logger.h index 2c35ef0d162..3b30ede7bd4 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/util/logger.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/util/logger.h @@ -60,6 +60,8 @@ namespace test_harness { */ #define LOG_TRACE 3 +void get_time(char *time_buf, size_t buf_size); + class logger { public: /* Current log level. Default is LOG_WARN. */ diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/util/scoped_types.cxx b/src/third_party/wiredtiger/test/cppsuite/test_harness/util/scoped_types.cxx index 599a8214be2..b0b20ccc0f8 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/util/scoped_types.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/util/scoped_types.cxx @@ -26,6 +26,7 @@ * OTHER DEALINGS IN THE SOFTWARE. */ +#include <string> #include <utility> #include "scoped_types.h" @@ -33,7 +34,7 @@ namespace test_harness { /* scoped_cursor implementation */ -scoped_cursor::scoped_cursor(WT_SESSION *session, const char *uri, const char *cfg) +scoped_cursor::scoped_cursor(WT_SESSION *session, const std::string &uri, const std::string &cfg) { reinit(session, uri, cfg); } @@ -45,8 +46,10 @@ scoped_cursor::scoped_cursor(scoped_cursor &&other) scoped_cursor::~scoped_cursor() { - if (_cursor != nullptr) + if (_cursor != nullptr) { testutil_check(_cursor->close(_cursor)); + _cursor = nullptr; + } } /* @@ -63,14 +66,16 @@ scoped_cursor::operator=(scoped_cursor &&other) } void -scoped_cursor::reinit(WT_SESSION *session, const char *uri, const char *cfg) +scoped_cursor::reinit(WT_SESSION *session, const std::string &uri, const std::string &cfg) { + testutil_assert(!uri.empty()); if (_cursor != nullptr) { testutil_check(_cursor->close(_cursor)); _cursor = nullptr; } if (session != nullptr) - testutil_check(session->open_cursor(session, uri, nullptr, cfg, &_cursor)); + testutil_check(session->open_cursor( + session, uri.c_str(), nullptr, cfg.empty() ? nullptr : cfg.c_str(), &_cursor)); } /* @@ -103,8 +108,10 @@ scoped_session::scoped_session(WT_CONNECTION *conn) scoped_session::~scoped_session() { - if (_session != nullptr) + if (_session != nullptr) { testutil_check(_session->close(_session, nullptr)); + _session = nullptr; + } } scoped_session::scoped_session(scoped_session &&other) @@ -155,7 +162,7 @@ scoped_session::get() } scoped_cursor -scoped_session::open_scoped_cursor(const char *uri, const char *cfg) +scoped_session::open_scoped_cursor(const std::string &uri, const std::string &cfg) { return (scoped_cursor(_session, uri, cfg)); } diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/util/scoped_types.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/util/scoped_types.h index 71fc24f6c26..c015c0c909b 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/util/scoped_types.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/util/scoped_types.h @@ -45,7 +45,7 @@ namespace test_harness { class scoped_cursor { public: scoped_cursor() = default; - explicit scoped_cursor(WT_SESSION *session, const char *uri, const char *cfg); + explicit scoped_cursor(WT_SESSION *session, const std::string &uri, const std::string &cfg); /* Moving is ok but copying is not. */ scoped_cursor(scoped_cursor &&other); @@ -56,7 +56,7 @@ class scoped_cursor { scoped_cursor(const scoped_cursor &) = delete; scoped_cursor &operator=(const scoped_cursor &) = delete; - void reinit(WT_SESSION *session, const char *uri, const char *cfg); + void reinit(WT_SESSION *session, const std::string &uri, const std::string &cfg); WT_CURSOR &operator*(); WT_CURSOR *operator->(); @@ -93,7 +93,7 @@ class scoped_session { WT_SESSION *get(); - scoped_cursor open_scoped_cursor(const char *uri, const char *cfg = nullptr); + scoped_cursor open_scoped_cursor(const std::string &uri, const std::string &cfg = ""); private: WT_SESSION *_session = nullptr; diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_model.cxx b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_model.cxx index 9692ff9183a..3e373f5c1bc 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_model.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_model.cxx @@ -39,7 +39,7 @@ namespace test_harness { /* collection class implementation */ collection::collection(const uint64_t id, const uint64_t key_count, const std::string &name) - : id(id), _key_count(key_count), name(name) + : name(name), id(id), _key_count(key_count) { } @@ -150,7 +150,7 @@ void database::set_create_config(bool use_compression) { _collection_create_config = use_compression ? - std::string(DEFAULT_FRAMEWORK_SCHEMA) + std::string(SNAPPY_BLK) : + DEFAULT_FRAMEWORK_SCHEMA + std::string(SNAPPY_BLK) : DEFAULT_FRAMEWORK_SCHEMA; } } // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_operation.cxx b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_operation.cxx index 402a1e5237e..1b9b12d689c 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_operation.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_operation.cxx @@ -48,7 +48,7 @@ populate_worker(thread_context *tc) * WiredTiger lets you open a cursor on a collection using the same pointer. When a session * is closed, WiredTiger APIs close the cursors too. */ - scoped_cursor cursor = tc->session.open_scoped_cursor(coll.name.c_str()); + scoped_cursor cursor = tc->session.open_scoped_cursor(coll.name); uint64_t j = 0; while (j < tc->key_count) { tc->transaction.begin(); @@ -142,13 +142,14 @@ database_operation::insert_operation(thread_context *tc) /* Collection cursor vector. */ std::vector<collection_cursor> ccv; uint64_t collection_count = tc->db.get_collection_count(); + testutil_assert(collection_count != 0); uint64_t collections_per_thread = collection_count / tc->thread_count; /* Must have unique collections for each thread. */ testutil_assert(collection_count % tc->thread_count == 0); for (int i = tc->id * collections_per_thread; i < (tc->id * collections_per_thread) + collections_per_thread && tc->running(); ++i) { collection &coll = tc->db.get_collection(i); - scoped_cursor cursor = tc->session.open_scoped_cursor(coll.name.c_str()); + scoped_cursor cursor = tc->session.open_scoped_cursor(coll.name); ccv.push_back({coll, std::move(cursor)}); } @@ -208,7 +209,7 @@ database_operation::read_operation(thread_context *tc) collection &coll = tc->db.get_random_collection(); if (cursors.find(coll.id) == cursors.end()) - cursors.emplace(coll.id, std::move(tc->session.open_scoped_cursor(coll.name.c_str()))); + cursors.emplace(coll.id, std::move(tc->session.open_scoped_cursor(coll.name))); /* Do a second lookup now that we know it exists. */ auto &cursor = cursors[coll.id]; @@ -265,7 +266,7 @@ database_operation::update_operation(thread_context *tc) "Thread {" + std::to_string(tc->id) + "} Creating cursor for collection: " + coll.name); /* Open a cursor for the chosen collection. */ - scoped_cursor cursor = tc->session.open_scoped_cursor(coll.name.c_str()); + scoped_cursor cursor = tc->session.open_scoped_cursor(coll.name); cursors.emplace(coll.id, std::move(cursor)); } @@ -287,7 +288,7 @@ database_operation::update_operation(thread_context *tc) /* Commit the current transaction if we're able to. */ if (tc->transaction.can_commit()) - WT_IGNORE_RET(tc->transaction.commit()); + WT_IGNORE_RET_BOOL(tc->transaction.commit()); } /* Make sure the last operation is rolled back now the work is finished. */ diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.cxx b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.cxx index 8c8c4f10f88..e8860acf522 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.cxx @@ -35,6 +35,22 @@ #include "thread_context.h" namespace test_harness { + +const std::string +type_string(thread_type type) +{ + switch (type) { + case thread_type::INSERT: + return ("insert"); + case thread_type::READ: + return ("read"); + case thread_type::UPDATE: + return ("update"); + default: + testutil_die(EINVAL, "unexpected thread_type: %d", static_cast<int>(type)); + } +} + /* transaction_context class implementation */ transaction_context::transaction_context( configuration *config, timestamp_manager *timestamp_manager, WT_SESSION *session) @@ -125,7 +141,7 @@ transaction_context::set_commit_timestamp(wt_timestamp_t ts) /* We don't want to set zero timestamps on transactions if we're not using timestamps. */ if (!_timestamp_manager->enabled()) return; - std::string config = std::string(COMMIT_TS) + "=" + timestamp_manager::decimal_to_hex(ts); + const std::string config = COMMIT_TS + "=" + timestamp_manager::decimal_to_hex(ts); testutil_check(_session->timestamp_transaction(_session, config.c_str())); } @@ -151,19 +167,18 @@ transaction_context::can_rollback() thread_context::thread_context(uint64_t id, thread_type type, configuration *config, scoped_session &&created_session, timestamp_manager *timestamp_manager, workload_tracking *tracking, database &dbase) - : id(id), type(type), db(dbase), tsm(timestamp_manager), tracking(tracking), - session(std::move(created_session)), - transaction(transaction_context(config, timestamp_manager, session.get())), - /* These won't exist for certain threads which is why we use optional here. */ + : /* These won't exist for certain threads which is why we use optional here. */ collection_count(config->get_optional_int(COLLECTION_COUNT, 1)), key_count(config->get_optional_int(KEY_COUNT_PER_COLLECTION, 1)), key_size(config->get_optional_int(KEY_SIZE, 1)), value_size(config->get_optional_int(VALUE_SIZE, 1)), - thread_count(config->get_int(THREAD_COUNT)) + thread_count(config->get_int(THREAD_COUNT)), type(type), id(id), db(dbase), + session(std::move(created_session)), tsm(timestamp_manager), + transaction(transaction_context(config, timestamp_manager, session.get())), + tracking(tracking), _throttle(config) { - _throttle = throttle(config); if (tracking->enabled()) - op_track_cursor = session.open_scoped_cursor(tracking->get_operation_table_name().c_str()); + op_track_cursor = session.open_scoped_cursor(tracking->get_operation_table_name()); testutil_assert(key_size > 0 && value_size > 0); } diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.h index 28c1ee4265b..38ce8a821ab 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.h @@ -46,20 +46,7 @@ class workload_tracking; namespace test_harness { enum thread_type { READ, INSERT, UPDATE }; -static std::string -type_string(thread_type type) -{ - switch (type) { - case thread_type::INSERT: - return ("insert"); - case thread_type::READ: - return ("read"); - case thread_type::UPDATE: - return ("update"); - default: - testutil_die(EINVAL, "unexpected thread_type: %d", static_cast<int>(type)); - } -} +const std::string type_string(thread_type type); class transaction_context { public: @@ -95,25 +82,25 @@ class transaction_context { bool can_rollback(); private: - /* - * op_count is the current number of operations that have been executed in the current - * transaction. - */ - int64_t _op_count = 0; + bool _in_txn = false; + bool _needs_rollback = false; /* * _min_op_count and _max_op_count are the minimum and maximum number of operations within one * transaction. is the current maximum number of operations that can be executed in the current * transaction. */ - int64_t _min_op_count = 0; int64_t _max_op_count = INT64_MAX; + int64_t _min_op_count = 0; + /* + * op_count is the current number of operations that have been executed in the current + * transaction. + */ + int64_t _op_count = 0; int64_t _target_op_count = 0; - bool _in_txn = false; - bool _needs_rollback = false; - WT_SESSION *_session = nullptr; timestamp_manager *_timestamp_manager = nullptr; + WT_SESSION *_session = nullptr; }; /* Container class for a thread and any data types it may need to interact with the database. */ @@ -157,24 +144,24 @@ class thread_context { bool running() const; public: - scoped_session session; - scoped_cursor op_track_cursor; - scoped_cursor stat_cursor; - transaction_context transaction; - timestamp_manager *tsm; - workload_tracking *tracking; - database &db; const int64_t collection_count; const int64_t key_count; const int64_t key_size; const int64_t value_size; const int64_t thread_count; - const uint64_t id; const thread_type type; + const uint64_t id; + database &db; + scoped_session session; + scoped_cursor op_track_cursor; + scoped_cursor stat_cursor; + timestamp_manager *tsm; + transaction_context transaction; + workload_tracking *tracking; private: - throttle _throttle; bool _running = true; + throttle _throttle; }; } // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/workload_tracking.cxx b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/workload_tracking.cxx index e0e7738590d..e8a2ebbcc45 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/workload_tracking.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/workload_tracking.cxx @@ -66,7 +66,7 @@ workload_tracking::load() _session = connection_manager::instance().create_session(); testutil_check( _session->create(_session.get(), _schema_table_name.c_str(), _schema_table_config.c_str())); - _schema_track_cursor = _session.open_scoped_cursor(_schema_table_name.c_str()); + _schema_track_cursor = _session.open_scoped_cursor(_schema_table_name); logger::log_msg(LOG_TRACE, "Schema tracking initiated"); /* Initiate operations tracking. */ @@ -78,7 +78,7 @@ workload_tracking::load() * Open sweep cursor. This cursor will be used to clear out obsolete data from the tracking * table. */ - _sweep_cursor = _session.open_scoped_cursor(_operation_table_name.c_str()); + _sweep_cursor = _session.open_scoped_cursor(_operation_table_name); logger::log_msg(LOG_TRACE, "Tracking table sweep initialized"); } diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/workload_validation.cxx b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/workload_validation.cxx index 833ec91d9e8..5f3976c0de1 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/workload_validation.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/workload_validation.cxx @@ -87,7 +87,7 @@ workload_validation::validate(const std::string &operation_table_name, /* Parse the tracking table. */ validation_collection current_collection_records; - scoped_cursor cursor = session.open_scoped_cursor(operation_table_name.c_str()); + scoped_cursor cursor = session.open_scoped_cursor(operation_table_name); while ((ret = cursor->next(cursor.get())) == 0) { testutil_check( cursor->get_key(cursor.get(), &tracked_collection_id, &tracked_key, &tracked_timestamp)); @@ -154,7 +154,7 @@ workload_validation::parse_schema_tracking_table(scoped_session &session, uint64_t key_collection_id; int value_operation_type; - scoped_cursor cursor = session.open_scoped_cursor(tracking_table_name.c_str()); + scoped_cursor cursor = session.open_scoped_cursor(tracking_table_name); while (cursor->next(cursor.get()) == 0) { testutil_check(cursor->get_key(cursor.get(), &key_collection_id, &key_timestamp)); @@ -246,7 +246,7 @@ workload_validation::verify_key_value(scoped_session &session, const uint64_t co const char *retrieved_value; scoped_cursor cursor = - session.open_scoped_cursor(database::build_collection_name(collection_id).c_str()); + session.open_scoped_cursor(database::build_collection_name(collection_id)); cursor->set_key(cursor.get(), key.c_str()); ret = cursor->search(cursor.get()); if (ret != 0) { diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/burst_inserts.cxx b/src/third_party/wiredtiger/test/cppsuite/tests/burst_inserts.cxx index 9d5318e7acb..81e0f717d6d 100644 --- a/src/third_party/wiredtiger/test/cppsuite/tests/burst_inserts.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/tests/burst_inserts.cxx @@ -58,8 +58,8 @@ class burst_inserts : public test { struct collection_cursor { collection_cursor( collection &coll, scoped_cursor &&write_cursor, scoped_cursor &&read_cursor) - : coll(coll), write_cursor(std::move(write_cursor)), - read_cursor(std::move(read_cursor)) + : coll(coll), read_cursor(std::move(read_cursor)), + write_cursor(std::move(write_cursor)) { } collection &coll; @@ -81,15 +81,14 @@ class burst_inserts : public test { * Create a reading cursor that will read random documents for every next call. This * will help generate cache pressure. */ - ccv.push_back({coll, std::move(tc->session.open_scoped_cursor(coll.name.c_str())), - std::move(tc->session.open_scoped_cursor(coll.name.c_str(), "next_random=true"))}); + ccv.push_back({coll, std::move(tc->session.open_scoped_cursor(coll.name)), + std::move(tc->session.open_scoped_cursor(coll.name, "next_random=true"))}); } uint64_t counter = 0; while (tc->running()) { uint64_t start_key = ccv[counter].coll.get_key_count(); uint64_t added_count = 0; - bool committed = true; auto &cc = ccv[counter]; auto burst_start = std::chrono::system_clock::now(); while (tc->running() && @@ -139,9 +138,7 @@ class burst_inserts : public test { logger::log_msg(LOG_TRACE, "Committed an insertion of " + std::to_string(added_count) + " keys."); cc.coll.increase_key_count(added_count); - start_key = cc.coll.get_key_count(); } - added_count = 0; } testutil_check(cc.write_cursor->reset(cc.write_cursor.get())); diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/csuite_style_example_test.cxx b/src/third_party/wiredtiger/test/cppsuite/tests/csuite_style_example_test.cxx index bcacf115665..50ca6f0da82 100644 --- a/src/third_party/wiredtiger/test/cppsuite/tests/csuite_style_example_test.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/tests/csuite_style_example_test.cxx @@ -46,6 +46,10 @@ extern "C" { using namespace test_harness; +/* Declarations to avoid the error raised by -Werror=missing-prototypes. */ +void insert_op(WT_CURSOR *cursor, int key_size, int value_size); +void read_op(WT_CURSOR *cursor, int key_size); + bool do_inserts = false; bool do_reads = false; @@ -93,7 +97,7 @@ main(int argc, char *argv[]) logger::log_msg(LOG_ERROR, "This could be an error."); /* Create a connection, set the cache size and specify the home directory. */ - const std::string conn_config = std::string(CONNECTION_CREATE) + ",cache_size=500MB"; + const std::string conn_config = CONNECTION_CREATE + ",cache_size=500MB"; const std::string home_dir = std::string(DEFAULT_DIR) + '_' + progname; /* * A smart pointer is used here so that the connection can automatically be closed by the @@ -109,8 +113,8 @@ main(int argc, char *argv[]) /* Create a collection. */ const std::string collection_name = "table:my_collection"; - testutil_check( - insert_session->create(insert_session, collection_name.c_str(), DEFAULT_FRAMEWORK_SCHEMA)); + testutil_check(insert_session->create( + insert_session, collection_name.c_str(), DEFAULT_FRAMEWORK_SCHEMA.c_str())); /* Open different cursors. */ WT_CURSOR *insert_cursor, *read_cursor; diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/hs_cleanup.cxx b/src/third_party/wiredtiger/test/cppsuite/tests/hs_cleanup.cxx index fc17f0c5efe..e3ead2e3818 100644 --- a/src/third_party/wiredtiger/test/cppsuite/tests/hs_cleanup.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/tests/hs_cleanup.cxx @@ -58,7 +58,7 @@ class hs_cleanup : public test { /* In this test each thread gets a single collection. */ testutil_assert(tc->db.get_collection_count() == tc->thread_count); - scoped_cursor cursor = tc->session.open_scoped_cursor(coll.name.c_str()); + scoped_cursor cursor = tc->session.open_scoped_cursor(coll.name); /* We don't know the keyrange we're operating over here so we can't be much smarter here. */ while (tc->running()) { diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/run.cxx b/src/third_party/wiredtiger/test/cppsuite/tests/run.cxx index 627921a9aa4..f86dfb77256 100755 --- a/src/third_party/wiredtiger/test/cppsuite/tests/run.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/tests/run.cxx @@ -40,7 +40,13 @@ #include "search_near_01.cxx" #include "search_near_02.cxx" -std::string +/* Declarations to avoid the error raised by -Werror=missing-prototypes. */ +const std::string parse_configuration_from_file(const std::string &filename); +void print_help(); +int64_t run_test( + const std::string &test_name, const std::string &config, const std::string &wt_open_config); + +const std::string parse_configuration_from_file(const std::string &filename) { std::string cfg, line, error; diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/search_near_01.cxx b/src/third_party/wiredtiger/test/cppsuite/tests/search_near_01.cxx index 20bcebff4a0..b61ee27c637 100644 --- a/src/third_party/wiredtiger/test/cppsuite/tests/search_near_01.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/tests/search_near_01.cxx @@ -63,7 +63,7 @@ class search_near_01 : public test_harness::test { */ for (int64_t i = 0; i < collections_per_thread; ++i) { collection &coll = tc->db.get_collection(i); - scoped_cursor cursor = tc->session.open_scoped_cursor(coll.name.c_str()); + scoped_cursor cursor = tc->session.open_scoped_cursor(coll.name); for (uint64_t j = 0; j < ALPHABET.size(); ++j) { for (uint64_t k = 0; k < ALPHABET.size(); ++k) { for (uint64_t count = 0; count < tc->key_count; ++count) { @@ -190,8 +190,8 @@ class search_near_01 : public test_harness::test { * per search near function call. The key we search near can be different in length, which * will increase the number of entries search by a factor of 26. */ - expected_entries = tc->thread_count * keys_per_prefix * 2 * - pow(ALPHABET.size(), PREFIX_KEY_LEN - srchkey_len); + expected_entries = + tc->thread_count * keys_per_prefix * pow(ALPHABET.size(), PREFIX_KEY_LEN - srchkey_len); /* * Read at timestamp 10, so that no keys are visible to this transaction. This allows prefix @@ -204,7 +204,7 @@ class search_near_01 : public test_harness::test { /* Get a collection and find a cached cursor. */ collection &coll = tc->db.get_random_collection(); if (cursors.find(coll.id) == cursors.end()) { - scoped_cursor cursor = tc->session.open_scoped_cursor(coll.name.c_str()); + scoped_cursor cursor = tc->session.open_scoped_cursor(coll.name); cursor->reconfigure(cursor.get(), "prefix_search=true"); cursors.emplace(coll.id, std::move(cursor)); } @@ -245,7 +245,24 @@ class search_near_01 : public test_harness::test { */ testutil_assert( (expected_entries + (2 * tc->thread_count)) >= entries_stat - prev_entries_stat); - testutil_assert(prefix_stat > prev_prefix_stat); + /* + * There is an edge case where we may not early exit the prefix search near call + * because the specified prefix matches the rest of the entries in the tree. + * + * In this test, the keys in our database start with prefixes aaa -> zzz. If we + * search with a prefix such as "z", we will not early exit the search near call + * because the rest of the keys will also start with "z" and match the prefix. The + * statistic will stay the same if we do not early exit search near. + * + * However, we still need to keep the assertion as >= rather than a strictly equals + * as the test is multithreaded and other threads may increment the statistic if + * they are searching with a different prefix that will early exit. + */ + if (srch_key == "z" || srch_key == "zz" || srch_key == "zzz") { + testutil_assert(prefix_stat >= prev_prefix_stat); + } else { + testutil_assert(prefix_stat > prev_prefix_stat); + } tc->transaction.add_op(); tc->sleep(); diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/search_near_02.cxx b/src/third_party/wiredtiger/test/cppsuite/tests/search_near_02.cxx index 58c3047dfa9..729aaef1dba 100644 --- a/src/third_party/wiredtiger/test/cppsuite/tests/search_near_02.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/tests/search_near_02.cxx @@ -90,7 +90,7 @@ class search_near_02 : public test_harness::test { for (uint64_t i = thread_offset; i < thread_offset + collections_per_thread && tc->running(); ++i) { collection &coll = tc->db.get_collection(i); - scoped_cursor cursor = tc->session.open_scoped_cursor(coll.name.c_str()); + scoped_cursor cursor = tc->session.open_scoped_cursor(coll.name); ccv.push_back({coll, std::move(cursor)}); } @@ -163,8 +163,7 @@ class search_near_02 : public test_harness::test { /* Find a cached cursor or create one if none exists. */ if (cursors.find(coll.id) == cursors.end()) { - cursors.emplace( - coll.id, std::move(tc->session.open_scoped_cursor(coll.name.c_str()))); + cursors.emplace(coll.id, std::move(tc->session.open_scoped_cursor(coll.name))); auto &cursor_prefix = cursors[coll.id]; /* The cached cursors have the prefix configuration enabled. */ testutil_check( @@ -211,7 +210,7 @@ class search_near_02 : public test_harness::test { } /* Open a cursor with the default configuration on the selected collection. */ - scoped_cursor cursor_default(tc->session.open_scoped_cursor(coll.name.c_str())); + scoped_cursor cursor_default(tc->session.open_scoped_cursor(coll.name)); /* Verify the prefix search_near output using the default cursor. */ validate_prefix_search_near( diff --git a/src/third_party/wiredtiger/test/csuite/random_abort/main.c b/src/third_party/wiredtiger/test/csuite/random_abort/main.c index ce935d26e3f..162b82ff547 100644 --- a/src/third_party/wiredtiger/test/csuite/random_abort/main.c +++ b/src/third_party/wiredtiger/test/csuite/random_abort/main.c @@ -346,7 +346,7 @@ fill_db(uint32_t nth) */ free(thr); free(td); - exit(EXIT_SUCCESS); + _exit(EXIT_SUCCESS); } extern int __wt_optind; @@ -694,12 +694,11 @@ main(int argc, char *argv[]) memset(&sa, 0, sizeof(sa)); sa.sa_handler = handler; testutil_checksys(sigaction(SIGCHLD, &sa, NULL)); - if ((pid = fork()) < 0) - testutil_die(errno, "fork"); + testutil_checksys((pid = fork()) < 0); if (pid == 0) { /* child */ fill_db(nth); - return (EXIT_SUCCESS); + /* NOTREACHED */ } /* parent */ @@ -739,10 +738,8 @@ main(int argc, char *argv[]) * here. */ printf("Kill child\n"); - if (kill(pid, SIGKILL) != 0) - testutil_die(errno, "kill"); - if (waitpid(pid, &status, 0) == -1) - testutil_die(errno, "waitpid"); + testutil_checksys(kill(pid, SIGKILL) != 0); + testutil_checksys(waitpid(pid, &status, 0) == -1); } /* * !!! If we wanted to take a copy of the directory before recovery, diff --git a/src/third_party/wiredtiger/test/csuite/random_directio/main.c b/src/third_party/wiredtiger/test/csuite/random_directio/main.c index 5f60b18bab4..6431c98a548 100644 --- a/src/third_party/wiredtiger/test/csuite/random_directio/main.c +++ b/src/third_party/wiredtiger/test/csuite/random_directio/main.c @@ -622,7 +622,7 @@ fill_db(uint32_t nth, uint32_t datasize, const char *method, uint32_t flags) */ free(thr); free(td); - exit(EXIT_SUCCESS); + _exit(EXIT_SUCCESS); } /* @@ -775,12 +775,9 @@ kill_child(pid_t pid) * abort, then signal continue so that the child process will process the abort and dump core. */ printf("Send abort to child process ID %d\n", (int)pid); - if (kill(pid, SIGABRT) != 0) - testutil_die(errno, "kill"); - if (kill(pid, SIGCONT) != 0) - testutil_die(errno, "kill"); - if (waitpid(pid, &status, 0) == -1) - testutil_die(errno, "waitpid"); + testutil_checksys(kill(pid, SIGABRT) != 0); + testutil_checksys(kill(pid, SIGCONT) != 0); + testutil_checksys(waitpid(pid, &status, 0) == -1); } /* @@ -1001,7 +998,7 @@ handler(int sig) int status, termsig; WT_UNUSED(sig); - pid = waitpid(-1, &status, WNOHANG | WUNTRACED); + testutil_checksys((pid = waitpid(-1, &status, WNOHANG | WUNTRACED)) == -1); if (pid == 0) return; /* Nothing to wait for. */ if (WIFSTOPPED(status)) @@ -1211,12 +1208,11 @@ main(int argc, char *argv[]) memset(&sa, 0, sizeof(sa)); sa.sa_handler = handler; testutil_checksys(sigaction(SIGCHLD, &sa, NULL)); - if ((pid = fork()) < 0) - testutil_die(errno, "fork"); + testutil_checksys((pid = fork()) < 0); } if (pid == 0) { /* child, or populate_only */ fill_db(nth, datasize, method, flags); - return (EXIT_SUCCESS); + /* NOTREACHED */ } /* parent */ @@ -1247,10 +1243,8 @@ main(int argc, char *argv[]) printf("Kill child\n"); sa.sa_handler = SIG_DFL; testutil_checksys(sigaction(SIGCHLD, &sa, NULL)); - if (kill(pid, SIGKILL) != 0) - testutil_die(errno, "kill"); - if (waitpid(pid, &status, 0) == -1) - testutil_die(errno, "waitpid"); + testutil_checksys(kill(pid, SIGKILL) != 0); + testutil_checksys(waitpid(pid, &status, 0) == -1); } if (verify_only && !check_db(nth, datasize, 0, false, flags)) { printf("FAIL\n"); diff --git a/src/third_party/wiredtiger/test/csuite/schema_abort/main.c b/src/third_party/wiredtiger/test/csuite/schema_abort/main.c index 072192eee0a..9322783581b 100644 --- a/src/third_party/wiredtiger/test/csuite/schema_abort/main.c +++ b/src/third_party/wiredtiger/test/csuite/schema_abort/main.c @@ -837,7 +837,7 @@ run_workload(uint32_t nth) */ free(thr); free(td); - exit(EXIT_SUCCESS); + _exit(EXIT_SUCCESS); } extern int __wt_optind; @@ -999,7 +999,7 @@ main(int argc, char *argv[]) if (pid == 0) { /* child */ run_workload(nth); - return (EXIT_SUCCESS); + /* NOTREACHED */ } /* parent */ diff --git a/src/third_party/wiredtiger/test/csuite/schema_abort/smoke.sh b/src/third_party/wiredtiger/test/csuite/schema_abort/smoke.sh index e7d21ec30e6..bed880271d6 100755 --- a/src/third_party/wiredtiger/test/csuite/schema_abort/smoke.sh +++ b/src/third_party/wiredtiger/test/csuite/schema_abort/smoke.sh @@ -27,8 +27,7 @@ $TEST_WRAPPER $test_bin -c -m -t 10 -T 5 $TEST_WRAPPER $test_bin -c -C -t 10 -T 5 $TEST_WRAPPER $test_bin -c -C -m -t 10 -T 5 -# FIXME: In WT-6116 the test is failing if timestamps are turned off. -#$TEST_WRAPPER $test_bin -m -t 10 -T 5 -z -#$TEST_WRAPPER $test_bin -c -m -t 10 -T 5 -z +$TEST_WRAPPER $test_bin -m -t 10 -T 5 -z +$TEST_WRAPPER $test_bin -c -m -t 10 -T 5 -z $TEST_WRAPPER $test_bin -m -t 10 -T 5 -x $TEST_WRAPPER $test_bin -c -m -t 10 -T 5 -x diff --git a/src/third_party/wiredtiger/test/csuite/tiered_abort/main.c b/src/third_party/wiredtiger/test/csuite/tiered_abort/main.c index 8111cd108ac..f3afd42ba5f 100644 --- a/src/third_party/wiredtiger/test/csuite/tiered_abort/main.c +++ b/src/third_party/wiredtiger/test/csuite/tiered_abort/main.c @@ -70,7 +70,9 @@ static char home[1024]; /* Program working dir */ #define RECORDS_FILE "records-%" PRIu32 /* Include worker threads and extra sessions */ #define SESSION_MAX (MAX_TH + 4) +#ifndef WT_STORAGE_LIB #define WT_STORAGE_LIB "ext/storage_sources/local_store/.libs/libwiredtiger_local_store.so" +#endif static const char *table_pfx = "table"; static const char *const uri_collection = "collection"; @@ -509,7 +511,7 @@ run_workload(uint32_t nth, const char *build_dir) */ free(thr); free(td); - exit(EXIT_SUCCESS); + _exit(EXIT_SUCCESS); } extern int __wt_optind; @@ -577,7 +579,8 @@ main(int argc, char *argv[]) bool fatal, rand_th, rand_time, verify_only; (void)testutil_set_progname(argv); - + opts = &_opts; + memset(opts, 0, sizeof(*opts)); use_ts = true; nth = MIN_TH; rand_th = rand_time = true; @@ -585,8 +588,11 @@ main(int argc, char *argv[]) verify_only = false; working_dir = "WT_TEST.tiered-abort"; - while ((ch = __wt_getopt(progname, argc, argv, "f:h:T:t:vz")) != EOF) + while ((ch = __wt_getopt(progname, argc, argv, "b:f:h:T:t:vz")) != EOF) switch (ch) { + case 'b': /* Build directory */ + opts->build_dir = dstrdup(__wt_optarg); + break; case 'f': flush_calls = (uint32_t)atoi(__wt_optarg); break; @@ -624,8 +630,6 @@ main(int argc, char *argv[]) * the opts variable other than for building the directory. We have already parsed the args * we're interested in above. */ - opts = &_opts; - memset(opts, 0, sizeof(*opts)); testutil_check(testutil_parse_opts(argc, argv, opts)); testutil_build_dir(opts, build_dir, 512); @@ -674,7 +678,7 @@ main(int argc, char *argv[]) if (pid == 0) { /* child */ run_workload(nth, build_dir); - return (EXIT_SUCCESS); + /* NOTREACHED */ } /* parent */ diff --git a/src/third_party/wiredtiger/test/csuite/tiered_abort/smoke.sh b/src/third_party/wiredtiger/test/csuite/tiered_abort/smoke.sh index b853c4fcd5e..98431781c2f 100755 --- a/src/third_party/wiredtiger/test/csuite/tiered_abort/smoke.sh +++ b/src/third_party/wiredtiger/test/csuite/tiered_abort/smoke.sh @@ -19,6 +19,3 @@ else test_bin=$top_builddir/test/csuite/test_tiered_abort fi $TEST_WRAPPER $test_bin -t 10 -T 5 -$TEST_WRAPPER $test_bin -m -t 10 -T 5 -$TEST_WRAPPER $test_bin -C -t 10 -T 5 -$TEST_WRAPPER $test_bin -C -m -t 10 -T 5 diff --git a/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c b/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c index 5f5c019d3ad..951d4119e5d 100644 --- a/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c +++ b/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c @@ -575,7 +575,7 @@ run_workload(uint32_t nth) */ free(thr); free(td); - exit(EXIT_SUCCESS); + _exit(EXIT_SUCCESS); } extern int __wt_optind; @@ -745,7 +745,7 @@ main(int argc, char *argv[]) if (pid == 0) { /* child */ run_workload(nth); - return (EXIT_SUCCESS); + /* NOTREACHED */ } /* parent */ diff --git a/src/third_party/wiredtiger/test/csuite/truncated_log/main.c b/src/third_party/wiredtiger/test/csuite/truncated_log/main.c index fa45e573781..e10b74bdd1a 100644 --- a/src/third_party/wiredtiger/test/csuite/truncated_log/main.c +++ b/src/third_party/wiredtiger/test/csuite/truncated_log/main.c @@ -218,8 +218,7 @@ fill_db(void) } if (fclose(fp) != 0) testutil_die(errno, "fclose"); - exit(0); - /* NOTREACHED */ + _exit(EXIT_SUCCESS); } extern int __wt_optind; @@ -263,18 +262,16 @@ main(int argc, char *argv[]) /* * Fork a child to do its work. Wait for it to exit. */ - if ((pid = fork()) < 0) - testutil_die(errno, "fork"); + testutil_checksys((pid = fork()) < 0); if (pid == 0) { /* child */ fill_db(); - return (EXIT_SUCCESS); + /* NOTREACHED */ } /* parent */ /* Wait for child to kill itself. */ - if (waitpid(pid, &status, 0) == -1) - testutil_die(errno, "waitpid"); + testutil_checksys(waitpid(pid, &status, 0) == -1); /* * !!! If we wanted to take a copy of the directory before recovery, diff --git a/src/third_party/wiredtiger/test/csuite/wt1965_col_efficiency/main.c b/src/third_party/wiredtiger/test/csuite/wt1965_col_efficiency/main.c index b01f2e40bfc..ea4a39d8378 100644 --- a/src/third_party/wiredtiger/test/csuite/wt1965_col_efficiency/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt1965_col_efficiency/main.c @@ -115,7 +115,8 @@ main(int argc, char *argv[]) testutil_check(wiredtiger_open(opts->home, NULL, "create,cache_size=1G,checkpoint=(wait=30),eviction_trigger=80,eviction_target=64,eviction_" - "dirty_target=65,log=(enabled,file_max=10M),transaction_sync=(enabled=true,method=none)", + "dirty_target=65,eviction_dirty_trigger=80,log=(enabled,file_max=10M),transaction_sync=(" + "enabled=true,method=none)", &opts->conn)); testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &session)); diff --git a/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c b/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c index ce64967197a..f46eb1e95b0 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c @@ -89,7 +89,7 @@ static void generate_value(uint32_t, uint64_t, char *, int *, int *, int *, char static void run_check_subtest(TEST_OPTS *, const char *, uint64_t, bool, uint64_t *); static int run_check_subtest_range(TEST_OPTS *, const char *, bool); static void run_check_subtest_range_retry(TEST_OPTS *, const char *, bool); -static int run_process(TEST_OPTS *, const char *, char *[], int *); +static void run_process(TEST_OPTS *, const char *, char *[], int *); static void subtest_main(int, char *[], bool); static void subtest_populate(TEST_OPTS *, bool); @@ -332,8 +332,7 @@ run_check_subtest( if (opts->verbose) printf("running a separate process with %" PRIu64 " operations until fail...\n", nops); testutil_clean_work_dir(opts->home); - testutil_check( - run_process(opts, debugger != NULL ? debugger : opts->argv0, subtest_args, &estatus)); + run_process(opts, debugger != NULL ? debugger : opts->argv0, subtest_args, &estatus); if (opts->verbose) printf("process exited %d\n", estatus); @@ -443,8 +442,8 @@ run_check_subtest_range_retry(TEST_OPTS *opts, const char *debugger, bool close_ * run_process -- * Run a program with arguments, wait until it completes. */ -static int -run_process(TEST_OPTS *opts, const char *prog, char *argv[], int *status) +static void +run_process(TEST_OPTS *opts, const char *prog, char *argv[], int *statusp) { int pid; char **arg; @@ -455,14 +454,13 @@ run_process(TEST_OPTS *opts, const char *prog, char *argv[], int *status) printf("%s ", *arg); printf("\n"); } - if ((pid = fork()) == 0) { + testutil_checksys((pid = fork()) < 0); + if (pid == 0) { (void)execv(prog, argv); - testutil_die(errno, "%s", prog); - } else if (pid < 0) - return (errno); + _exit(EXIT_FAILURE); + } - (void)waitpid(pid, status, 0); - return (0); + testutil_checksys(waitpid(pid, statusp, 0) == -1); } /* diff --git a/src/third_party/wiredtiger/test/csuite/wt4803_history_store_abort/main.c b/src/third_party/wiredtiger/test/csuite/wt4803_history_store_abort/main.c index be275c4de28..c1e3ae8dcb1 100644 --- a/src/third_party/wiredtiger/test/csuite/wt4803_history_store_abort/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt4803_history_store_abort/main.c @@ -66,7 +66,7 @@ handle_message(WT_EVENT_HANDLER *handler, WT_SESSION *session, int error, const * panic. */ if (expect_panic) - exit(EXIT_SUCCESS); + _exit(EXIT_SUCCESS); } return (0); @@ -82,6 +82,12 @@ hs_workload(TEST_OPTS *opts, const char *hs_file_max) int i; char buf[WT_MEGABYTE], open_config[128]; + /* + * We're going to run this workload for different configurations of file_max. So clean out the + * work directory each time. + */ + testutil_make_work_dir(opts->home); + testutil_check(__wt_snprintf(open_config, sizeof(open_config), "create,cache_size=50MB,history_store=(file_max=%s)", hs_file_max)); @@ -125,37 +131,25 @@ hs_workload(TEST_OPTS *opts, const char *hs_file_max) * Cleanup. We do not get here when the file_max size is small because we will have already hit * the maximum and exited. This code only executes on the successful path. */ - testutil_check(other_session->rollback_transaction(other_session, NULL)); - testutil_check(other_session->close(other_session, NULL)); - - testutil_check(cursor->close(cursor)); - testutil_check(session->close(session, NULL)); + testutil_check(opts->conn->close(opts->conn, NULL)); } -static int +static void test_hs_workload(TEST_OPTS *opts, const char *hs_file_max) { pid_t pid; int status; /* - * We're going to run this workload for different configurations of file_max. So clean out the - * work directory each time. - */ - testutil_make_work_dir(opts->home); - - /* * Since it's possible that the workload will panic and abort, we will fork the process and * execute the workload in the child process. * * This way, we can safely check the exit code of the child process and confirm that it is what * we expected. */ - pid = fork(); - if (pid < 0) - /* Failed fork. */ - testutil_die(errno, "fork"); - else if (pid == 0) { + testutil_checksys((pid = fork()) < 0); + + if (pid == 0) { /* Child process from here. */ hs_workload(opts, hs_file_max); @@ -167,16 +161,14 @@ test_hs_workload(TEST_OPTS *opts, const char *hs_file_max) expect_panic ? "true" : "false"); if (expect_panic) - exit(EXIT_FAILURE); + _exit(EXIT_FAILURE); else - exit(EXIT_SUCCESS); + _exit(EXIT_SUCCESS); } /* Parent process from here. */ - if (waitpid(pid, &status, 0) == -1) - testutil_die(errno, "waitpid"); - - return (status); + testutil_checksys(waitpid(pid, &status, 0) == -1); + testutil_assert(status == 0); } int @@ -192,23 +184,23 @@ main(int argc, char **argv) * needed. */ expect_panic = false; - testutil_check(test_hs_workload(&opts, "0")); + test_hs_workload(&opts, "0"); /* * The history store is limited to 5GB. This is more than enough for this workload so we don't * expect any failure. */ expect_panic = false; - testutil_check(test_hs_workload(&opts, "5GB")); + test_hs_workload(&opts, "5GB"); /* * The history store is limited to 100MB. This is insufficient for this workload so we're * expecting a failure. */ expect_panic = true; - testutil_check(test_hs_workload(&opts, "100MB")); + test_hs_workload(&opts, "100MB"); testutil_cleanup(&opts); - return (0); + return (EXIT_SUCCESS); } diff --git a/src/third_party/wiredtiger/test/csuite/wt6616_checkpoint_oldest_ts/main.c b/src/third_party/wiredtiger/test/csuite/wt6616_checkpoint_oldest_ts/main.c index 2e9648efea0..96454890e85 100644 --- a/src/third_party/wiredtiger/test/csuite/wt6616_checkpoint_oldest_ts/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt6616_checkpoint_oldest_ts/main.c @@ -228,7 +228,7 @@ run_workload(void) /* NOTREACHED */ free(thr); - exit(EXIT_SUCCESS); + _exit(EXIT_SUCCESS); } /* @@ -316,7 +316,7 @@ main(int argc, char *argv[]) if (pid == 0) { /* child */ run_workload(); - return (EXIT_SUCCESS); + /* NOTREACHED */ } /* parent */ diff --git a/src/third_party/wiredtiger/test/csuite/wt7989_compact_checkpoint/main.c b/src/third_party/wiredtiger/test/csuite/wt7989_compact_checkpoint/main.c index 8cd8c4b3e77..057c1c9737c 100644 --- a/src/third_party/wiredtiger/test/csuite/wt7989_compact_checkpoint/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt7989_compact_checkpoint/main.c @@ -45,8 +45,10 @@ * better view on what is happening. */ static const char conn_config[] = "create,cache_size=2GB,statistics=(all)"; -static const char table_config[] = - "allocation_size=4KB,leaf_page_max=4KB,key_format=i,value_format=QQQS"; +static const char table_config_row[] = + "allocation_size=4KB,leaf_page_max=4KB,key_format=Q,value_format=QQQS"; +static const char table_config_col[] = + "allocation_size=4KB,leaf_page_max=4KB,key_format=r,value_format=QQQS"; static char data_str[1024] = ""; static pthread_t thread_compact; @@ -58,42 +60,49 @@ struct thread_data { }; /* Forward declarations. */ -static void run_test(bool stress_test, const char *home, const char *uri); -static void *thread_func_compact(void *arg); -static void *thread_func_checkpoint(void *arg); -static void populate(WT_SESSION *session, const char *uri); -static void remove_records(WT_SESSION *session, const char *uri); -static uint64_t get_file_size(WT_SESSION *session, const char *uri); -static void set_timing_stress_checkpoint(WT_CONNECTION *conn); +static void run_test_clean(bool, bool, bool, const char *, const char *, const char *ri); +static void run_test(bool, bool, const char *, const char *); +static void *thread_func_compact(void *); +static void *thread_func_checkpoint(void *); +static void populate(WT_SESSION *, const char *); +static void remove_records(WT_SESSION *, const char *); +static void get_file_stats(WT_SESSION *, const char *, uint64_t *, uint64_t *); +static void set_timing_stress_checkpoint(WT_CONNECTION *); +static bool check_db_size(WT_SESSION *, const char *); +static void get_compact_progress( + WT_SESSION *session, const char *, uint64_t *, uint64_t *, uint64_t *); /* Methods implementation. */ int main(int argc, char *argv[]) { TEST_OPTS *opts, _opts; - char home_cv[512]; opts = &_opts; memset(opts, 0, sizeof(*opts)); testutil_check(testutil_parse_opts(argc, argv, opts)); /* - * First run test with WT_TIMING_STRESS_CHECKPOINT_SLOW. + * First, run test with WT_TIMING_STRESS_CHECKPOINT_SLOW. Row store case. */ - printf("Running stress test...\n"); - run_test(true, opts->home, opts->uri); + run_test_clean(true, false, opts->preserve, opts->home, "SR", opts->uri); /* - * Now run test where compact and checkpoint threads are synchronized using condition variable. + * Now, run test where compact and checkpoint threads are synchronized using condition variable. + * Row store case. */ - printf("Running normal test...\n"); - testutil_assert(sizeof(home_cv) > strlen(opts->home) + 3); - sprintf(home_cv, "%s.CV", opts->home); - run_test(false, home_cv, opts->uri); + run_test_clean(false, false, opts->preserve, opts->home, "NR", opts->uri); - /* Cleanup */ - if (!opts->preserve) - testutil_clean_work_dir(home_cv); + /* + * Next, run test with WT_TIMING_STRESS_CHECKPOINT_SLOW. Column store case. + */ + run_test_clean(true, true, opts->preserve, opts->home, "SC", opts->uri); + + /* + * Finally, run test where compact and checkpoint threads are synchronized using condition + * variable. Column store case. + */ + run_test_clean(false, true, opts->preserve, opts->home, "NC", opts->uri); testutil_cleanup(opts); @@ -101,13 +110,32 @@ main(int argc, char *argv[]) } static void -run_test(bool stress_test, const char *home, const char *uri) +run_test_clean(bool stress_test, bool column_store, bool preserve, const char *home, + const char *suffix, const char *uri) +{ + char home_full[512]; + + printf("\n"); + printf("Running %s test with %s store...\n", stress_test ? "stress" : "normal", + column_store ? "column" : "row"); + testutil_assert(sizeof(home_full) > strlen(home) + strlen(suffix) + 2); + sprintf(home_full, "%s.%s", home, suffix); + run_test(stress_test, column_store, home_full, uri); + + /* Cleanup */ + if (!preserve) + testutil_clean_work_dir(home_full); +} + +static void +run_test(bool stress_test, bool column_store, const char *home, const char *uri) { struct thread_data td; WT_CONNECTION *conn; WT_SESSION *session; pthread_t thread_checkpoint; - uint64_t file_sz_after, file_sz_before; + uint64_t pages_reviewed, pages_rewritten, pages_skipped; + bool size_check_res; testutil_make_work_dir(home); testutil_check(wiredtiger_open(home, NULL, conn_config, &conn)); @@ -123,7 +151,8 @@ run_test(bool stress_test, const char *home, const char *uri) testutil_check(conn->open_session(conn, NULL, NULL, &session)); /* Create and populate table. Checkpoint the data after that. */ - testutil_check(session->create(session, uri, table_config)); + testutil_check( + session->create(session, uri, column_store ? table_config_col : table_config_row)); populate(session, uri); testutil_check(session->checkpoint(session, NULL)); @@ -134,8 +163,6 @@ run_test(bool stress_test, const char *home, const char *uri) */ remove_records(session, uri); - file_sz_before = get_file_size(session, uri); - td.conn = conn; td.uri = uri; td.cond = NULL; @@ -156,7 +183,9 @@ run_test(bool stress_test, const char *home, const char *uri) (void)pthread_join(thread_checkpoint, NULL); (void)pthread_join(thread_compact, NULL); - file_sz_after = get_file_size(session, uri); + /* Collect compact progress stats. */ + get_compact_progress(session, uri, &pages_reviewed, &pages_rewritten, &pages_skipped); + size_check_res = check_db_size(session, uri); /* Cleanup */ if (!stress_test) { @@ -170,12 +199,16 @@ run_test(bool stress_test, const char *home, const char *uri) testutil_check(conn->close(conn, NULL)); conn = NULL; - /* Check if there's at least 10% compaction. */ - printf(" - Compressed file size MB: %f\n - Original file size MB: %f\n", - file_sz_after / (1024.0 * 1024), file_sz_before / (1024.0 * 1024)); - - /* Make sure the compact operation has reduced the file size by at least 20%. */ - testutil_assert((file_sz_before / 100) * 80 > file_sz_after); + printf(" - Pages reviewed: %" PRIu64 "\n", pages_reviewed); + printf(" - Pages selected for being rewritten: %" PRIu64 "\n", pages_rewritten); + printf(" - Pages skipped: %" PRIu64 "\n", pages_skipped); + testutil_assert(pages_reviewed > 0); + testutil_assert(pages_rewritten > 0); + /* + * Check if there's more than 10% available space in the file. Checking result here to allow + * connection to close properly. + */ + testutil_assert(size_check_res); } static void * @@ -274,20 +307,19 @@ populate(WT_SESSION *session, const char *uri) { WT_CURSOR *cursor; WT_RAND_STATE rnd; - uint64_t val; - int i, str_len; + uint64_t i, str_len, val; __wt_random_init_seed((WT_SESSION_IMPL *)session, &rnd); str_len = sizeof(data_str) / sizeof(data_str[0]); for (i = 0; i < str_len - 1; i++) - data_str[i] = 'a' + __wt_random(&rnd) % 26; + data_str[i] = 'a' + (uint32_t)__wt_random(&rnd) % 26; data_str[str_len - 1] = '\0'; testutil_check(session->open_cursor(session, uri, NULL, NULL, &cursor)); for (i = 0; i < NUM_RECORDS; i++) { - cursor->set_key(cursor, i); + cursor->set_key(cursor, i + 1); val = (uint64_t)__wt_random(&rnd); cursor->set_value(cursor, val, val, val, data_str); testutil_check(cursor->insert(cursor)); @@ -301,13 +333,13 @@ static void remove_records(WT_SESSION *session, const char *uri) { WT_CURSOR *cursor; - int i; + uint64_t i; testutil_check(session->open_cursor(session, uri, NULL, NULL, &cursor)); /* Remove 1/3 of the records from the middle of the key range. */ for (i = NUM_RECORDS / 3; i < (NUM_RECORDS * 2) / 3; i++) { - cursor->set_key(cursor, i); + cursor->set_key(cursor, i + 1); testutil_check(cursor->remove(cursor)); } @@ -315,22 +347,27 @@ remove_records(WT_SESSION *session, const char *uri) cursor = NULL; } -static uint64_t -get_file_size(WT_SESSION *session, const char *uri) +static void +get_file_stats(WT_SESSION *session, const char *uri, uint64_t *file_sz, uint64_t *avail_bytes) { WT_CURSOR *cur_stat; - uint64_t val; char *descr, *str_val, stat_uri[128]; sprintf(stat_uri, "statistics:%s", uri); testutil_check(session->open_cursor(session, stat_uri, NULL, "statistics=(all)", &cur_stat)); + + /* Get file size. */ cur_stat->set_key(cur_stat, WT_STAT_DSRC_BLOCK_SIZE); testutil_check(cur_stat->search(cur_stat)); - testutil_check(cur_stat->get_value(cur_stat, &descr, &str_val, &val)); + testutil_check(cur_stat->get_value(cur_stat, &descr, &str_val, file_sz)); + + /* Get bytes available for reuse. */ + cur_stat->set_key(cur_stat, WT_STAT_DSRC_BLOCK_REUSE_BYTES); + testutil_check(cur_stat->search(cur_stat)); + testutil_check(cur_stat->get_value(cur_stat, &descr, &str_val, avail_bytes)); + testutil_check(cur_stat->close(cur_stat)); cur_stat = NULL; - - return (val); } static void @@ -341,3 +378,44 @@ set_timing_stress_checkpoint(WT_CONNECTION *conn) conn_impl = (WT_CONNECTION_IMPL *)conn; conn_impl->timing_stress_flags |= WT_TIMING_STRESS_CHECKPOINT_SLOW; } + +static void +get_compact_progress(WT_SESSION *session, const char *uri, uint64_t *pages_reviewed, + uint64_t *pages_skipped, uint64_t *pages_rewritten) +{ + + WT_CURSOR *cur_stat; + char *descr, *str_val; + char stat_uri[128]; + + sprintf(stat_uri, "statistics:%s", uri); + testutil_check(session->open_cursor(session, stat_uri, NULL, "statistics=(all)", &cur_stat)); + + cur_stat->set_key(cur_stat, WT_STAT_DSRC_BTREE_COMPACT_PAGES_REVIEWED); + testutil_check(cur_stat->search(cur_stat)); + testutil_check(cur_stat->get_value(cur_stat, &descr, &str_val, pages_reviewed)); + cur_stat->set_key(cur_stat, WT_STAT_DSRC_BTREE_COMPACT_PAGES_SKIPPED); + testutil_check(cur_stat->search(cur_stat)); + testutil_check(cur_stat->get_value(cur_stat, &descr, &str_val, pages_skipped)); + cur_stat->set_key(cur_stat, WT_STAT_DSRC_BTREE_COMPACT_PAGES_REWRITTEN); + testutil_check(cur_stat->search(cur_stat)); + testutil_check(cur_stat->get_value(cur_stat, &descr, &str_val, pages_rewritten)); + + testutil_check(cur_stat->close(cur_stat)); +} + +static bool +check_db_size(WT_SESSION *session, const char *uri) +{ + uint64_t file_sz, avail_bytes, available_pct; + + get_file_stats(session, uri, &file_sz, &avail_bytes); + + /* Check if there's maximum of 10% space available after compaction. */ + available_pct = (avail_bytes * 100) / file_sz; + printf(" - Compacted file size: %" PRIu64 "MB (%" PRIu64 "B)\n - Available for reuse: %" PRIu64 + "MB (%" PRIu64 "B)\n - %" PRIu64 "%% space available in the file.\n", + file_sz / WT_MEGABYTE, file_sz, avail_bytes / WT_MEGABYTE, avail_bytes, available_pct); + + return (available_pct <= 10); +} diff --git a/src/third_party/wiredtiger/test/csuite/wt8057_compact_stress/main.c b/src/third_party/wiredtiger/test/csuite/wt8057_compact_stress/main.c index 1a495ccf06f..03f4bae3236 100644 --- a/src/third_party/wiredtiger/test/csuite/wt8057_compact_stress/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt8057_compact_stress/main.c @@ -46,12 +46,15 @@ * better view on what is happening. */ static const char conn_config[] = "create,cache_size=2GB,statistics=(all)"; -static const char table_config[] = - "allocation_size=4KB,leaf_page_max=4KB,key_format=i,value_format=S"; +static const char table_config_row[] = + "allocation_size=4KB,leaf_page_max=4KB,key_format=Q,value_format=" WT_UNCHECKED_STRING(QS); +static const char table_config_col[] = + "allocation_size=4KB,leaf_page_max=4KB,key_format=r,value_format=" WT_UNCHECKED_STRING(QS); static char data_str[1024] = ""; -static const char ckpt_file[] = "WT_TEST.compact-stress/checkpoint_done"; -static const char working_dir[] = "WT_TEST.compact-stress"; +static const char ckpt_file_fmt[] = "%s/checkpoint_done"; +static const char working_dir_row[] = "WT_TEST.compact-stress-row"; +static const char working_dir_col[] = "WT_TEST.compact-stress-col"; static const char uri1[] = "table:compact1"; static const char uri2[] = "table:compact2"; @@ -79,12 +82,16 @@ static WT_EVENT_HANDLER event_handler = { static void sig_handler(int) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); /* Forward declarations. */ -static void run_test(const char *home); -static void populate(WT_SESSION *session, int start, int end); -static void remove_records(WT_SESSION *session, const char *uri, int start, int end); -static void verify_tables(WT_SESSION *session); -static int verify_tables_helper(WT_SESSION *session, const char *table1, const char *table2); -static uint64_t get_file_size(WT_SESSION *session, const char *uri); +static void run_test(bool, bool); +static void workload_compact(const char *, const char *); +static void populate(WT_SESSION *, uint64_t, uint64_t); +static void remove_records(WT_SESSION *, const char *, uint64_t, uint64_t); +static void verify_tables(WT_SESSION *); +static int verify_tables_helper(WT_SESSION *, const char *, const char *); +static void get_file_stats(WT_SESSION *, const char *, uint64_t *, uint64_t *); +static void log_db_size(WT_SESSION *, const char *); +static void get_compact_progress( + WT_SESSION *session, const char *, uint64_t *, uint64_t *, uint64_t *); /* * Signal handler to catch if the child died unexpectedly. @@ -107,21 +114,36 @@ int main(int argc, char *argv[]) { TEST_OPTS *opts, _opts; + + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + testutil_check(testutil_parse_opts(argc, argv, opts)); + + run_test(false, opts->preserve); + + run_test(true, opts->preserve); + + testutil_cleanup(opts); + + return (EXIT_SUCCESS); +} + +static void +run_test(bool column_store, bool preserve) +{ WT_CONNECTION *conn; WT_SESSION *session; - char home[1024]; + char ckpt_file[2048], home[1024]; int status; pid_t pid; struct sigaction sa; struct stat sb; - opts = &_opts; - memset(opts, 0, sizeof(*opts)); - testutil_check(testutil_parse_opts(argc, argv, opts)); - - testutil_work_dir_from_path(home, sizeof(home), working_dir); + testutil_work_dir_from_path( + home, sizeof(home), column_store ? working_dir_col : working_dir_row); + printf("\n"); printf("Work directory: %s\n", home); testutil_make_work_dir(home); @@ -133,13 +155,13 @@ main(int argc, char *argv[]) if (pid == 0) { /* child */ - run_test(home); + workload_compact(home, column_store ? table_config_col : table_config_row); /* * We do not expect test to reach here. The child process should have been killed by the * parent process. */ printf("Child finished processing...\n"); - return (EXIT_FAILURE); + _exit(EXIT_FAILURE); } /* parent */ @@ -148,6 +170,7 @@ main(int argc, char *argv[]) * time we notice that child process has written a checkpoint. That allows the test to run * correctly on really slow machines. */ + testutil_check(__wt_snprintf(ckpt_file, sizeof(ckpt_file), ckpt_file_fmt, home)); while (stat(ckpt_file, &sb) != 0) testutil_sleep_wait(1, pid); @@ -168,34 +191,37 @@ main(int argc, char *argv[]) /* Clean-up. */ testutil_check(session->close(session, NULL)); + session = NULL; + testutil_check(conn->close(conn, NULL)); - if (!opts->preserve) - testutil_clean_work_dir(home); - testutil_cleanup(opts); + conn = NULL; - return (EXIT_SUCCESS); + if (!preserve) + testutil_clean_work_dir(home); } static void -run_test(const char *home) +workload_compact(const char *home, const char *table_config) { - + FILE *fp; WT_CONNECTION *conn; + WT_RAND_STATE rnd; WT_SESSION *session; - FILE *fp; bool first_ckpt; - int i; - int key_range_start; - time_t t; - uint64_t file_sz_after, file_sz_before; + char ckpt_file[2048]; + uint32_t i; + uint64_t key_range_start; + + uint64_t pages_reviewed, pages_rewritten, pages_skipped; first_ckpt = false; - srand((u_int)time(&t)); testutil_check(wiredtiger_open(home, &event_handler, conn_config, &conn)); testutil_check(conn->open_session(conn, NULL, NULL, &session)); + __wt_random_init_seed((WT_SESSION_IMPL *)session, &rnd); + /* Create and populate table. Checkpoint the data after that. */ testutil_check(session->create(session, uri1, table_config)); testutil_check(session->create(session, uri2, table_config)); @@ -208,13 +234,14 @@ run_test(const char *home) */ for (i = 0; i < 40; i++) { - printf("Running Loop: %i\n", i + 1); + printf("Running Loop: %" PRIu32 "\n", i + 1); testutil_check(session->checkpoint(session, NULL)); /* * Create the checkpoint file so that the parent process knows at least one checkpoint has * finished and can start its timer. */ if (!first_ckpt) { + testutil_check(__wt_snprintf(ckpt_file, sizeof(ckpt_file), ckpt_file_fmt, home)); testutil_checksys((fp = fopen(ckpt_file, "w")) == NULL); testutil_checksys(fclose(fp) != 0); first_ckpt = true; @@ -224,20 +251,20 @@ run_test(const char *home) * Remove 1/3 of data from the middle of the key range to let compact relocate blocks from * the end of the file. */ - key_range_start = rand() % (((NUM_RECORDS * 2) / 3) - 1); + key_range_start = (uint64_t)__wt_random(&rnd) % (((NUM_RECORDS * 2) / 3) - 1); remove_records(session, uri1, key_range_start, key_range_start + NUM_RECORDS / 3); remove_records(session, uri2, key_range_start, key_range_start + NUM_RECORDS / 3); /* Only perform compaction on the first table. */ - file_sz_before = get_file_size(session, uri1); - testutil_check(session->compact(session, uri1, NULL)); - file_sz_after = get_file_size(session, uri1); + log_db_size(session, uri1); - /* Check if there's at least 10% compaction. */ - printf(" - Compressed file size MB: %f\n - Original file size MB: %f\n", - file_sz_after / (1024.0 * 1024), file_sz_before / (1024.0 * 1024)); + /* If we made progress with compact, verify that compact stats support that. */ + get_compact_progress(session, uri1, &pages_reviewed, &pages_rewritten, &pages_skipped); + printf(" - Pages reviewed: %" PRIu64 "\n", pages_reviewed); + printf(" - Pages selected for being rewritten: %" PRIu64 "\n", pages_rewritten); + printf(" - Pages skipped: %" PRIu64 "\n", pages_skipped); /* Put the deleted records back. */ populate(session, key_range_start, key_range_start + NUM_RECORDS / 3); @@ -246,35 +273,36 @@ run_test(const char *home) } /* Clean-up. */ - testutil_check(session->close(session, NULL)); testutil_check(conn->close(conn, NULL)); } static void -populate(WT_SESSION *session, int start, int end) +populate(WT_SESSION *session, uint64_t start, uint64_t end) { WT_CURSOR *cursor_1; WT_CURSOR *cursor_2; - time_t t; - int i, str_len; + WT_RAND_STATE rnd; - srand((u_int)time(&t)); + uint64_t i, str_len, val; + + __wt_random_init_seed((WT_SESSION_IMPL *)session, &rnd); str_len = sizeof(data_str) / sizeof(data_str[0]); for (i = 0; i < str_len - 1; i++) - data_str[i] = 'a' + rand() % 26; + data_str[i] = 'a' + (uint32_t)__wt_random(&rnd) % 26; data_str[str_len - 1] = '\0'; testutil_check(session->open_cursor(session, uri1, NULL, NULL, &cursor_1)); testutil_check(session->open_cursor(session, uri2, NULL, NULL, &cursor_2)); for (i = start; i < end; i++) { - cursor_1->set_key(cursor_1, i); - cursor_1->set_value(cursor_1, data_str); + val = (uint64_t)__wt_random(&rnd); + cursor_1->set_key(cursor_1, i + 1); + cursor_1->set_value(cursor_1, val, data_str); testutil_check(cursor_1->insert(cursor_1)); - cursor_2->set_key(cursor_2, i); - cursor_2->set_value(cursor_2, data_str); + cursor_2->set_key(cursor_2, i + 1); + cursor_2->set_value(cursor_2, val, data_str); testutil_check(cursor_2->insert(cursor_2)); } @@ -283,15 +311,15 @@ populate(WT_SESSION *session, int start, int end) } static void -remove_records(WT_SESSION *session, const char *uri, int start, int end) +remove_records(WT_SESSION *session, const char *uri, uint64_t start, uint64_t end) { WT_CURSOR *cursor; - int i; + uint64_t i; testutil_check(session->open_cursor(session, uri, NULL, NULL, &cursor)); for (i = start; i < end; i++) { - cursor->set_key(cursor, i); + cursor->set_key(cursor, i + 1); testutil_check(cursor->remove(cursor)); } @@ -301,18 +329,12 @@ remove_records(WT_SESSION *session, const char *uri, int start, int end) static int verify_tables_helper(WT_SESSION *session, const char *table1, const char *table2) { - WT_CURSOR *cursor_1; - WT_CURSOR *cursor_2; - - char *val_1; - char *val_2; + WT_CURSOR *cursor_1, *cursor_2; - int key; - int ret; - int total_keys; - - size_t val_1_size; - size_t val_2_size; + char *str_val_1, *str_val_2; + int ret, total_keys; + size_t val_1_size, val_2_size; + uint64_t key, val_1, val_2; testutil_check(session->open_cursor(session, table1, NULL, NULL, &cursor_1)); testutil_check(session->open_cursor(session, table2, NULL, NULL, &cursor_2)); @@ -323,14 +345,15 @@ verify_tables_helper(WT_SESSION *session, const char *table1, const char *table2 cursor_1->get_key(cursor_1, &key); cursor_2->set_key(cursor_2, key); testutil_check(cursor_2->search(cursor_2)); - testutil_check(cursor_1->get_value(cursor_1, &val_1)); - testutil_check(cursor_2->get_value(cursor_2, &val_2)); + testutil_check(cursor_1->get_value(cursor_1, &val_1, &str_val_1)); + testutil_check(cursor_2->get_value(cursor_2, &val_2, &str_val_2)); - val_1_size = strlen(val_1); - val_2_size = strlen(val_2); + val_1_size = strlen(str_val_1); + val_2_size = strlen(str_val_2); + testutil_assert(val_1 == val_2); testutil_assert(val_1_size == val_2_size); - testutil_assert(memcmp(val_1, val_2, val_1_size) == 0); + testutil_assert(memcmp(str_val_1, str_val_2, val_1_size) == 0); ++total_keys; } @@ -346,8 +369,7 @@ verify_tables_helper(WT_SESSION *session, const char *table1, const char *table2 static void verify_tables(WT_SESSION *session) { - int total_keys_1; - int total_keys_2; + int total_keys_1, total_keys_2; /* * Run over all keys in first table and verify they are present in the second table. Repeat with @@ -361,20 +383,63 @@ verify_tables(WT_SESSION *session) printf("%i Keys verified from the two tables. \n", total_keys_1); } -static uint64_t -get_file_size(WT_SESSION *session, const char *uri) +static void +get_file_stats(WT_SESSION *session, const char *uri, uint64_t *file_sz, uint64_t *avail_bytes) { WT_CURSOR *cur_stat; - uint64_t val; - char *descr, *str_val; - char stat_uri[128]; + char *descr, stat_uri[128], *str_val; - sprintf(stat_uri, "statistics:%s", uri); + testutil_check(__wt_snprintf(stat_uri, sizeof(stat_uri), "statistics:%s", uri)); testutil_check(session->open_cursor(session, stat_uri, NULL, "statistics=(all)", &cur_stat)); + + /* Get file size. */ cur_stat->set_key(cur_stat, WT_STAT_DSRC_BLOCK_SIZE); testutil_check(cur_stat->search(cur_stat)); - testutil_check(cur_stat->get_value(cur_stat, &descr, &str_val, &val)); + testutil_check(cur_stat->get_value(cur_stat, &descr, &str_val, file_sz)); + + /* Get bytes available for reuse. */ + cur_stat->set_key(cur_stat, WT_STAT_DSRC_BLOCK_REUSE_BYTES); + testutil_check(cur_stat->search(cur_stat)); + testutil_check(cur_stat->get_value(cur_stat, &descr, &str_val, avail_bytes)); + testutil_check(cur_stat->close(cur_stat)); + cur_stat = NULL; +} + +static void +log_db_size(WT_SESSION *session, const char *uri) +{ + uint64_t avail_bytes, available_pct, file_sz; + + get_file_stats(session, uri, &file_sz, &avail_bytes); - return (val); + /* Check if there's maximum of 10% space available after compaction. */ + available_pct = (avail_bytes * 100) / file_sz; + printf(" - Compacted file size: %" PRIu64 "MB (%" PRIu64 "B)\n - Available for reuse: %" PRIu64 + "MB (%" PRIu64 "B)\n - %" PRIu64 "%% space available in the file.\n", + file_sz / WT_MEGABYTE, file_sz, avail_bytes / WT_MEGABYTE, avail_bytes, available_pct); +} + +static void +get_compact_progress(WT_SESSION *session, const char *uri, uint64_t *pages_reviewed, + uint64_t *pages_skipped, uint64_t *pages_rewritten) +{ + + WT_CURSOR *cur_stat; + char *descr, *str_val, stat_uri[128]; + + testutil_check(__wt_snprintf(stat_uri, sizeof(stat_uri), "statistics:%s", uri)); + testutil_check(session->open_cursor(session, stat_uri, NULL, "statistics=(all)", &cur_stat)); + + cur_stat->set_key(cur_stat, WT_STAT_DSRC_BTREE_COMPACT_PAGES_REVIEWED); + testutil_check(cur_stat->search(cur_stat)); + testutil_check(cur_stat->get_value(cur_stat, &descr, &str_val, pages_reviewed)); + cur_stat->set_key(cur_stat, WT_STAT_DSRC_BTREE_COMPACT_PAGES_SKIPPED); + testutil_check(cur_stat->search(cur_stat)); + testutil_check(cur_stat->get_value(cur_stat, &descr, &str_val, pages_skipped)); + cur_stat->set_key(cur_stat, WT_STAT_DSRC_BTREE_COMPACT_PAGES_REWRITTEN); + testutil_check(cur_stat->search(cur_stat)); + testutil_check(cur_stat->get_value(cur_stat, &descr, &str_val, pages_rewritten)); + + testutil_check(cur_stat->close(cur_stat)); } diff --git a/src/third_party/wiredtiger/test/ctest_helpers.cmake b/src/third_party/wiredtiger/test/ctest_helpers.cmake index a6afb901f71..ea87b6f4015 100644 --- a/src/third_party/wiredtiger/test/ctest_helpers.cmake +++ b/src/third_party/wiredtiger/test/ctest_helpers.cmake @@ -28,7 +28,7 @@ # create_test_executable(target SOURCES <source files> [EXECUTABLE_NAME <name>] [BINARY_DIR <dir>] [INCLUDES <includes>] # [ADDITIONAL_FILES <files>] [ADDITIONAL_DIRECTORIES <dirs>] [LIBS <libs>] [FLAGS <flags>]) -# Defines a C test executable binary. This helper does the necessary initialisation to ensure the correct flags and libraries +# Defines a C/CXX test executable binary. This helper does the necessary initialisation to ensure the correct flags and libraries # are used when compiling the test executable. # target - Target name of the test. # SOURCES <source files> - Sources to compile for the given test. @@ -79,15 +79,16 @@ function(create_test_executable target) # Append the necessary compiler flags. if(NOT CREATE_TEST_CXX) - # Don't append the strict diagnostic flags to C++ targets (as these are chosen for C targets). - set(test_c_flags "${COMPILER_DIAGNOSTIC_FLAGS}") + set(test_flags "${COMPILER_DIAGNOSTIC_C_FLAGS}") + else() + set(test_flags "${COMPILER_DIAGNOSTIC_CXX_FLAGS}") endif() if(NOT "${CREATE_TEST_FLAGS}" STREQUAL "") - list(APPEND test_c_flags ${CREATE_TEST_FLAGS}) + list(APPEND test_flags ${CREATE_TEST_FLAGS}) endif() - target_compile_options(${target} PRIVATE ${test_c_flags}) + target_compile_options(${target} PRIVATE ${test_flags}) - # Include the base set of directories for a wiredtiger C test. + # Include the base set of directories for a wiredtiger C/CXX test. target_include_directories(${target} PRIVATE ${CMAKE_SOURCE_DIR}/src/include @@ -98,16 +99,12 @@ function(create_test_executable target) target_include_directories(${target} PRIVATE ${CREATE_TEST_INCLUDES}) endif() - # Link the base set of libraries for a wiredtiger C test. - target_link_libraries(${target} wiredtiger test_util) + # Link the base set of libraries for a wiredtiger C/CXX test. + target_link_libraries(${target} wt::wiredtiger test_util) if(NOT "${CREATE_TEST_LIBS}" STREQUAL "") target_link_libraries(${target} ${CREATE_TEST_LIBS}) endif() - if(ENABLE_TCMALLOC AND HAVE_LIBTCMALLOC) - target_link_libraries(${target} ${HAVE_LIBTCMALLOC}) - endif() - # If compiling for windows, additionally link in the shim library. if(WT_WIN) target_include_directories( diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml index 6b100d1d431..dcb0ea09c54 100755 --- a/src/third_party/wiredtiger/test/evergreen.yml +++ b/src/third_party/wiredtiger/test/evergreen.yml @@ -384,7 +384,7 @@ functions: set -o errexit set -o verbose for i in $(seq ${times|1}); do - ./t -1 -c ${config|../../../test/format/CONFIG.stress} ${extra_args|} || ( [ -f RUNDIR/CONFIG ] && cat RUNDIR/CONFIG ) 2>&1 + ./t -c ${config|../../../test/format/CONFIG.stress} ${extra_args|} || ( [ -f RUNDIR/CONFIG ] && cat RUNDIR/CONFIG ) 2>&1 done "format test script": command: shell.exec @@ -590,34 +590,62 @@ functions: done done - "generic-perf-test": + "run-perf-test": # Run a performance test # Parameterised using the 'perf-test-name' and 'maxruns' variables - command: shell.exec params: - working_dir: "wiredtiger/bench/wtperf/wtperf_run_py" + working_dir: "wiredtiger/cmake_build/bench/wtperf" shell: bash script: | set -o errexit set -o verbose ${virtualenv_binary} -p ${python_binary} venv source venv/bin/activate - ${pip3_binary} install psutil - ${python_binary} wtperf_run.py -p ../../../cmake_build/bench/wtperf/wtperf -t ../runners/${perf-test-name}.wtperf -v -ho WT_TEST -m ${maxruns} -o out.json + ${pip3_binary} install psutil pygit2 + JSON_TASK_INFO='{ "evergreen_task_info": { "is_patch": "'${is_patch}'", "task_id": "'${task_id}'", "distro_id": "'${distro_id}'" } }' + echo "JSON_TASK_INFO: $JSON_TASK_INFO" + ${test_env_vars|} ${python_binary} ../../../bench/wtperf/wtperf_run_py/wtperf_run.py -p ./wtperf -t ../../../bench/wtperf/runners/${perf-test-name}.wtperf -ho WT_TEST -m ${maxruns} -g "../.." -v -i "$JSON_TASK_INFO" -b -o test_stats/evergreen_out.json ${wtarg} + ${test_env_vars|} ${python_binary} ../../../bench/wtperf/wtperf_run_py/wtperf_run.py -p ./wtperf -t ../../../bench/wtperf/runners/${perf-test-name}.wtperf -ho WT_TEST -m ${maxruns} -g "../.." -v -i "$JSON_TASK_INFO" -re -o test_stats/atlas_out.json ${wtarg} - "generic-perf-test-push-results": + "upload-perf-test-stats": + - command: shell.exec + params: + working_dir: "wiredtiger/cmake_build/bench/wtperf" + shell: bash + silent: true + script: | + set -o errexit + ${virtualenv_binary} -p ${python_binary} venv + source venv/bin/activate + ${pip3_binary} install pymongo[srv] + git clone git@github.com:wiredtiger/automation-scripts.git + ${python_binary} automation-scripts/evergreen/upload_stats_atlas.py -u ${atlas_wt_perf_test_user} -p ${atlas_wt_perf_pass} -f test_stats/atlas_out.json -b ${branch_name} + - command: perf.send + params: + file: ./wiredtiger/cmake_build/bench/wtperf/test_stats/evergreen_out.json # Push the json results to the 'Files' tab of the task in Evergreen # Parameterised using the 'perf-test-name' variable - command: s3.put params: aws_secret: ${aws_secret} aws_key: ${aws_key} - local_file: wiredtiger/bench/wtperf/wtperf_run_py/out.json + local_files_include_filter: wiredtiger/cmake_build/bench/wtperf/test_stats/* bucket: build_external permissions: public-read content_type: text/html - display_name: "Test results (JSON)" - remote_file: wiredtiger/${build_variant}/${revision}/perf-test-${perf-test-name}-${build_id}-${execution}/test-results.json + remote_file: wiredtiger/${build_variant}/${revision}/perf-test-${perf-test-name}-${build_id}-${execution}/ + + "validate-expected-stats": + - command: shell.exec + params: + working_dir: "wiredtiger/cmake_build/bench/wtperf" + shell: bash + script: | + set -o errexit + ${virtualenv_binary} -p ${python_binary} venv + source venv/bin/activate + ${python_binary} ../../../bench/wtperf/wtperf_run_py/validate_expected_stats.py '${expected-stats}' '${stat_file}' ####################################### # Variables # @@ -628,8 +656,8 @@ variables: ######################################################################################### # The following stress tests are configured to run for six hours via the "-t 360" # argument to format.sh: format-stress-test, format-stress-sanitizer-test, and -# race-condition-stress-sanitizer-test. The smoke and recovery tests run in a loop, -# with the number of runs adjusted to provide aproximately six hours of testing. +# race-condition-stress-sanitizer-test. The recovery tests run in a loop, with +# the number of runs adjusted to provide aproximately six hours of testing. ######################################################################################### - &format-stress-test @@ -653,29 +681,6 @@ variables: ASAN_OPTIONS="detect_leaks=1:abort_on_error=1:disable_coredump=0" ASAN_SYMBOLIZER_PATH=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer - - &format-stress-smoke-test - exec_timeout_secs: 25200 - commands: - - func: "get project" - - func: "compile wiredtiger with builtins" - - func: "format test script" - vars: - format_test_script_args: -e "SEGFAULT_SIGNALS=all" -b "catchsegv ./t" -S - times: 30 - - - &format-stress-sanitizer-smoke-test - exec_timeout_secs: 25200 - commands: - - func: "get project" - - func: "compile wiredtiger address sanitizer" - - func: "format test script" - vars: - test_env_vars: - ASAN_OPTIONS="detect_leaks=1:abort_on_error=1:disable_coredump=0" - ASAN_SYMBOLIZER_PATH=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer - format_test_script_args: -S - times: 20 - - &race-condition-stress-sanitizer-test exec_timeout_secs: 25200 commands: @@ -1072,6 +1077,21 @@ tasks: set -o verbose ${test_env_vars|} $(pwd)/run -t hs_cleanup -f configs/hs_cleanup_stress.txt -l 2 + + - name: cppsuite-search-near-stress + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - command: shell.exec + params: + working_dir: "wiredtiger/build_posix/" + script: | + set -o errexit + set -o verbose + + ${test_env_vars|} $(pwd)/test/cppsuite/run -t search_near_01 -f test/cppsuite/configs/search_near_01_stress.txt -l 2 + ${test_env_vars|} $(pwd)/test/cppsuite/run -t search_near_02 -f test/cppsuite/configs/search_near_02_stress.txt -l 2 # End of cppsuite test tasks. # Start of csuite test tasks @@ -2356,6 +2376,9 @@ tasks: dir=../bench/wtperf/stress for file in `ls $dir` do + echo "====" + echo "==== Initiating wtperf test using $dir/$file ====" + echo "====" ./bench/wtperf/wtperf -O $dir/$file -o verbose=2 cp -rf WT_TEST WT_TEST_$file done @@ -2495,6 +2518,49 @@ tasks: # run for 10 minutes. format_test_script_args: -t 10 rows=10000 ops=50000 + - name: format-smoke-test + commands: + - func: "get project" + - func: "compile wiredtiger with builtins" + - func: "format test script" + vars: + format_test_script_args: -e "SEGFAULT_SIGNALS=all" -b "catchsegv ./t" -S + + - name: format-asan-smoke-test + commands: + - func: "get project" + - func: "compile wiredtiger address sanitizer" + - func: "format test script" + vars: + test_env_vars: + ASAN_OPTIONS="detect_leaks=1:abort_on_error=1:disable_coredump=0" + ASAN_SYMBOLIZER_PATH=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer + format_test_script_args: -S + + # Replace this test with format-asan-smoke-test after BUILD-10248 fix. + - name: format-asan-smoke-ppc-test + commands: + - func: "get project" + - func: "compile wiredtiger" + vars: + # CC is set to the system default "clang" binary here as a workaround. Change it back + # to mongodbtoolchain "clang" binary after BUILD-10248 fix. + configure_env_vars: + CCAS=/opt/mongodbtoolchain/v3/bin/gcc CC=/usr/bin/clang + CXX=/opt/mongodbtoolchain/v3/bin/clang++ PATH=/opt/mongodbtoolchain/v3/bin:$PATH + CFLAGS="-ggdb -fPIC -fsanitize=address -fno-omit-frame-pointer + -I/opt/mongodbtoolchain/v3/lib/gcc/ppc64le-mongodb-linux/8.2.0/include" + CXXFLAGS="-ggdb -fPIC -fsanitize=address -fno-omit-frame-pointer + -I/opt/mongodbtoolchain/v3/lib/gcc/ppc64le-mongodb-linux/8.2.0/include" + posix_configure_flags: --enable-diagnostic --with-builtins=lz4,snappy,zlib + - func: "format test script" + # Run smoke tests, don't stop at failed tests, use default config + vars: + test_env_vars: + ASAN_OPTIONS="detect_leaks=1:abort_on_error=1:disable_coredump=0" + ASAN_SYMBOLIZER_PATH=/usr/lib/llvm-6.0/bin/llvm-symbolizer + format_test_script_args: -S + - name: format-wtperf-test commands: - func: "get project" @@ -2753,34 +2819,6 @@ tasks: # Run for 2 hours (2 * 60 = 120 minutes), don't stop at failed tests, use default config format_test_script_args: -t 120 - # Replace this test with format-stress-sanitizer-smoke-test after BUILD-10248 fix. - - name: format-stress-sanitizer-smoke-ppc-test - tags: ["stress-test-ppc-1"] - # Set 7 hours timeout (60 * 60 * 7) - exec_timeout_secs: 25200 - commands: - - func: "get project" - - func: "compile wiredtiger" - vars: - # CC is set to the system default "clang" binary here as a workaround. Change it back - # to mongodbtoolchain "clang" binary after BUILD-10248 fix. - configure_env_vars: - CCAS=/opt/mongodbtoolchain/v3/bin/gcc CC=/usr/bin/clang - CXX=/opt/mongodbtoolchain/v3/bin/clang++ PATH=/opt/mongodbtoolchain/v3/bin:$PATH - CFLAGS="-ggdb -fPIC -fsanitize=address -fno-omit-frame-pointer - -I/opt/mongodbtoolchain/v3/lib/gcc/ppc64le-mongodb-linux/8.2.0/include" - CXXFLAGS="-ggdb -fPIC -fsanitize=address -fno-omit-frame-pointer - -I/opt/mongodbtoolchain/v3/lib/gcc/ppc64le-mongodb-linux/8.2.0/include" - posix_configure_flags: --enable-diagnostic --with-builtins=lz4,snappy,zlib - - func: "format test script" - # To emulate the original Jenkins job's test coverage, we are running the smoke test 16 times - # Run smoke tests, don't stop at failed tests, use default config - vars: - test_env_vars: - ASAN_OPTIONS="detect_leaks=1:abort_on_error=1:disable_coredump=0" - ASAN_SYMBOLIZER_PATH=/usr/lib/llvm-6.0/bin/llvm-symbolizer - format_test_script_args: -S - times: 16 - <<: *format-stress-test name: format-stress-test-1 @@ -2806,18 +2844,6 @@ tasks: - <<: *format-stress-sanitizer-test name: format-stress-sanitizer-test-4 tags: ["stress-test-4"] - - <<: *format-stress-smoke-test - name: format-stress-smoke-test-1 - tags: ["stress-test-1", "stress-test-ppc-1", "stress-test-zseries-1"] - - <<: *format-stress-smoke-test - name: format-stress-smoke-test-2 - tags: ["stress-test-2", "stress-test-ppc-2", "stress-test-zseries-2"] - - <<: *format-stress-sanitizer-smoke-test - name: format-stress-sanitizer-smoke-test-1 - tags: ["stress-test-1"] - - <<: *format-stress-sanitizer-smoke-test - name: format-stress-sanitizer-smoke-test-2 - tags: ["stress-test-2"] - <<: *race-condition-stress-sanitizer-test name: race-condition-stress-sanitizer-test-1 tags: ["stress-test-1"] @@ -2842,9 +2868,11 @@ tasks: - name: format-abort-recovery-stress-test commands: + # Allow 30 minutes beyond test runtime because recovery under load can cause the test to + # run longer. - command: timeout.update params: - exec_timeout_secs: 2500 + exec_timeout_secs: 3600 - func: "get project" - func: "compile wiredtiger with builtins" - func: "format test script" @@ -2895,6 +2923,7 @@ tasks: source venv/bin/activate pip3 install lorem pymongo ./run_many_coll.sh ../../mongo/build/opt/install/bin/mongod mongodb.log config/many-collection-testing many-collection clean-and-populate + - command: shell.exec params: working_dir: mongo-tests/largescale @@ -2940,74 +2969,92 @@ tasks: ############################# - name: perf-test-small-lsm + tags: ["lsm-perf"] + depends_on: + - name: compile commands: - - func: "get project" - - func: "compile wiredtiger" - - func: "generic-perf-test" + - func: "fetch artifacts" + - func: "run-perf-test" vars: perf-test-name: small-lsm maxruns: 3 - - func: "generic-perf-test-push-results" + wtarg: -ops ['"load", "read"'] + - func: "upload-perf-test-stats" vars: perf-test-name: small-lsm - name: perf-test-medium-lsm + tags: ["lsm-perf"] + depends_on: + - name: compile commands: - - func: "get project" - - func: "compile wiredtiger" - - func: "generic-perf-test" + - func: "fetch artifacts" + - func: "run-perf-test" vars: perf-test-name: medium-lsm maxruns: 1 - - func: "generic-perf-test-push-results" + wtarg: -ops ['"load", "read"'] + - func: "upload-perf-test-stats" vars: perf-test-name: medium-lsm - name: perf-test-medium-lsm-compact + tags: ["lsm-perf"] + depends_on: + - name: compile commands: - - func: "get project" - - func: "compile wiredtiger" - - func: "generic-perf-test" + - func: "fetch artifacts" + - func: "run-perf-test" vars: perf-test-name: medium-lsm-compact maxruns: 1 - - func: "generic-perf-test-push-results" + wtarg: -ops ['"load", "read"'] + - func: "upload-perf-test-stats" vars: perf-test-name: medium-lsm-compact - name: perf-test-medium-multi-lsm + tags: ["lsm-perf"] + depends_on: + - name: compile commands: - - func: "get project" - - func: "compile wiredtiger" - - func: "generic-perf-test" + - func: "fetch artifacts" + - func: "run-perf-test" vars: perf-test-name: medium-multi-lsm maxruns: 1 - - func: "generic-perf-test-push-results" + wtarg: -ops ['"load", "read", "update"'] + - func: "upload-perf-test-stats" vars: perf-test-name: medium-multi-lsm - name: perf-test-parallel-pop-lsm + tags: ["lsm-perf"] + depends_on: + - name: compile commands: - - func: "get project" - - func: "compile wiredtiger" - - func: "generic-perf-test" + - func: "fetch artifacts" + - func: "run-perf-test" vars: perf-test-name: parallel-pop-lsm maxruns: 1 - - func: "generic-perf-test-push-results" + wtarg: -ops ['"load"'] + - func: "upload-perf-test-stats" vars: perf-test-name: parallel-pop-lsm - name: perf-test-update-lsm + tags: ["lsm-perf"] + depends_on: + - name: compile commands: - - func: "get project" - - func: "compile wiredtiger" - - func: "generic-perf-test" + - func: "fetch artifacts" + - func: "run-perf-test" vars: perf-test-name: update-lsm maxruns: 1 - - func: "generic-perf-test-push-results" + wtarg: -ops ['"load", "read", "update", "insert"'] + - func: "upload-perf-test-stats" vars: perf-test-name: update-lsm @@ -3016,53 +3063,544 @@ tasks: ############################### - name: perf-test-small-btree + tags: ["btree-perf"] + depends_on: + - name: compile commands: - - func: "get project" - - func: "compile wiredtiger" - - func: "generic-perf-test" + - func: "fetch artifacts" + - func: "run-perf-test" vars: perf-test-name: small-btree maxruns: 1 - - func: "generic-perf-test-push-results" + wtarg: -ops ['"load", "read"'] + - func: "upload-perf-test-stats" vars: perf-test-name: small-btree - name: perf-test-small-btree-backup + tags: ["btree-perf"] + depends_on: + - name: compile commands: - - func: "get project" - - func: "compile wiredtiger" - - func: "generic-perf-test" + - func: "fetch artifacts" + - func: "run-perf-test" vars: perf-test-name: small-btree-backup maxruns: 1 - - func: "generic-perf-test-push-results" + wtarg: -ops ['"load", "read"'] + - func: "upload-perf-test-stats" vars: perf-test-name: small-btree-backup - name: perf-test-medium-btree + tags: ["btree-perf"] + depends_on: + - name: compile commands: - - func: "get project" - - func: "compile wiredtiger" - - func: "generic-perf-test" + - func: "fetch artifacts" + - func: "run-perf-test" vars: perf-test-name: medium-btree maxruns: 3 - - func: "generic-perf-test-push-results" + wtarg: -ops ['"load", "read"'] + - func: "upload-perf-test-stats" vars: perf-test-name: medium-btree - name: perf-test-medium-btree-backup + tags: ["btree-perf"] + depends_on: + - name: compile commands: - - func: "get project" - - func: "compile wiredtiger" - - func: "generic-perf-test" + - func: "fetch artifacts" + - func: "run-perf-test" vars: perf-test-name: medium-btree-backup maxruns: 3 - - func: "generic-perf-test-push-results" + wtarg: -ops ['"load", "read"'] + - func: "upload-perf-test-stats" vars: perf-test-name: medium-btree-backup + - name: perf-test-parallel-pop-btree + tags: ["btree-perf"] + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - func: "run-perf-test" + vars: + perf-test-name: parallel-pop-btree + maxruns: 1 + wtarg: -ops ['"load"'] + - func: "upload-perf-test-stats" + vars: + perf-test-name: parallel-pop-btree + + - name: perf-test-update-only-btree + tags: ["btree-perf"] + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - func: "run-perf-test" + vars: + perf-test-name: update-only-btree + maxruns: 3 + wtarg: -ops ['"update"'] + - func: "upload-perf-test-stats" + vars: + perf-test-name: update-only-btree + + - name: perf-test-update-btree + tags: ["btree-perf"] + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - func: "run-perf-test" + vars: + perf-test-name: update-btree + maxruns: 1 + wtarg: "-bf ../../../bench/wtperf/runners/update-btree.json" + - func: "upload-perf-test-stats" + vars: + perf-test-name: update-btree + + - name: perf-test-update-large-record-btree + tags: ["btree-perf"] + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - func: "run-perf-test" + vars: + perf-test-name: update-large-record-btree + maxruns: 3 + wtarg: -ops ['"load", "update"'] + - func: "upload-perf-test-stats" + vars: + perf-test-name: update-large-record-btree + + - name: perf-test-modify-large-record-btree + tags: ["btree-perf"] + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - func: "run-perf-test" + vars: + perf-test-name: modify-large-record-btree + maxruns: 3 + wtarg: -ops ['"load", "modify"'] + - func: "upload-perf-test-stats" + vars: + perf-test-name: modify-large-record-btree + + - name: perf-test-modify-force-update-large-record-btree + tags: ["btree-perf"] + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - func: "run-perf-test" + vars: + perf-test-name: modify-force-update-large-record-btree + maxruns: 3 + wtarg: -ops ['"load", "modify"'] + - func: "upload-perf-test-stats" + vars: + perf-test-name: modify-force-update-large-record-btree + + ######################################### + # Performance Tests for perf-checkpoint # + ######################################### + + - name: perf-test-update-checkpoint-btree + tags: ["checkpoint-perf"] + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - func: "run-perf-test" + vars: + perf-test-name: update-checkpoint-btree + maxruns: 1 + wtarg: "-bf ../../../bench/wtperf/runners/update-checkpoint.json" + - func: "upload-perf-test-stats" + vars: + perf-test-name: update-checkpoint-btree + + - name: perf-test-update-checkpoint-lsm + tags: ["checkpoint-perf"] + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - func: "run-perf-test" + vars: + perf-test-name: update-checkpoint-lsm + maxruns: 1 + wtarg: "-bf ../../../bench/wtperf/runners/update-checkpoint.json" + - func: "upload-perf-test-stats" + vars: + perf-test-name: update-checkpoint-lsm + + ############################### + # Performance Tests for stress # + ############################### + + - name: perf-test-overflow-10k + tags: ["stress-perf"] + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - func: "run-perf-test" + vars: + perf-test-name: overflow-10k + maxruns: 1 + wtarg: -ops ['"load", "read", "update"'] + - func: "upload-perf-test-stats" + vars: + perf-test-name: overflow-10k + + - name: perf-test-overflow-130k + tags: ["stress-perf"] + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - func: "run-perf-test" + vars: + perf-test-name: overflow-130k + maxruns: 1 + wtarg: -ops ['"load", "read", "update"'] + - func: "upload-perf-test-stats" + vars: + perf-test-name: overflow-130k + + - name: perf-test-parallel-pop-stress + tags: ["stress-perf"] + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - func: "run-perf-test" + vars: + perf-test-name: parallel-pop-stress + maxruns: 1 + wtarg: -ops ['"load"'] + - func: "upload-perf-test-stats" + vars: + perf-test-name: parallel-pop-stress + + - name: perf-test-update-grow-stress + tags: ["stress-perf"] + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - func: "run-perf-test" + vars: + perf-test-name: update-grow-stress + maxruns: 1 + wtarg: -ops ['"update"'] + - func: "upload-perf-test-stats" + vars: + perf-test-name: update-grow-stress + + - name: perf-test-update-shrink-stress + tags: ["stress-perf"] + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - func: "run-perf-test" + vars: + perf-test-name: update-shrink-stress + maxruns: 1 + wtarg: -ops ['"update"'] + - func: "upload-perf-test-stats" + vars: + perf-test-name: update-shrink-stress + + - name: perf-test-update-delta-mix1 + tags: ["stress-perf"] + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - func: "run-perf-test" + vars: + perf-test-name: update-delta-mix1 + maxruns: 1 + wtarg: -ops ['"update"'] + - func: "upload-perf-test-stats" + vars: + perf-test-name: update-delta-mix1 + + - name: perf-test-update-delta-mix2 + tags: ["stress-perf"] + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - func: "run-perf-test" + vars: + perf-test-name: update-delta-mix2 + maxruns: 1 + wtarg: -ops ['"update"'] + - func: "upload-perf-test-stats" + vars: + perf-test-name: update-delta-mix2 + + - name: perf-test-update-delta-mix3 + tags: ["stress-perf"] + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - func: "run-perf-test" + vars: + perf-test-name: update-delta-mix3 + maxruns: 1 + wtarg: -ops ['"update"'] + - func: "upload-perf-test-stats" + vars: + perf-test-name: update-delta-mix3 + + - name: perf-test-multi-btree-zipfian + tags: ["stress-perf"] + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - func: "run-perf-test" + vars: + perf-test-name: multi-btree-zipfian-populate + maxruns: 1 + wtarg: -ops ['"read"'] + - func: "run-perf-test" + vars: + perf-test-name: multi-btree-zipfian-workload + maxruns: 1 + wtarg: -ops ['"read"'] + - func: "upload-perf-test-stats" + vars: + perf-test-name: multi-btree-zipfian + + - name: perf-test-many-table-stress + tags: ["stress-perf"] + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - func: "run-perf-test" + vars: + perf-test-name: many-table-stress + maxruns: 1 + + - name: perf-test-many-table-stress-backup + tags: ["stress-perf"] + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - func: "run-perf-test" + vars: + perf-test-name: many-table-stress-backup + maxruns: 1 + + - name: perf-test-evict-fairness + tags: ["stress-perf"] + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - func: "run-perf-test" + vars: + perf-test-name: evict-fairness + maxruns: 1 + wtarg: -args ['"-C statistics_log=(wait=10000,on_close=true,json=false,sources=[file:])", "-o reopen_connection=false"'] -ops ['"eviction_page_seen"'] + - func: "validate-expected-stats" + vars: + expected-stats: '{"Pages seen by eviction": 200}' + stat_file: './test_stats/evergreen_out.json' + + - name: perf-test-evict-btree-stress-multi + tags: ["stress-perf"] + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - func: "run-perf-test" + vars: + perf-test-name: evict-btree-stress-multi + maxruns: 1 + wtarg: -ops ['"warnings", "max_latencies"'] + - func: "upload-perf-test-stats" + vars: + perf-test-name: evict-btree-stress-multi + + ################################## + # Performance Tests for eviction # + ################################## + + - name: perf-test-evict-btree + tags: ["evict-perf"] + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - func: "run-perf-test" + vars: + perf-test-name: evict-btree + maxruns: 1 + wtarg: -ops ['"load", "read"'] + - func: "upload-perf-test-stats" + vars: + perf-test-name: evict-btree + + - name: perf-test-evict-btree-1 + tags: ["evict-perf"] + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - func: "run-perf-test" + vars: + perf-test-name: evict-btree-1 + maxruns: 1 + wtarg: -ops ['"read"'] + - func: "upload-perf-test-stats" + vars: + perf-test-name: evict-btree-1 + + - name: perf-test-evict-lsm + tags: ["evict-perf"] + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - func: "run-perf-test" + vars: + perf-test-name: evict-lsm + maxruns: 1 + wtarg: -ops ['"load", "read"'] + - func: "upload-perf-test-stats" + vars: + perf-test-name: evict-lsm + + - name: perf-test-evict-lsm-1 + tags: ["evict-perf"] + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - func: "run-perf-test" + vars: + perf-test-name: evict-lsm-1 + maxruns: 1 + wtarg: -ops ['"read"'] + - func: "upload-perf-test-stats" + vars: + perf-test-name: evict-lsm-1 + + ########################################### + # Performance Tests for log consolidation # + ########################################### + + - name: perf-test-log + tags: ["log-perf"] + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - func: "run-perf-test" + vars: + perf-test-name: log + maxruns: 1 + wtarg: -ops ['"update", "max_update_throughput", "min_update_throughput"'] + - func: "upload-perf-test-stats" + vars: + perf-test-name: log + + - name: perf-test-log-small-files + tags: ["log-perf"] + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - func: "run-perf-test" + vars: + perf-test-name: log + maxruns: 1 + wtarg: -args ['"-C log=(enabled,file_max=1M)"'] -ops ['"update"'] + - func: "upload-perf-test-stats" + vars: + perf-test-name: log-small-files + + - name: perf-test-log-no-checkpoints + tags: ["log-perf"] + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - func: "run-perf-test" + vars: + perf-test-name: log + maxruns: 1 + wtarg: -args ['"-C checkpoint=(wait=0)"'] -ops ['"update"'] + - func: "upload-perf-test-stats" + vars: + perf-test-name: log-no-checkpoints + + - name: perf-test-log-no-prealloc + tags: ["log-perf"] + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - func: "run-perf-test" + vars: + perf-test-name: log + maxruns: 1 + wtarg: -args ['"-C log=(enabled,file_max=1M,prealloc=false)"'] -ops ['"update"'] + - func: "upload-perf-test-stats" + vars: + perf-test-name: log-no-prealloc + + - name: perf-test-log-zero-fill + tags: ["log-perf"] + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - func: "run-perf-test" + vars: + perf-test-name: log + maxruns: 1 + wtarg: -args ['"-C log=(enabled,file_max=1M,zero_fill=true)"'] -ops ['"update"'] + - func: "upload-perf-test-stats" + vars: + perf-test-name: log-zero-fill + + - name: perf-test-log-many-threads + tags: ["log-perf"] + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - func: "run-perf-test" + vars: + perf-test-name: log + maxruns: 1 + wtarg: -args ['"-C log=(enabled,file_max=1M),session_max=256", "-o threads=((count=128,updates=1))"'] -ops ['"update"'] + - func: "upload-perf-test-stats" + vars: + perf-test-name: log-many-threads + ####################################### # Buildvariants # ####################################### @@ -3112,6 +3650,7 @@ buildvariants: - name: ftruncate-test - name: long-test - name: static-wt-build-test + - name: format-smoke-test - name: format-failure-configs-test - name: data-validation-stress-test-checkpoint - name: data-validation-stress-test-checkpoint-fp-hs-insert-s1 @@ -3168,21 +3707,24 @@ buildvariants: --enable-strict --enable-diagnostic --disable-static + --enable-tcmalloc --prefix=$(pwd)/LOCAL_INSTALL smp_command: -j $(grep -c ^processor /proc/cpuinfo) make_command: PATH=/opt/mongodbtoolchain/v3/bin:$PATH make test_env_vars: + top_dir=$(git rev-parse --show-toplevel) + top_builddir=$top_dir/build_posix ASAN_OPTIONS="detect_leaks=1:abort_on_error=1:disable_coredump=0" + LSAN_OPTIONS="print_suppressions=0:suppressions=$top_dir/test/evergreen/asan_leaks.supp" ASAN_SYMBOLIZER_PATH=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer TESTUTIL_BYPASS_ASAN=1 LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libeatmydata.so PATH=/opt/mongodbtoolchain/v3/bin:$PATH - top_dir=$(git rev-parse --show-toplevel) - top_builddir=$top_dir/build_posix - LD_LIBRARY_PATH=$top_builddir/.libs + LD_LIBRARY_PATH=$top_builddir/.libs:$top_dir/TCMALLOC_LIB/lib tasks: - name: ".pull_request !.windows_only !.pull_request_compilers !.python" - name: examples-c-test + - name: format-asan-smoke-test - name: ubuntu2004-msan display_name: "! Ubuntu 20.04 MSAN" @@ -3316,10 +3858,20 @@ buildvariants: - name: ubuntu2004-perf-tests display_name: Ubuntu 20.04 Performance tests run_on: - - ubuntu2004-test + - ubuntu2004-large expansions: - test_env_vars: LD_LIBRARY_PATH=$(pwd) WT_BUILDDIR=$(pwd) - posix_configure_flags: -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchains/mongodbtoolchain_v3_gcc.cmake -DCMAKE_C_FLAGS="-ggdb" -DHAVE_DIAGNOSTIC=1 -DENABLE_ZLIB=1 -DENABLE_SNAPPY=1 -DENABLE_STRICT=1 -DCMAKE_INSTALL_PREFIX=$(pwd)/LOCAL_INSTALL + test_env_vars: + top_dir=$(git rev-parse --show-toplevel) + top_builddir=$top_dir/cmake_build + LD_LIBRARY_PATH=$top_builddir/test/utility/:$top_builddir + posix_configure_flags: + -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchains/mongodbtoolchain_v3_gcc.cmake + -DCMAKE_C_FLAGS="-ggdb" + -DHAVE_DIAGNOSTIC=1 + -DENABLE_ZLIB=1 + -DENABLE_SNAPPY=1 + -DENABLE_STRICT=1 + -DCMAKE_INSTALL_PREFIX=$(pwd)/LOCAL_INSTALL python_binary: '/opt/mongodbtoolchain/v3/bin/python3' pip3_binary: '/opt/mongodbtoolchain/v3/bin/pip3' virtualenv_binary: '/opt/mongodbtoolchain/v3/bin/virtualenv' @@ -3328,19 +3880,32 @@ buildvariants: make_command: ninja is_cmake_build: true tasks: - # btree tests - - name: perf-test-small-btree - - name: perf-test-small-btree-backup - - name: perf-test-medium-btree - - name: perf-test-medium-btree-backup - # lsm tests - - name: perf-test-small-lsm - - name: perf-test-medium-lsm - - name: perf-test-medium-lsm-compact - - name: perf-test-medium-multi-lsm - - name: perf-test-parallel-pop-lsm - - name: perf-test-update-lsm - + - name: compile + - name: ".btree-perf" + - name: ".lsm-perf" + - name: ".stress-perf" + - name: ".checkpoint-perf" + - name: ".evict-perf" + - name: ".log-perf" + display_tasks: + - name: Wiredtiger-perf-btree-jobs + execution_tasks: + - ".btree-perf" + - name: Wiredtiger-perf-lsm-jobs + execution_tasks: + - ".lsm-perf" + - name: Wiredtiger-perf-stress-jobs + execution_tasks: + - ".stress-perf" + - name: Wiredtiger-perf-checkpoint-jobs + execution_tasks: + - ".checkpoint-perf" + - name: Wiredtiger-perf-evict-jobs + execution_tasks: + - ".evict-perf" + - name: Wiredtiger-perf-log-jobs + execution_tasks: + - ".log-perf" - name: large-scale-tests display_name: "Large scale tests" @@ -3390,8 +3955,9 @@ buildvariants: --enable-zlib tasks: - name: compile - - name: cppsuite-hs-cleanup-stress - name: cppsuite-base-test-stress + - name: cppsuite-hs-cleanup-stress + - name: cppsuite-search-near-stress - name: package display_name: "~ Package" @@ -3650,9 +4216,10 @@ buildvariants: tasks: - name: compile - name: unit-test + - name: format-smoke-test + - name: format-asan-smoke-ppc-test - name: format-wtperf-test - name: ".stress-test-ppc-1" - - name: ".stress-test-ppc-2" - name: ubuntu1804-ppc-cmake display_name: "* Ubuntu 18.04 PPC CMake" @@ -3708,6 +4275,7 @@ buildvariants: tasks: - name: compile - name: unit-test + - name: format-smoke-test - name: ".stress-test-zseries-1" - name: ".stress-test-zseries-2" - name: ".stress-test-zseries-3" diff --git a/src/third_party/wiredtiger/test/evergreen/asan_leaks.supp b/src/third_party/wiredtiger/test/evergreen/asan_leaks.supp new file mode 100644 index 00000000000..7b2133bf1f9 --- /dev/null +++ b/src/third_party/wiredtiger/test/evergreen/asan_leaks.supp @@ -0,0 +1,3 @@ +# TCMalloc intentionally leaks newed objects. More information can be found in +# https://github.com/gperftools/gperftools/issues/758. +leak:InitModule diff --git a/src/third_party/wiredtiger/test/format/CONFIG.stress b/src/third_party/wiredtiger/test/format/CONFIG.stress index a57a4f2df51..5ea078899f3 100644 --- a/src/third_party/wiredtiger/test/format/CONFIG.stress +++ b/src/third_party/wiredtiger/test/format/CONFIG.stress @@ -2,7 +2,7 @@ btree.huffman_value=0 cache.minimum=20 runs.rows=1000000:5000000 +runs.tables=3:10 runs.threads=4:32 runs.timer=6:30 runs.type=row,var -runs=100 diff --git a/src/third_party/wiredtiger/test/format/Makefile.am b/src/third_party/wiredtiger/test/format/Makefile.am index 771e41fd662..8897feee112 100644 --- a/src/third_party/wiredtiger/test/format/Makefile.am +++ b/src/third_party/wiredtiger/test/format/Makefile.am @@ -4,8 +4,8 @@ AM_CPPFLAGS +=-I$(top_srcdir)/test/utility noinst_PROGRAMS = t t_SOURCES =\ - alter.c backup.c bulk.c checkpoint.c compact.c config.c config_compat.c hs.c import.c kv.c ops.c \ - random.c salvage.c snap.c t.c trace.c util.c wts.c + alter.c backup.c bulk.c checkpoint.c compact.c config.c config_compat.c config_def.c hs.c \ + import.c kv.c ops.c random.c salvage.c snap.c t.c trace.c util.c wts.c t_LDADD = $(top_builddir)/test/utility/libtest_util.la t_LDADD +=$(top_builddir)/libwiredtiger.la diff --git a/src/third_party/wiredtiger/test/format/alter.c b/src/third_party/wiredtiger/test/format/alter.c index 7c3dc4f02d9..2a87f0654ec 100644 --- a/src/third_party/wiredtiger/test/format/alter.c +++ b/src/third_party/wiredtiger/test/format/alter.c @@ -35,6 +35,7 @@ WT_THREAD_RET alter(void *arg) { + TABLE *table; WT_CONNECTION *conn; WT_DECL_RET; WT_SESSION *session; @@ -60,10 +61,10 @@ alter(void *arg) testutil_check(__wt_snprintf( buf, sizeof(buf), "access_pattern_hint=%s", access_value ? "random" : "none")); access_value = !access_value; - /* - * Alter can return EBUSY if concurrent with other operations. - */ - while ((ret = session->alter(session, g.uri, buf)) != 0 && ret != EBUSY) + + /* Alter can return EBUSY if concurrent with other operations. */ + table = table_select(NULL); + while ((ret = session->alter(session, table->uri, buf)) != 0 && ret != EBUSY) testutil_die(ret, "session.alter"); while (period > 0 && !g.workers_finished) { --period; diff --git a/src/third_party/wiredtiger/test/format/backup.c b/src/third_party/wiredtiger/test/format/backup.c index 8ec113dd6d8..3d53b5324f7 100644 --- a/src/third_party/wiredtiger/test/format/backup.c +++ b/src/third_party/wiredtiger/test/format/backup.c @@ -71,12 +71,8 @@ check_copy(void) testutil_check(__wt_snprintf(path, len, "%s/BACKUP", g.home)); wts_open(path, &conn, &session, true); - /* - * Verify can return EBUSY if the handle isn't available. Don't yield and retry, in the case of - * LSM, the handle may not be available for a long time. - */ - ret = session->verify(session, g.uri, NULL); - testutil_assertfmt(ret == 0 || ret == EBUSY, "WT_SESSION.verify: %s: %s", path, g.uri); + /* Verify the objects. */ + tables_apply(wts_verify, conn); wts_close(&conn, &session); @@ -114,7 +110,7 @@ active_files_print(ACTIVE_FILES *active, const char *msg) if (active == NULL) return; - fprintf(stderr, "Active files: %s, %d entries\n", msg, (int)active->count); + fprintf(stderr, "Active files: %s, %" PRIu32 " entries\n", msg, active->count); for (i = 0; i < active->count; i++) fprintf(stderr, " %s\n", active->names[i]); } @@ -167,7 +163,7 @@ active_files_remove_missing(ACTIVE_FILES *prev, ACTIVE_FILES *cur) { uint32_t curpos, prevpos; int cmp; - char filename[1024]; + char filename[MAX_FORMAT_PATH]; if (prev == NULL) return; @@ -244,7 +240,7 @@ copy_blocks(WT_SESSION *session, WT_CURSOR *bkup_c, const char *name) ssize_t rdsize; uint64_t offset, size, this_size, total, type; int rfd, wfd1, wfd2; - char config[512], *tmp; + char config[MAX_FORMAT_PATH], *tmp; bool first_pass; tmp_sz = 0; @@ -366,7 +362,7 @@ restore_backup_info(WT_SESSION *session, ACTIVE_FILES *active) uint32_t i; char buf[512], *path; - testutil_assert(g.c_backup_incr_flag == INCREMENTAL_BLOCK); + testutil_assert(g.backup_incr_flag == INCREMENTAL_BLOCK); len = strlen(g.home) + strlen(BACKUP_INFO_FILE) + 2; path = dmalloc(len); testutil_check(__wt_snprintf(path, len, "%s/%s", g.home, BACKUP_INFO_FILE)); @@ -440,7 +436,7 @@ save_backup_info(ACTIVE_FILES *active, uint64_t id) uint32_t i; char *from_path, *to_path; - if (g.c_backup_incr_flag != INCREMENTAL_BLOCK) + if (g.backup_incr_flag != INCREMENTAL_BLOCK) return; len = strlen(g.home) + strlen(BACKUP_INFO_FILE_TMP) + 2; from_path = dmalloc(len); @@ -498,7 +494,7 @@ backup(void *arg) * that we can take an incremental backup and use the older id as a source identifier. We force * that only if the restore function was successful in restoring the backup information. */ - if (g.reopen && g.c_backup_incr_flag == INCREMENTAL_BLOCK && + if (g.reopen && g.backup_incr_flag == INCREMENTAL_BLOCK && restore_backup_info(session, &active[0]) == RESTORE_SUCCESS) { incr_full = false; full = false; @@ -529,7 +525,7 @@ backup(void *arg) break; } - if (g.c_backup_incr_flag == INCREMENTAL_BLOCK) { + if (g.backup_incr_flag == INCREMENTAL_BLOCK) { /* * If we're doing a full backup as the start of the incremental backup, only send in an * identifier for this one. Also set the block granularity. @@ -541,7 +537,7 @@ backup(void *arg) active_prev = NULL; testutil_check(__wt_snprintf(cfg, sizeof(cfg), "incremental=(enabled,granularity=%" PRIu32 "K,this_id=%" PRIu64 ")", - g.c_backup_incr_granularity, g.backup_id)); + GV(BACKUP_INCR_GRANULARITY), g.backup_id)); full = true; incr_full = false; } else { @@ -562,7 +558,7 @@ backup(void *arg) config = cfg; /* Free up the old active file list we're going to overwrite. */ active_files_free(active_now); - } else if (g.c_logging && g.c_backup_incr_flag == INCREMENTAL_LOG) { + } else if (GV(LOGGING) && g.backup_incr_flag == INCREMENTAL_LOG) { if (incr_full) { config = NULL; full = true; @@ -595,7 +591,7 @@ backup(void *arg) while ((ret = backup_cursor->next(backup_cursor)) == 0) { testutil_check(backup_cursor->get_key(backup_cursor, &key)); - if (g.c_backup_incr_flag == INCREMENTAL_BLOCK) { + if (g.backup_incr_flag == INCREMENTAL_BLOCK) { if (full) testutil_copy_file(session, key); else @@ -609,7 +605,7 @@ backup(void *arg) testutil_die(ret, "backup-cursor"); /* After a log-based incremental backup, truncate the log files. */ - if (g.c_backup_incr_flag == INCREMENTAL_LOG) + if (g.backup_incr_flag == INCREMENTAL_LOG) testutil_check(session->truncate(session, "log:", backup_cursor, NULL, NULL)); testutil_check(backup_cursor->close(backup_cursor)); @@ -628,9 +624,9 @@ backup(void *arg) */ if (full) { incremental = 1; - if (g.c_backup_incr_flag == INCREMENTAL_LOG) - incremental = g.c_logging_archive ? 1 : mmrand(NULL, 1, 8); - else if (g.c_backup_incr_flag == INCREMENTAL_BLOCK) + if (g.backup_incr_flag == INCREMENTAL_LOG) + incremental = GV(LOGGING_ARCHIVE) ? 1 : mmrand(NULL, 1, 8); + else if (g.backup_incr_flag == INCREMENTAL_BLOCK) incremental = mmrand(NULL, 1, 8); } if (--incremental == 0) { diff --git a/src/third_party/wiredtiger/test/format/bulk.c b/src/third_party/wiredtiger/test/format/bulk.c index d1864741a77..def8241f045 100644 --- a/src/third_party/wiredtiger/test/format/bulk.c +++ b/src/third_party/wiredtiger/test/format/bulk.c @@ -73,8 +73,12 @@ bulk_rollback_transaction(WT_SESSION *session) testutil_check(session->rollback_transaction(session, NULL)); } +/* + * wts_load -- + * Bulk load a table. + */ void -wts_load(void) +wts_load(TABLE *table, void *arg) { WT_CONNECTION *conn; WT_CURSOR *cursor; @@ -82,41 +86,38 @@ wts_load(void) WT_ITEM key, value; WT_SESSION *session; uint32_t committed_keyno, keyno, v; + char track_buf[128]; bool is_bulk; - conn = g.wts_conn; + (void)arg; /* unused argument */ - testutil_check(conn->open_session(conn, NULL, NULL, &session)); + conn = g.wts_conn; - trace_msg("%s", "=============== bulk load start"); + testutil_check(__wt_snprintf(track_buf, sizeof(track_buf), "table %u bulk load", table->id)); + trace_msg("=============== %s bulk load start", table->uri); /* * No bulk load with custom collators, the order of insertion will not match the collation * order. */ is_bulk = true; - if (g.c_reverse) + if (TV(BTREE_REVERSE)) is_bulk = false; - /* - * open_cursor can return EBUSY if concurrent with a metadata operation, retry in that case. - */ - while ((ret = session->open_cursor( - session, g.uri, NULL, is_bulk ? "bulk,append" : NULL, &cursor)) == EBUSY) - __wt_yield(); - testutil_check(ret); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + wiredtiger_open_cursor(session, table->uri, is_bulk ? "bulk,append" : NULL, &cursor); /* Set up the key/value buffers. */ key_gen_init(&key); val_gen_init(&value); - if (g.c_txn_timestamps) + if (g.transaction_timestamps_config) bulk_begin_transaction(session); - for (committed_keyno = keyno = 0; ++keyno <= g.c_rows;) { - val_gen(NULL, &value, keyno); + for (committed_keyno = keyno = 0; ++keyno <= table->rows_current;) { + val_gen(table, NULL, &value, keyno); - switch (g.type) { + switch (table->type) { case FIX: if (!is_bulk) cursor->set_key(cursor, keyno); @@ -132,7 +133,7 @@ wts_load(void) trace_msg("bulk %" PRIu32 " {%.*s}", keyno, (int)value.size, (char *)value.data); break; case ROW: - key_gen(&key, keyno); + key_gen(table, &key, keyno); cursor->set_key(cursor, &key); cursor->set_value(cursor, &value); if (g.trace_all) @@ -150,7 +151,7 @@ wts_load(void) if ((ret = cursor->insert(cursor)) != 0) { testutil_assert(ret == WT_CACHE_FULL || ret == WT_ROLLBACK); - if (g.c_txn_timestamps) { + if (g.transaction_timestamps_config) { bulk_rollback_transaction(session); bulk_begin_transaction(session); } @@ -161,13 +162,13 @@ wts_load(void) * simply modify the values because they have to equal 100 when the database is reopened * (we are going to rewrite the CONFIG file, too). */ - if (g.c_insert_pct > 5) { - g.c_delete_pct += g.c_insert_pct - 5; - g.c_insert_pct = 5; + if (TV(OPS_PCT_INSERT) > 5) { + TV(OPS_PCT_DELETE) += TV(OPS_PCT_INSERT) - 5; + TV(OPS_PCT_INSERT) = 5; } - v = g.c_write_pct / 2; - g.c_delete_pct += v; - g.c_write_pct -= v; + v = TV(OPS_PCT_WRITE) / 2; + TV(OPS_PCT_DELETE) += v; + TV(OPS_PCT_WRITE) -= v; break; } @@ -179,9 +180,9 @@ wts_load(void) */ if ((keyno < 5000 && keyno % 10 == 0) || keyno % 5000 == 0) { /* Report on progress. */ - track("bulk load", keyno, NULL); + track(track_buf, keyno); - if (g.c_txn_timestamps) { + if (g.transaction_timestamps_config) { bulk_commit_transaction(session); committed_keyno = keyno; bulk_begin_transaction(session); @@ -189,27 +190,23 @@ wts_load(void) } } - if (g.c_txn_timestamps) + if (g.transaction_timestamps_config) bulk_commit_transaction(session); /* * Ideally, the insert loop runs until the number of rows plus one, in which case row counts are - * correct. If the loop exited early, reset the counters and rewrite the CONFIG file (so reopens - * aren't surprised). + * correct. If the loop exited early, reset the table's row count and rewrite the CONFIG file + * (so reopens aren't surprised). */ - if (keyno != g.c_rows + 1) { - g.c_rows = g.c_txn_timestamps ? committed_keyno : (keyno - 1); - testutil_assert(g.c_rows > 0); - g.rows = g.c_rows; + if (keyno != table->rows_current + 1) { + table->rows_current = g.transaction_timestamps_config ? committed_keyno : (keyno - 1); + testutil_assert(table->rows_current > 0); config_print(false); } - testutil_check(cursor->close(cursor)); - - trace_msg("%s", "=============== bulk load stop"); - testutil_check(session->close(session, NULL)); + trace_msg("=============== %s bulk load stop", table->uri); key_gen_teardown(&key); val_gen_teardown(&value); diff --git a/src/third_party/wiredtiger/test/format/checkpoint.c b/src/third_party/wiredtiger/test/format/checkpoint.c index dac538348a6..b2eef25bea0 100644 --- a/src/third_party/wiredtiger/test/format/checkpoint.c +++ b/src/third_party/wiredtiger/test/format/checkpoint.c @@ -35,7 +35,7 @@ void wts_checkpoints(void) { - char config[1024]; + char config[128]; /* * Configuring WiredTiger library checkpoints is done separately, rather than as part of the @@ -45,12 +45,12 @@ wts_checkpoints(void) * checkpoint running in the tree, and the cache can get stuck. That workload is unlikely enough * we're not going to fix it in the library, so configure it away by delaying checkpoint start. */ - if (g.c_checkpoint_flag != CHECKPOINT_WIREDTIGER) + if (g.checkpoint_config != CHECKPOINT_WIREDTIGER) return; testutil_check( __wt_snprintf(config, sizeof(config), ",checkpoint=(wait=%" PRIu32 ",log_size=%" PRIu32 ")", - g.c_checkpoint_wait, MEGABYTE(g.c_checkpoint_log_size))); + GV(CHECKPOINT_WAIT), MEGABYTE(GV(CHECKPOINT_LOG_SIZE)))); testutil_check(g.wts_conn->reconfigure(g.wts_conn, config)); } @@ -67,11 +67,13 @@ checkpoint(void *arg) u_int secs; char config_buf[64]; const char *ckpt_config; - bool backup_locked; + bool backup_locked, named_checkpoints; (void)arg; + conn = g.wts_conn; testutil_check(conn->open_session(conn, NULL, NULL, &session)); + named_checkpoints = !g.lsm_config; for (secs = mmrand(NULL, 1, 10); !g.workers_finished;) { if (secs > 0) { @@ -88,7 +90,7 @@ checkpoint(void *arg) */ ckpt_config = NULL; backup_locked = false; - if (!DATASOURCE("lsm")) + if (named_checkpoints) switch (mmrand(NULL, 1, 20)) { case 1: /* diff --git a/src/third_party/wiredtiger/test/format/compact.c b/src/third_party/wiredtiger/test/format/compact.c index 4de1d898bcc..f4b53f21f75 100644 --- a/src/third_party/wiredtiger/test/format/compact.c +++ b/src/third_party/wiredtiger/test/format/compact.c @@ -29,12 +29,13 @@ #include "format.h" /* - * compaction -- + * compact -- * Periodically do a compaction operation. */ WT_THREAD_RET compact(void *arg) { + TABLE *table; WT_CONNECTION *conn; WT_DECL_RET; WT_SESSION *session; @@ -66,7 +67,8 @@ compact(void *arg) * Compact returns ETIMEDOUT if the compaction doesn't finish in in some number of seconds. * We don't configure a timeout and occasionally exceed the default of 1200 seconds. */ - ret = session->compact(session, g.uri, NULL); + table = table_select(NULL); + ret = session->compact(session, table->uri, NULL); if (ret != 0 && ret != EBUSY && ret != ETIMEDOUT && ret != WT_ROLLBACK && ret != WT_CACHE_FULL) testutil_die(ret, "session.compact"); diff --git a/src/third_party/wiredtiger/test/format/config.c b/src/third_party/wiredtiger/test/format/config.c index 6cf7c795774..837aaa264b5 100644 --- a/src/third_party/wiredtiger/test/format/config.c +++ b/src/third_party/wiredtiger/test/format/config.c @@ -27,7 +27,6 @@ */ #include "format.h" -#include "config.h" static void config_backup_incr(void); static void config_backup_incr_granularity(void); @@ -35,81 +34,119 @@ static void config_backup_incr_log_compatibility_check(void); static void config_backward_compatible(void); static void config_cache(void); static void config_checkpoint(void); -static void config_checksum(void); -static void config_compression(const char *); +static void config_checksum(TABLE *); +static void config_compression(TABLE *, const char *); static void config_directio(void); static void config_encryption(void); static const char *config_file_type(u_int); -static bool config_fix(void); +static bool config_explicit(TABLE *, const char *); +static bool config_fix(TABLE *); static void config_in_memory(void); static void config_in_memory_reset(void); -static int config_is_perm(const char *); -static void config_lsm_reset(void); +static void config_lsm_reset(TABLE *); static void config_map_backup_incr(const char *, u_int *); static void config_map_checkpoint(const char *, u_int *); -static void config_map_checksum(const char *, u_int *); -static void config_map_compression(const char *, u_int *); -static void config_map_encryption(const char *, u_int *); static void config_map_file_type(const char *, u_int *); -static void config_pct(void); -static void config_prefix(void); -static void config_reset(void); +static void config_off(TABLE *, const char *); +static void config_pct(TABLE *); static void config_transaction(void); /* - * We currently disable random LSM testing, that is, it can be specified explicitly but we won't - * randomly choose LSM as a data_source configuration. + * config_random -- + * Do random configuration on the remaining global or table space. */ -#define DISABLE_RANDOM_LSM_TESTING 1 - -/* - * config_final -- - * Final run initialization. - */ -void -config_final(void) +static void +config_random(TABLE *table, bool table_only) { - config_print(false); + CONFIG *cp; + CONFIGV *v; + char buf[128]; + + for (cp = configuration_list; cp->name != NULL; ++cp) { + if (F_ISSET(cp, C_IGNORE)) + continue; + if (table_only && !F_ISSET(cp, C_TABLE)) + continue; + if (!table_only && F_ISSET(cp, C_TABLE)) + continue; - g.rows = g.c_rows; /* Set the key count. */ + /* + * Don't randomly configure runs.tables if we read a CONFIG file, that prevents us from + * turning old-style CONFIG files into multi-table tests. + */ + if (cp->off == V_GLOBAL_RUNS_TABLES && !g.multi_table_config) + continue; - key_init(); /* Initialize key/value information. */ - val_init(); + v = &table->v[cp->off]; + if (v->set) + continue; + + /* Configure key prefixes only rarely, 5% if the length isn't set explicitly. */ + if (cp->off == V_TABLE_BTREE_PREFIX_LEN && mmrand(NULL, 1, 100) > 5) + continue; + + /* + * Boolean flags are 0 or 1, where the variable's "min" value is the percent chance the flag + * is "on" (so "on" if random rolled <= N, otherwise "off"). + */ + if (F_ISSET(cp, C_BOOL)) + testutil_check(__wt_snprintf( + buf, sizeof(buf), "%s=%s", cp->name, mmrand(NULL, 1, 100) <= cp->min ? "on" : "off")); + else + testutil_check(__wt_snprintf( + buf, sizeof(buf), "%s=%" PRIu32, cp->name, mmrand(NULL, cp->min, cp->maxrand))); + config_single(table, buf, false); + } } /* - * config -- - * Initialize the configuration itself. + * config_promote -- + * Promote a base value to a table. */ -void -config_run(void) +static void +config_promote(TABLE *table, CONFIG *cp, CONFIGV *v) { - CONFIG *cp; char buf[128]; - /* Clear any temporary values. */ - config_reset(); + if (F_ISSET(cp, C_STRING)) + testutil_check(__wt_snprintf(buf, sizeof(buf), "%s=%s", cp->name, v->vstr)); + else + testutil_check(__wt_snprintf(buf, sizeof(buf), "%s=%" PRIu32, cp->name, v->v)); + config_single(table, buf, true); +} - /* Periodically run in-memory. */ - config_in_memory(); +/* + * config_table_am -- + * Configure the table's access methods (type and source). + */ +static void +config_table_am(TABLE *table) +{ + char buf[128]; /* - * Choose a file format and a data source: they're interrelated (LSM is only compatible with - * row-store) and other items depend on them. + * The runs.type configuration allows more than a single type, for example, choosing from either + * RS and VLCS but not FLCS. If there's no table value but there was a global value, re-evaluate + * the original global specification, not the choice set for the global table. */ - if (!config_is_perm("runs.type")) { - if (config_is_perm("runs.source") && DATASOURCE("lsm")) - config_single("runs.type=row", false); + if (!table->v[V_TABLE_RUNS_TYPE].set && tables[0]->v[V_TABLE_RUNS_TYPE].set) { + testutil_check(__wt_snprintf(buf, sizeof(buf), "runs.type=%s", g.runs_type)); + config_single(table, buf, true); + } + + if (!config_explicit(table, "runs.type")) { + if (config_explicit(table, "runs.source") && DATASOURCE(table, "lsm")) + config_single(table, "runs.type=row", false); else switch (mmrand(NULL, 1, 10)) { case 1: case 2: case 3: /* 30% */ - config_single("runs.type=var", false); + config_single(table, "runs.type=var", false); break; case 4: /* 10% */ - if (config_fix()) { - config_single("runs.type=fix", false); + if (config_fix(table)) { + config_single(table, "runs.type=fix", false); break; } /* FALLTHROUGH */ /* 60% */ @@ -119,118 +156,202 @@ config_run(void) case 8: case 9: case 10: - config_single("runs.type=row", false); + config_single(table, "runs.type=row", false); break; } } - if (!config_is_perm("runs.source")) { - config_single("runs.source=table", false); + if (!config_explicit(table, "runs.source")) switch (mmrand(NULL, 1, 5)) { case 1: /* 20% */ - config_single("runs.source=file", false); + config_single(table, "runs.source=file", false); break; case 2: /* 20% */ -#if !defined(DISABLE_RANDOM_LSM_TESTING) +#if 0 /* - * LSM requires a row-store and backing disk. + * We currently disable random LSM testing, that is, it can be specified explicitly but + * we won't randomly choose LSM as a data_source configuration. * - * Configuring truncation or timestamps results in LSM cache problems, don't configure - * LSM if those set. + * LSM requires a row-store and backing disk. Don't configure LSM if in-memory, + * timestamps or truncation are configured, they result in cache problems. * - * XXX Remove the timestamp test when WT-4162 resolved. + * FIXME WT-4162: Remove the timestamp test when WT-4162 resolved. */ - if (g.type != ROW || g.c_in_memory) + if (table->type != ROW || GV(RUNS_IN_MEMORY)) + break; + if (config_explicit(NULL, "transaction.timestamps") && GV(TRANSACTION_TIMESTAMPS)) break; - if (config_is_perm("transaction.timestamps") && g.c_txn_timestamps) + if (GV(BACKUP) && config_explicit(NULL, "backup.incremental") && + g.backup_incr_flag == INCREMENTAL_BLOCK) break; - if (config_is_perm("ops.truncate") && g.c_truncate) + if (config_explicit(table, "ops.truncate") && TV(OPS_TRUNCATE)) break; - config_single("runs.source=lsm", false); + config_single(table, "runs.source=lsm", false); #endif - break; + /* FALLTHROUGH */ case 3: case 4: case 5: /* 60% */ + config_single(table, "runs.source=table", false); break; } - } - /* If data_source and file_type were both "permanent", we may still have a mismatch. */ - if (DATASOURCE("lsm") && g.type != ROW) - testutil_die( - EINVAL, "%s: lsm data_source is only compatible with row file_type\n", progname); + /* If data_source and file_type were both set explicitly, we may still have a mismatch. */ + if (DATASOURCE(table, "lsm") && table->type != ROW) + testutil_die(EINVAL, "%s: lsm data_source is only compatible with row file_type", progname); +} + +/* + * config_table -- + * Finish initialization of a single table. + */ +static void +config_table(TABLE *table, void *arg) +{ + CONFIG *cp; + + (void)arg; /* unused argument */ + + /* + * Choose a file format and a data source: they're interrelated (LSM is only compatible with + * row-store) and other items depend on them. + */ + config_table_am(table); /* * Build the top-level object name: we're overloading data_source in our configuration, LSM * objects are "tables", but files are tested as well. */ - g.uri = dmalloc(256); - strcpy(g.uri, DATASOURCE("file") ? "file:" : "table:"); - strcat(g.uri, WT_NAME); + if (ntables == 0) + testutil_check(__wt_snprintf(table->uri, sizeof(table->uri), "%s", + DATASOURCE(table, "file") ? "file:wt" : "table:wt")); + else + testutil_check(__wt_snprintf(table->uri, sizeof(table->uri), + DATASOURCE(table, "file") ? "file:F%05u" : "table:T%05u", table->id)); + testutil_check( + __wt_snprintf(table->track_prefix, sizeof(table->track_prefix), "table %u", table->id)); + + /* + * For any values set in the base configuration, export them to this table (where this table + * doesn't already have a value set). + */ + if (ntables != 0) + for (cp = configuration_list; cp->name != NULL; ++cp) + if (F_ISSET(cp, C_TABLE) && !table->v[cp->off].set && tables[0]->v[cp->off].set) + config_promote(table, cp, &tables[0]->v[cp->off]); /* Fill in random values for the rest of the run. */ - for (cp = c; cp->name != NULL; ++cp) { - if (F_ISSET(cp, C_IGNORE | C_PERM | C_TEMP)) - continue; + config_random(table, true); - /* - * Boolean flags are 0 or 1, where the variable's "min" value is the percent chance the flag - * is "on" (so "on" if random rolled <= N, otherwise "off"). - */ - if (F_ISSET(cp, C_BOOL)) - testutil_check(__wt_snprintf( - buf, sizeof(buf), "%s=%s", cp->name, mmrand(NULL, 1, 100) <= cp->min ? "on" : "off")); - else - testutil_check(__wt_snprintf( - buf, sizeof(buf), "%s=%" PRIu32, cp->name, mmrand(NULL, cp->min, cp->maxrand))); - config_single(buf, false); + /* Page sizes are configured using powers-of-two or megabytes, convert them. */ + table->max_intl_page = 1U << TV(BTREE_INTERNAL_PAGE_MAX); + table->max_leaf_page = 1U << TV(BTREE_LEAF_PAGE_MAX); + table->max_mem_page = MEGABYTE(TV(BTREE_MEMORY_PAGE_MAX)); + + /* + * Keep the number of rows and keys/values small for in-memory runs (overflow items aren't an + * issue for in-memory configurations and it helps prevents cache overflow). + */ + if (GV(RUNS_IN_MEMORY)) { + if (!config_explicit(table, "runs.rows") && TV(RUNS_ROWS) > 1000000) + config_single(table, "runs.rows=1000000", false); + if (!config_explicit(table, "btree.key_max")) + config_single(table, "btree.key_max=32", false); + if (!config_explicit(table, "btree.key_min")) + config_single(table, "btree.key_min=15", false); + if (!config_explicit(table, "btree.value_max")) + config_single(table, "btree.value_max=80", false); + if (!config_explicit(table, "btree.value_min")) + config_single(table, "btree.value_min=20", false); + } + + /* + * Key/value minimum/maximum are related, correct unless specified by the configuration. Key + * sizes are a row-store consideration: column-store doesn't store keys, a constant of 8 will + * reserve a small amount of additional space. + */ + if (table->type == ROW) { + if (!config_explicit(table, "btree.key_min") && TV(BTREE_KEY_MIN) > TV(BTREE_KEY_MAX)) + TV(BTREE_KEY_MIN) = TV(BTREE_KEY_MAX); + if (!config_explicit(table, "btree.key_max") && TV(BTREE_KEY_MAX) < TV(BTREE_KEY_MIN)) + TV(BTREE_KEY_MAX) = TV(BTREE_KEY_MIN); + if (TV(BTREE_KEY_MIN) > TV(BTREE_KEY_MAX)) + testutil_die(EINVAL, "btree.key_min may not be larger than btree.key_max"); + } else + TV(BTREE_KEY_MIN) = TV(BTREE_KEY_MAX) = 8; + if (!config_explicit(table, "btree.value_min") && TV(BTREE_VALUE_MIN) > TV(BTREE_VALUE_MAX)) + TV(BTREE_VALUE_MIN) = TV(BTREE_VALUE_MAX); + if (!config_explicit(table, "btree.value_max") && TV(BTREE_VALUE_MAX) < TV(BTREE_VALUE_MIN)) + TV(BTREE_VALUE_MAX) = TV(BTREE_VALUE_MIN); + if (TV(BTREE_VALUE_MIN) > TV(BTREE_VALUE_MAX)) + testutil_die(EINVAL, "btree.value_min may not be larger than btree.value_max"); + + /* + * If common key prefixes are configured, add prefix compression if no explicit choice was made + * and track the largest common key prefix in the run. + */ + if (TV(BTREE_PREFIX_LEN) != 0) { + if (TV(BTREE_PREFIX_COMPRESSION) == 0 && + !config_explicit(table, "btree.prefix_compression")) + config_single(table, "btree.prefix_compression=on", false); + g.prefix_len_max = WT_MAX(g.prefix_len_max, TV(BTREE_PREFIX_LEN)); } - /* Only row-store tables support collation order. */ - if (g.type != ROW) - config_single("btree.reverse=off", false); - - /* First, transaction configuration, it configures other features. */ - config_transaction(); - - /* Simple selection. */ - config_backup_incr(); - config_checkpoint(); - config_checksum(); - config_compression("btree.compression"); - config_compression("logging.compression"); - config_encryption(); - config_prefix(); - - /* Configuration based on the configuration already chosen. */ - config_directio(); - config_pct(); - config_cache(); - - /* Give in-memory, LSM and backward compatible configurations a final review. */ - if (g.c_in_memory != 0) + config_checksum(table); + config_compression(table, "btree.compression"); + config_pct(table); + + /* The number of rows in the table can change, get a local copy of the starting value. */ + table->rows_current = TV(RUNS_ROWS); + + /* Column-store tables require special row insert resolution. */ + if (table->type != ROW) + g.column_store_config = true; + + /* Only row-store tables support a collation order. */ + if (table->type != ROW) + config_off(table, "btree.reverse"); + + /* Give LSM a final review and flag if there's at least one LSM data source. */ + if (DATASOURCE(table, "lsm")) { + g.lsm_config = true; + config_lsm_reset(table); + } +} + +/* + * config_run -- + * Run initialization. + */ +void +config_run(void) +{ + config_in_memory(); /* Periodically run in-memory. */ + + config_random(tables[0], false); /* Configure the remaining global name space. */ + + tables_apply(config_table, NULL); /* Configure the tables. */ + + /* Order can be important, don't shuffle without careful consideration. */ + config_transaction(); /* Transactions */ + config_backup_incr(); /* Incremental backup */ + config_checkpoint(); /* Checkpoints */ + config_compression(NULL, "logging.compression"); /* Logging compression */ + config_directio(); /* Direct I/O */ + config_encryption(); /* Encryption */ + + /* If doing an in-memory run, make sure we haven't configured something that won't work. */ + if (GV(RUNS_IN_MEMORY)) config_in_memory_reset(); - if (DATASOURCE("lsm")) - config_lsm_reset(); - config_backward_compatible(); /* - * Key/value minimum/maximum are related, correct unless specified by the configuration. + * If built in a branch that doesn't support all current options, or creating a database for + * such an environment, strip out configurations that won't work. */ - if (!config_is_perm("btree.key_min") && g.c_key_min > g.c_key_max) - g.c_key_min = g.c_key_max; - if (!config_is_perm("btree.key_max") && g.c_key_max < g.c_key_min) - g.c_key_max = g.c_key_min; - if (g.c_key_min > g.c_key_max) - testutil_die(EINVAL, "key_min may not be larger than key_max"); - - if (!config_is_perm("btree.value_min") && g.c_value_min > g.c_value_max) - g.c_value_min = g.c_value_max; - if (!config_is_perm("btree.value_max") && g.c_value_max < g.c_value_min) - g.c_value_max = g.c_value_min; - if (g.c_value_min > g.c_value_max) - testutil_die(EINVAL, "value_min may not be larger than value_max"); + if (g.backward_compatible) + config_backward_compatible(); + + config_cache(); /* Cache */ /* * Run-length is configured by a number of operations and a timer. @@ -244,14 +365,14 @@ config_run(void) * operations but the rest of the configuration means operations take a long time to complete * (for example, a small cache and many worker threads), don't let it run forever. */ - if (config_is_perm("runs.timer")) { - if (!config_is_perm("runs.ops")) - config_single("runs.ops=0", false); + if (config_explicit(NULL, "runs.timer")) { + if (!config_explicit(NULL, "runs.ops")) + config_single(NULL, "runs.ops=0", false); } else { - if (!config_is_perm("runs.ops")) - config_single("runs.timer=30", false); + if (!config_explicit(NULL, "runs.ops")) + config_single(NULL, "runs.timer=30", false); else - config_single("runs.timer=360", false); + config_single(NULL, "runs.timer=360", false); } } @@ -262,12 +383,8 @@ config_run(void) static void config_backup_incr(void) { - /* Incremental backup requires backup. */ - if (g.c_backups == 0) { - if (!config_is_perm("backup.incremental")) - config_single("backup.incremental=off", false); - if (g.c_backup_incr_flag != INCREMENTAL_OFF) - testutil_die(EINVAL, "backup.incremental requires backups be configured"); + if (GV(BACKUP) == 0) { + config_off(NULL, "backup.incremental"); return; } @@ -275,10 +392,10 @@ config_backup_incr(void) * Incremental backup using log files is incompatible with logging archival. Testing log file * archival doesn't seem as useful as testing backup, let the backup configuration override. */ - if (config_is_perm("backup.incremental")) { - if (g.c_backup_incr_flag == INCREMENTAL_LOG) + if (config_explicit(NULL, "backup.incremental")) { + if (g.backup_incr_flag == INCREMENTAL_LOG) config_backup_incr_log_compatibility_check(); - if (g.c_backup_incr_flag == INCREMENTAL_BLOCK) + if (g.backup_incr_flag == INCREMENTAL_BLOCK) config_backup_incr_granularity(); return; } @@ -291,15 +408,15 @@ config_backup_incr(void) case 1: /* 30% full backup only */ case 2: case 3: - config_single("backup.incremental=off", false); + config_off(NULL, "backup.incremental"); break; case 4: /* 30% log based incremental */ case 5: case 6: - if (!g.c_logging_archive || !config_is_perm("logging.archive")) { - if (g.c_logging_archive) - config_single("logging.archive=0", false); - config_single("backup.incremental=log", false); + if (!GV(LOGGING_ARCHIVE) || !config_explicit(NULL, "logging.archive")) { + if (GV(LOGGING_ARCHIVE)) + config_off(NULL, "logging.archive"); + config_single(NULL, "backup.incremental=log", false); break; } /* FALLTHROUGH */ @@ -307,7 +424,7 @@ config_backup_incr(void) case 8: case 9: case 10: - config_single("backup.incremental=block", false); + config_single(NULL, "backup.incremental=block", false); config_backup_incr_granularity(); break; } @@ -323,7 +440,7 @@ config_backup_incr_granularity(void) uint32_t granularity, i; char confbuf[128]; - if (config_is_perm("backup.incr_granularity")) + if (config_explicit(NULL, "backup.incr_granularity")) return; /* @@ -353,8 +470,27 @@ config_backup_incr_granularity(void) } testutil_check( - __wt_snprintf(confbuf, sizeof(confbuf), "backup.incr_granularity=%u", granularity)); - config_single(confbuf, false); + __wt_snprintf(confbuf, sizeof(confbuf), "backup.incr_granularity=%" PRIu32, granularity)); + config_single(NULL, confbuf, false); +} + +/* + * config_backward_compatible_table -- + * Backward compatibility configuration, per table. + */ +static void +config_backward_compatible_table(TABLE *table, void *arg) +{ + (void)arg; /* unused argument */ + +#undef BC_CHECK +#define BC_CHECK(name, flag) \ + if (TV(flag)) { \ + if (config_explicit(table, name)) \ + testutil_die(EINVAL, "%s not supported in backward compatibility mode", name); \ + config_off(table, name); \ + } + BC_CHECK("btree.prefix_len", BTREE_PREFIX_LEN); } /* @@ -364,51 +500,22 @@ config_backup_incr_granularity(void) static void config_backward_compatible(void) { - bool backward_compatible; - - /* - * If built in a branch that doesn't support all current options, or creating a database for - * such an environment, strip out configurations that won't work. - */ - backward_compatible = g.backward_compatible; -#if WIREDTIGER_VERSION_MAJOR < 10 - backward_compatible = true; -#endif - if (!backward_compatible) - return; - - if (g.c_mmap_all) { - if (config_is_perm("disk.mmap_all")) - testutil_die(EINVAL, "disk.mmap_all not supported in backward compatibility mode"); - config_single("disk.mmap_all=off", false); - } - - if (g.c_timing_stress_checkpoint_reserved_txnid_delay) { - if (config_is_perm("stress.checkpoint_reserved_txnid_delay")) - testutil_die(EINVAL, - "stress.checkpoint_reserved_txnid_delay not supported in backward compatibility " - "mode"); - config_single("stress.checkpoint_reserved_txnid_delay=off", false); - } - - if (g.c_timing_stress_hs_sweep) { - if (config_is_perm("stress.hs_sweep")) - testutil_die(EINVAL, "stress.hs_sweep not supported in backward compatibility mode"); - config_single("stress.hs_sweep=off", false); +#undef BC_CHECK +#define BC_CHECK(name, flag) \ + if (GV(flag)) { \ + if (config_explicit(NULL, name)) \ + testutil_die(EINVAL, "%s not supported in backward compatibility mode", name); \ + config_off(NULL, name); \ } - if (g.c_timing_stress_hs_checkpoint_delay) { - if (config_is_perm("stress.hs_checkpoint_delay")) - testutil_die( - EINVAL, "stress.hs_checkpoint_delay not supported in backward compatibility mode"); - config_single("stress.hs_checkpoint_delay=off", false); - } + BC_CHECK("disk.mmap_all", DISK_MMAP_ALL); + BC_CHECK("block_cache", BLOCK_CACHE); + BC_CHECK("stress.checkpoint_reserved_txnid_delay", STRESS_CHECKPOINT_RESERVED_TXNID_DELAY); + BC_CHECK("stress.hs_checkpoint_delay", STRESS_HS_CHECKPOINT_DELAY); + BC_CHECK("stress.hs_search", STRESS_HS_SEARCH); + BC_CHECK("stress.hs_sweep", STRESS_HS_SWEEP); - if (g.c_timing_stress_hs_search) { - if (config_is_perm("stress.hs_search")) - testutil_die(EINVAL, "stress.hs_search not supported in backward compatibility mode"); - config_single("stress.hs_search=off", false); - } + tables_apply(config_backward_compatible_table, NULL); } /* @@ -418,22 +525,38 @@ config_backward_compatible(void) static void config_cache(void) { - uint32_t required, workers; - - /* Page sizes are powers-of-two for bad historic reasons. */ - g.intl_page_max = 1U << g.c_intl_page_max; - g.leaf_page_max = 1U << g.c_leaf_page_max; + uint64_t cache; + uint32_t workers; - /* Check if a minimum cache size has been specified. */ - if (config_is_perm("cache")) { - if (config_is_perm("cache.minimum") && g.c_cache_minimum != 0 && - g.c_cache < g.c_cache_minimum) + /* Check if both min and max cache sizes have been specified and if they're consistent. */ + if (config_explicit(NULL, "cache")) { + if (config_explicit(NULL, "cache.minimum") && GV(CACHE) < GV(CACHE_MINIMUM)) testutil_die(EINVAL, "minimum cache set larger than cache (%" PRIu32 " > %" PRIu32 ")", - g.c_cache_minimum, g.c_cache); + GV(CACHE_MINIMUM), GV(CACHE)); return; } - g.c_cache = WT_MAX(g.c_cache, g.c_cache_minimum); + GV(CACHE) = GV(CACHE_MINIMUM); + + /* + * If it's an in-memory run, size the cache at 2x the maximum initial data set. This calculation + * is done in bytes, convert to megabytes before testing against the cache. + */ + if (GV(RUNS_IN_MEMORY)) { + cache = table_sumv(V_TABLE_BTREE_KEY_MAX) + table_sumv(V_TABLE_BTREE_VALUE_MAX); + cache *= table_sumv(V_TABLE_RUNS_ROWS); + cache *= 2; + cache /= WT_MEGABYTE; /* NOT in MB units, convert for cache test */ + if (GV(CACHE) < cache) + GV(CACHE) = (uint32_t)cache; + } + + /* Sum the number of workers. */ + workers = GV(RUNS_THREADS); + if (GV(OPS_HS_CURSOR)) + ++workers; + if (GV(OPS_RANDOM_CURSOR)) + ++workers; /* * Maximum internal/leaf page size sanity. @@ -442,33 +565,34 @@ config_cache(void) * cache with pinned pages, that is, every thread consuming an internal page and a leaf page (or * a pair of leaf pages for cursor movements). * - * Maximum memory pages are in units of MB. - * * This code is what dramatically increases the cache size when there are lots of threads, it * grows the cache to several megabytes per thread. */ - workers = g.c_threads; - if (g.c_hs_cursor) - ++workers; - if (g.c_random_cursor) - ++workers; - g.c_cache = WT_MAX(g.c_cache, 2 * workers * g.c_memory_page_max); + cache = table_sumv(V_TABLE_BTREE_MEMORY_PAGE_MAX); /* in MB units, no conversion to cache */ + cache *= workers; + cache *= 2; + if (GV(CACHE) < cache) + GV(CACHE) = (uint32_t)cache; /* - * Ensure cache size sanity for LSM runs. An LSM tree open requires 3 - * chunks plus a page for each participant in up to three concurrent - * merges. Integrate a thread count into that calculation by requiring - * 3 chunks/pages per configured thread. That might be overkill, but - * LSM runs are more sensitive to small caches than other runs, and a - * generous cache avoids stalls we're not interested in chasing. + * Ensure cache size sanity for LSM runs. An LSM tree open requires 3 chunks plus a page for + * each participant in up to three concurrent merges. Integrate a thread count into that + * calculation by requiring 3 chunks/pages per configured thread. That might be overkill, but + * LSM runs are more sensitive to small caches than other runs, and a generous cache avoids + * stalls we're not interested in chasing. */ - if (DATASOURCE("lsm")) { - required = WT_LSM_TREE_MINIMUM_SIZE( - g.c_chunk_size * WT_MEGABYTE, workers * g.c_merge_max, workers * g.leaf_page_max); - required = (required + (WT_MEGABYTE - 1)) / WT_MEGABYTE; - if (g.c_cache < required) - g.c_cache = required; + if (g.lsm_config) { + cache = WT_LSM_TREE_MINIMUM_SIZE(table_sumv(V_TABLE_LSM_CHUNK_SIZE) * WT_MEGABYTE, + workers * table_sumv(V_TABLE_LSM_MERGE_MAX), + workers * table_sumv(V_TABLE_BTREE_LEAF_PAGE_MAX) * WT_MEGABYTE); + cache = (cache + (WT_MEGABYTE - 1)) / WT_MEGABYTE; + if (GV(CACHE) < cache) + GV(CACHE) = (uint32_t)cache; } + + /* Give any block cache 20% of the total cache size, over and above the cache. */ + if (GV(BLOCK_CACHE) != 0) + GV(BLOCK_CACHE_SIZE) = (GV(CACHE) + 4) / 5; } /* @@ -479,19 +603,19 @@ static void config_checkpoint(void) { /* Choose a checkpoint mode if nothing was specified. */ - if (!config_is_perm("checkpoint")) + if (!config_explicit(NULL, "checkpoint")) switch (mmrand(NULL, 1, 20)) { case 1: case 2: case 3: case 4: /* 20% */ - config_single("checkpoint=wiredtiger", false); + config_single(NULL, "checkpoint=wiredtiger", false); break; case 5: /* 5 % */ - config_single("checkpoint=off", false); + config_off(NULL, "checkpoint"); break; default: /* 75% */ - config_single("checkpoint=on", false); + config_single(NULL, "checkpoint=on", false); break; } } @@ -501,25 +625,25 @@ config_checkpoint(void) * Checksum configuration. */ static void -config_checksum(void) +config_checksum(TABLE *table) { /* Choose a checksum mode if nothing was specified. */ - if (!config_is_perm("disk.checksum")) + if (!config_explicit(table, "disk.checksum")) switch (mmrand(NULL, 1, 10)) { case 1: case 2: case 3: case 4: /* 40% */ - config_single("disk.checksum=on", false); + config_single(table, "disk.checksum=on", false); break; case 5: /* 10% */ - config_single("disk.checksum=off", false); + config_off(table, "disk.checksum"); break; case 6: /* 10% */ - config_single("disk.checksum=uncompressed", false); + config_single(table, "disk.checksum=uncompressed", false); break; default: /* 40% */ - config_single("disk.checksum=unencrypted", false); + config_single(table, "disk.checksum=unencrypted", false); break; } } @@ -529,31 +653,34 @@ config_checksum(void) * Compression configuration. */ static void -config_compression(const char *conf_name) +config_compression(TABLE *table, const char *conf_name) { char confbuf[128]; const char *cstr; - /* Return if already specified. */ - if (config_is_perm(conf_name)) + /* Ignore logging compression if we're not doing logging. */ + if (strcmp(conf_name, "logging.compression") == 0 && GV(LOGGING) == 0) { + config_off(NULL, "logging.compression"); return; + } - /* - * Don't configure a compression engine for logging if logging isn't configured (it won't break, - * but it's confusing). - */ - cstr = "none"; - if (strcmp(conf_name, "logging.compression") == 0 && g.c_logging == 0) { - testutil_check(__wt_snprintf(confbuf, sizeof(confbuf), "%s=%s", conf_name, cstr)); - config_single(confbuf, false); - return; + /* Return if already specified and it's a current compression engine. */ + if (config_explicit(table, conf_name)) { + cstr = "off"; + if (strcmp(conf_name, "logging.compression") == 0) + cstr = GVS(LOGGING_COMPRESSION); + else if (strcmp(conf_name, "btree.compression") == 0) + cstr = TVS(BTREE_COMPRESSION); + if (memcmp(cstr, "bzip", strlen("bzip")) != 0) + return; + WARN("%s: bzip compression no longer supported", conf_name); } /* - * Select a compression type from the list of built-in engines. - * - * Listed percentages are only correct if all of the possible engines are compiled in. + * Select a compression type from the list of built-in engines. Listed percentages are only + * correct if all of the possible engines are compiled in. */ + cstr = "off"; switch (mmrand(NULL, 1, 20)) { #ifdef HAVE_BUILTIN_EXTENSION_LZ4 case 1: @@ -566,9 +693,9 @@ config_compression(const char *conf_name) case 4: case 5: case 6: - case 7: /* 30% snappy */ + case 7: case 8: - case 9: + case 9: /* 30% snappy */ cstr = "snappy"; break; #endif @@ -596,52 +723,51 @@ config_compression(const char *conf_name) } testutil_check(__wt_snprintf(confbuf, sizeof(confbuf), "%s=%s", conf_name, cstr)); - config_single(confbuf, false); + config_single(table, confbuf, false); } /* - * config_directio + * config_directio -- * Direct I/O configuration. */ static void config_directio(void) { /* - * We don't roll the dice and set direct I/O, it has to be set explicitly. For that reason, any - * incompatible "permanent" option set with direct I/O is a configuration error. + * We don't roll the dice and set direct I/O, it has to be set explicitly. If there are any + * incompatible configurations set explicitly, turn off direct I/O, otherwise turn off the + * incompatible configurations. */ - if (!g.c_direct_io) + if (!GV(DISK_DIRECT_IO)) return; + testutil_assert(config_explicit(NULL, "disk.direct_io") == true); + +#undef DIO_CHECK +#define DIO_CHECK(name, flag) \ + if (GV(flag)) { \ + if (config_explicit(NULL, name)) { \ + WARN("%s not supported with direct I/O, turning off direct I/O", name); \ + config_off(NULL, "disk.direct_io"); \ + return; \ + } \ + config_off(NULL, name); \ + } /* * Direct I/O may not work with backups, doing copies through the buffer cache after configuring * direct I/O in Linux won't work. If direct I/O is configured, turn off backups. */ - if (g.c_backups) { - if (config_is_perm("backup")) - testutil_die(EINVAL, "direct I/O is incompatible with backup configurations"); - config_single("backup=off", false); - } + DIO_CHECK("backup", BACKUP); - /* - * Direct I/O may not work with imports for the same reason as for backups. - */ - if (g.c_import) { - if (config_is_perm("import")) - testutil_die(EINVAL, "direct I/O is incompatible with import configurations"); - config_single("import=0", false); - } + /* Direct I/O may not work with imports for the same reason as for backups. */ + DIO_CHECK("import", IMPORT); /* * Direct I/O may not work with mmap. Theoretically, Linux ignores direct I/O configurations in * the presence of shared cache configurations (including mmap), but we've seen file corruption * and it doesn't make much sense (the library disallows the combination). */ - if (g.c_mmap_all != 0) { - if (config_is_perm("disk.mmap_all")) - testutil_die(EINVAL, "direct I/O is incompatible with mmap_all configurations"); - config_single("disk.mmap_all=off", false); - } + DIO_CHECK("disk.mmap_all", DISK_MMAP_ALL); /* * Turn off all external programs. Direct I/O is really, really slow on some machines and it can @@ -649,11 +775,7 @@ config_directio(void) * format just hung, and the 15-minute timeout isn't effective. We could play games to handle * child process termination, but it's not worth the effort. */ - if (g.c_salvage) { - if (config_is_perm("ops.salvage")) - testutil_die(EINVAL, "direct I/O is incompatible with salvage configurations"); - config_single("ops.salvage=off", false); - } + DIO_CHECK("ops.salvage", OPS_SALVAGE); } /* @@ -663,31 +785,15 @@ config_directio(void) static void config_encryption(void) { - const char *cstr; - - /* - * Encryption: choose something if encryption wasn't specified. - */ - if (!config_is_perm("disk.encryption")) { - cstr = "disk.encryption=none"; - switch (mmrand(NULL, 1, 10)) { - case 1: - case 2: - case 3: - case 4: - case 5: /* 70% no encryption */ - case 6: - case 7: - break; - case 8: - case 9: - case 10: /* 30% rotn */ - cstr = "disk.encryption=rotn-7"; - break; - } + /* Encryption: choose something if encryption wasn't specified. */ + if (config_explicit(NULL, "disk.encryption")) + return; - config_single(cstr, false); - } + /* 70% no encryption, 30% rotn */ + if (mmrand(NULL, 1, 10) < 8) + config_off(NULL, "disk.encryption"); + else + config_single(NULL, "disk.encryption=rotn-7", false); } /* @@ -695,12 +801,10 @@ config_encryption(void) * Fixed-length column-store configuration. */ static bool -config_fix(void) +config_fix(TABLE *table) { - /* Fixed-length column stores don't support the history store table, so no modify operations. */ - if (config_is_perm("ops.pct.modify")) - return (false); - return (true); + /* Fixed-length column stores don't support modify operations. */ + return (!config_explicit(table, "ops.pct.modify")); } /* @@ -716,29 +820,31 @@ config_in_memory(void) * don't have to configure in-memory every time we configure something like LSM, that's too * painful. */ - if (config_is_perm("backup")) + if (config_explicit(NULL, "backup")) return; - if (config_is_perm("btree.compression")) + if (config_explicit(NULL, "block_cache")) return; - if (config_is_perm("checkpoint")) + if (config_explicit(NULL, "btree.compression")) return; - if (config_is_perm("format.abort")) + if (config_explicit(NULL, "checkpoint")) return; - if (config_is_perm("import")) + if (config_explicit(NULL, "format.abort")) return; - if (config_is_perm("logging")) + if (config_explicit(NULL, "import")) return; - if (config_is_perm("ops.hs_cursor")) + if (config_explicit(NULL, "logging")) return; - if (config_is_perm("ops.salvage")) + if (config_explicit(NULL, "ops.alter")) return; - if (config_is_perm("ops.verify")) + if (config_explicit(NULL, "ops.hs_cursor")) return; - if (config_is_perm("runs.source") && DATASOURCE("lsm")) + if (config_explicit(NULL, "ops.salvage")) + return; + if (config_explicit(NULL, "ops.verify")) return; - if (!config_is_perm("runs.in_memory") && mmrand(NULL, 1, 20) == 1) - g.c_in_memory = 1; + if (!config_explicit(NULL, "runs.in_memory") && mmrand(NULL, 1, 20) == 1) + config_single(NULL, "runs.in_memory=1", false); } /* @@ -748,54 +854,29 @@ config_in_memory(void) static void config_in_memory_reset(void) { - uint32_t cache; - /* Turn off a lot of stuff. */ - if (!config_is_perm("backup")) - config_single("backup=off", false); - if (!config_is_perm("btree.compression")) - config_single("btree.compression=none", false); - if (!config_is_perm("checkpoint")) - config_single("checkpoint=off", false); - if (!config_is_perm("import")) - config_single("import=off", false); - if (!config_is_perm("logging")) - config_single("logging=off", false); - if (!config_is_perm("ops.alter")) - config_single("ops.alter=off", false); - if (!config_is_perm("ops.hs_cursor")) - config_single("ops.hs_cursor=off", false); - if (!config_is_perm("ops.salvage")) - config_single("ops.salvage=off", false); - if (!config_is_perm("ops.verify")) - config_single("ops.verify=off", false); - - /* - * Keep keys/values small, overflow items aren't an issue for in-memory configurations and it - * keeps us from overflowing the cache. - */ - if (!config_is_perm("btree.key_max")) - config_single("btree.key_max=32", false); - if (!config_is_perm("btree.value_max")) - config_single("btree.value_max=80", false); - - /* - * Size the cache relative to the initial data set, use 2x the base size as a minimum. - */ - if (!config_is_perm("cache")) { - cache = g.c_value_max; - if (g.type == ROW) - cache += g.c_key_max; - cache *= g.c_rows; - cache *= 2; - cache /= WT_MEGABYTE; - if (g.c_cache < cache) - g.c_cache = cache; - } + if (!config_explicit(NULL, "backup")) + config_off(NULL, "backup"); + if (!config_explicit(NULL, "block_cache")) + config_off(NULL, "block_cache"); + if (!config_explicit(NULL, "checkpoint")) + config_off(NULL, "checkpoint"); + if (!config_explicit(NULL, "import")) + config_off(NULL, "import"); + if (!config_explicit(NULL, "logging")) + config_off(NULL, "logging"); + if (!config_explicit(NULL, "ops.alter")) + config_off(NULL, "ops.alter"); + if (!config_explicit(NULL, "ops.hs_cursor")) + config_off(NULL, "ops.hs_cursor"); + if (!config_explicit(NULL, "ops.salvage")) + config_off(NULL, "ops.salvage"); + if (!config_explicit(NULL, "ops.verify")) + config_off(NULL, "ops.verify"); } /* - * config_backup_incr_compatibility_check -- + * config_backup_incr_log_compatibility_check -- * Backup incremental log compatibility check. */ static void @@ -805,10 +886,12 @@ config_backup_incr_log_compatibility_check(void) * Incremental backup using log files is incompatible with logging archival. Disable logging * archival if log incremental backup is set. */ - if (g.c_logging_archive && config_is_perm("logging.archive")) - testutil_die(EINVAL, "backup.incremental=log is incompatible with logging.archive"); - if (g.c_logging_archive) - config_single("logging.archive=0", false); + if (GV(LOGGING_ARCHIVE) && config_explicit(NULL, "logging.archive")) + WARN("%s", + "backup.incremental=log is incompatible with logging.archive, turning off " + "logging.archive"); + if (GV(LOGGING_ARCHIVE)) + config_off(NULL, "logging.archive"); } /* @@ -816,45 +899,39 @@ config_backup_incr_log_compatibility_check(void) * LSM configuration review. */ static void -config_lsm_reset(void) +config_lsm_reset(TABLE *table) { /* * Turn off truncate for LSM runs (some configurations with truncate always result in a * timeout). */ - if (!config_is_perm("ops.truncate")) - config_single("ops.truncate=off", false); + if (config_explicit(table, "ops.truncate")) { + if (DATASOURCE(table, "lsm")) + testutil_die(EINVAL, "LSM (currently) incompatible with truncate configurations"); + config_off(table, "ops.truncate"); + } /* - * LSM doesn't currently play nicely with timestamps, don't choose the pair unless forced to. If - * we turn off timestamps, make sure we turn off prepare as well, it requires timestamps. Remove - * this code with WT-4162. + * Turn off prepare and timestamps for LSM runs (prepare requires timestamps). + * + * FIXME: WT-4162. */ - if (!config_is_perm("ops.prepare") && !config_is_perm("transaction.timestamps")) { - config_single("ops.prepare=off", false); - config_single("transaction.timestamps=off", false); - } + if (config_explicit(NULL, "ops.prepare")) + testutil_die(EINVAL, "LSM (currently) incompatible with prepare configurations"); + config_off(NULL, "ops.prepare"); + if (config_explicit(NULL, "transaction.timestamps")) + testutil_die(EINVAL, "LSM (currently) incompatible with timestamp configurations"); + config_off(NULL, "transaction.timestamps"); /* * LSM does not work with block-based incremental backup, change the incremental backup - * mechanism if block based in configured. + * mechanism if configured to be block based. */ - if (g.c_backups) { - if (config_is_perm("backup.incremental") && g.c_backup_incr_flag == INCREMENTAL_BLOCK) - testutil_die(EINVAL, "LSM does not work with backup.incremental=block configuration."); - - if (g.c_backup_incr_flag == INCREMENTAL_BLOCK) - switch (mmrand(NULL, 1, 2)) { - case 1: - /* 50% */ - config_single("backup.incremental=off", false); - break; - case 2: - /* 50% */ - config_single("backup.incremental=log", false); - config_backup_incr_log_compatibility_check(); - break; - } + if (GV(BACKUP)) { + if (config_explicit(NULL, "backup.incremental")) + testutil_die( + EINVAL, "LSM (currently) incompatible with incremental backup configurations"); + config_single(NULL, "backup.incremental=log", false); } } @@ -863,38 +940,62 @@ config_lsm_reset(void) * Configure operation percentages. */ static void -config_pct(void) +config_pct(TABLE *table) { - static struct { + struct { const char *name; /* Operation */ uint32_t *vp; /* Value store */ u_int order; /* Order of assignment */ - } list[] = { - {"ops.pct.delete", &g.c_delete_pct, 0}, - {"ops.pct.insert", &g.c_insert_pct, 0}, -#define CONFIG_MODIFY_ENTRY 2 - {"ops.pct.modify", &g.c_modify_pct, 0}, - {"ops.pct.read", &g.c_read_pct, 0}, - {"ops.pct.write", &g.c_write_pct, 0}, - }; + } list[5]; u_int i, max_order, max_slot, n, pct; + bool slot_available; + +#define CONFIG_MODIFY_ENTRY 2 + list[0].name = "ops.pct.delete"; + list[0].vp = &TV(OPS_PCT_DELETE); + list[0].order = 0; + list[1].name = "ops.pct.insert"; + list[1].vp = &TV(OPS_PCT_INSERT); + list[1].order = 0; + list[2].name = "ops.pct.modify"; + list[2].vp = &TV(OPS_PCT_MODIFY); + list[2].order = 0; + list[3].name = "ops.pct.read"; + list[3].vp = &TV(OPS_PCT_READ); + list[3].order = 0; + list[4].name = "ops.pct.write"; + list[4].vp = &TV(OPS_PCT_WRITE); + list[4].order = 0; /* * Walk the list of operations, checking for an illegal configuration and creating a random * order in the list. */ pct = 0; + slot_available = false; for (i = 0; i < WT_ELEMENTS(list); ++i) - if (config_is_perm(list[i].name)) + if (config_explicit(table, list[i].name)) pct += *list[i].vp; - else + else { list[i].order = mmrand(NULL, 1, 1000); - if (pct > 100) - testutil_die(EINVAL, "operation percentages do not total to 100%%"); + slot_available = true; + } + + /* + * Some older configurations had broken percentages. If summing the explicitly specified + * percentages maxes us out, warn and keep running, leaving unspecified operations at 0. + */ + if (pct > 100 || (pct < 100 && !slot_available)) { + WARN("operation percentages %s than 100, resetting to random values", + pct > 100 ? "greater" : "less"); + for (i = 0; i < WT_ELEMENTS(list); ++i) + list[i].order = mmrand(NULL, 1, 1000); + pct = 0; + } /* Cursor modify isn't possible for fixed-length column store. */ - if (g.type == FIX) { - if (config_is_perm("ops.pct.modify") && g.c_modify_pct != 0) + if (table->type == FIX) { + if (config_explicit(table, "ops.pct.modify") && TV(OPS_PCT_MODIFY) != 0) testutil_die(EINVAL, "WT_CURSOR.modify not supported by fixed-length column store"); list[CONFIG_MODIFY_ENTRY].order = 0; *list[CONFIG_MODIFY_ENTRY].vp = 0; @@ -927,21 +1028,9 @@ config_pct(void) pct -= *list[max_slot].vp; } - testutil_assert( - g.c_delete_pct + g.c_insert_pct + g.c_modify_pct + g.c_read_pct + g.c_write_pct == 100); -} - -/* - * config_prefix -- - * Prefix configuration. - */ -static void -config_prefix(void) -{ - /* Add prefix compression if prefixes are configured and no explicit choice was made. */ - if (g.c_prefix != 0 && g.c_prefix_compression == 0 && - !config_is_perm("btree.prefix_compression")) - config_single("btree.prefix_compression=on", false); + testutil_assert(TV(OPS_PCT_DELETE) + TV(OPS_PCT_INSERT) + TV(OPS_PCT_MODIFY) + + TV(OPS_PCT_READ) + TV(OPS_PCT_WRITE) == + 100); } /* @@ -952,22 +1041,18 @@ static void config_transaction(void) { /* Transaction prepare requires timestamps and is incompatible with logging. */ - if (g.c_prepare && config_is_perm("ops.prepare")) { - if (g.c_logging && config_is_perm("logging")) - testutil_die(EINVAL, "prepare is incompatible with logging"); - if (!g.c_txn_timestamps && config_is_perm("transaction.timestamps")) + if (GV(OPS_PREPARE) && config_explicit(NULL, "ops.prepare")) { + if (!GV(TRANSACTION_TIMESTAMPS) && config_explicit(NULL, "transaction.timestamps")) testutil_die(EINVAL, "prepare requires transaction timestamps"); + if (GV(LOGGING) && config_explicit(NULL, "logging")) + testutil_die(EINVAL, "prepare is incompatible with logging"); } /* Transaction timestamps are incompatible with implicit transactions. */ - if (g.c_txn_timestamps && config_is_perm("transaction.timestamps")) { - if (g.c_txn_implicit && config_is_perm("transaction.implicit")) + if (GV(TRANSACTION_TIMESTAMPS) && config_explicit(NULL, "transaction.timestamps")) { + if (GV(TRANSACTION_IMPLICIT) && config_explicit(NULL, "transaction.implicit")) testutil_die( EINVAL, "transaction.timestamps is incompatible with implicit transactions"); - - /* FIXME-WT-6431: temporarily disable salvage with timestamps. */ - if (g.c_salvage && config_is_perm("ops.salvage")) - testutil_die(EINVAL, "transaction.timestamps is incompatible with salvage"); } /* @@ -978,24 +1063,28 @@ config_transaction(void) * time we check logging, logging must have been required by the run if both logging and prepare * are still set, so we can just turn off prepare in that case). */ - if (g.c_prepare) { - if (!config_is_perm("logging")) - config_single("logging=off", false); - if (!config_is_perm("transaction.timestamps")) - config_single("transaction.timestamps=on", false); + if (GV(OPS_PREPARE)) { + if (!config_explicit(NULL, "logging")) + config_off(NULL, "logging"); + if (!config_explicit(NULL, "transaction.timestamps")) + config_single(NULL, "transaction.timestamps=on", false); } - if (g.c_txn_timestamps) { - if (!config_is_perm("transaction.implicit")) - config_single("transaction.implicit=0", false); - if (!config_is_perm("ops.salvage")) - config_single("ops.salvage=off", false); + if (GV(TRANSACTION_TIMESTAMPS)) { + if (!config_explicit(NULL, "transaction.implicit")) + config_off(NULL, "transaction.implicit"); + if (!config_explicit(NULL, "ops.salvage")) + config_off(NULL, "ops.salvage"); } - if (g.c_logging) - config_single("ops.prepare=off", false); - if (g.c_txn_implicit) - config_single("transaction.timestamps=off", false); - if (g.c_salvage) - config_single("transaction.timestamps=off", false); + if (GV(LOGGING)) + config_off(NULL, "ops.prepare"); + if (GV(TRANSACTION_IMPLICIT)) + config_off(NULL, "transaction.timestamps"); + if (GV(OPS_SALVAGE)) + config_off(NULL, "transaction.timestamps"); + + /* Transaction timestamps configures format behavior, flag it. */ + if (GV(TRANSACTION_TIMESTAMPS)) + g.transaction_timestamps_config = true; } /* @@ -1023,13 +1112,75 @@ config_error(void) fprintf(stderr, "\n"); fprintf(stderr, "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=\n"); fprintf(stderr, "Configuration names:\n"); - for (max_name = 0, cp = c; cp->name != NULL; ++cp) + for (max_name = 0, cp = configuration_list; cp->name != NULL; ++cp) max_name = WT_MAX(max_name, strlen(cp->name)); - for (cp = c; cp->name != NULL; ++cp) + for (cp = configuration_list; cp->name != NULL; ++cp) fprintf(stderr, "%*s: %s\n", (int)max_name, cp->name, cp->desc); } /* + * config_print_one -- + * Print out a single configuration setting. + */ +static void +config_print_one(FILE *fp, CONFIG *cp, CONFIGV *v, const char *prefix) +{ + const char *cstr; + + if (F_ISSET(cp, C_STRING)) { + /* Historic versions of format expect "none", instead of "off", for a few configurations. */ + cstr = v->vstr == NULL ? "off" : v->vstr; + if (strcmp(cstr, "off") == 0 && + (cp->off == V_GLOBAL_DISK_ENCRYPTION || cp->off == V_GLOBAL_LOGGING_COMPRESSION || + cp->off == V_TABLE_BTREE_COMPRESSION)) + cstr = "none"; + fprintf(fp, "%s%s=%s\n", prefix, cp->name, cstr); + } else + fprintf(fp, "%s%s=%" PRIu32 "\n", prefix, cp->name, v->v); +} + +/* + * config_print_table -- + * Print per-table information. + */ +static void +config_print_table(FILE *fp, TABLE *table) +{ + CONFIG *cp; + CONFIGV *v, *gv; + char buf[128]; + bool lsm; + + testutil_check(__wt_snprintf(buf, sizeof(buf), "table%u.", table->id)); + fprintf(fp, "############################################\n"); + fprintf(fp, "# TABLE PARAMETERS: table %u\n", table->id); + fprintf(fp, "############################################\n"); + + lsm = DATASOURCE(table, "lsm"); + for (cp = configuration_list; cp->name != NULL; ++cp) { + /* Skip global items. */ + if (!F_ISSET(cp, C_TABLE)) + continue; + /* Skip mismatched objects and configurations. */ + if (!lsm && F_ISSET(cp, C_TYPE_LSM)) + continue; + if (!C_TYPE_MATCH(cp, table->type)) + continue; + + gv = &tables[0]->v[cp->off]; + v = &table->v[cp->off]; + + /* Skip entries that match any global setting. */ + if (gv->set && v->v == gv->v && + ((v->vstr == NULL && gv->vstr == NULL) || + (v->vstr != NULL && gv->vstr != NULL && strcmp(v->vstr, gv->vstr) == 0))) + continue; + + config_print_one(fp, cp, v, buf); + } +} + +/* * config_print -- * Print configuration information. */ @@ -1037,7 +1188,9 @@ void config_print(bool error_display) { CONFIG *cp; + CONFIGV *gv; FILE *fp; + uint32_t i; /* Reopening an existing database should leave the existing CONFIG file. */ if (g.reopen) @@ -1049,17 +1202,36 @@ config_print(bool error_display) testutil_die(errno, "fopen: %s", g.home_config); fprintf(fp, "############################################\n"); - fprintf(fp, "# RUN PARAMETERS: V2\n"); + fprintf(fp, "# RUN PARAMETERS: V3\n"); fprintf(fp, "############################################\n"); - /* Display configuration values. */ - for (cp = c; cp->name != NULL; ++cp) - if (F_ISSET(cp, C_STRING)) - fprintf(fp, "%s=%s\n", cp->name, *cp->vstr == NULL ? "" : *cp->vstr); - else - fprintf(fp, "%s=%" PRIu32 "\n", cp->name, *cp->v); + /* Display global configuration values. */ + for (cp = configuration_list; cp->name != NULL; ++cp) { + /* Skip table count if tables not configured (implying an old-style CONFIG file). */ + if (ntables == 0 && cp->off == V_GLOBAL_RUNS_TABLES) + continue; - fprintf(fp, "############################################\n"); + /* Skip mismatched objects and configurations. */ + if (!g.lsm_config && F_ISSET(cp, C_TYPE_LSM)) + continue; + + /* Skip mismatched table items if the global table is the only table. */ + if (ntables == 0 && F_ISSET(cp, C_TABLE) && !C_TYPE_MATCH(cp, tables[0]->type)) + continue; + + /* Skip table items if not explicitly set and the global table isn't the only table. */ + gv = &tables[0]->v[cp->off]; + if (ntables > 0 && F_ISSET(cp, C_TABLE) && !gv->set) + continue; + + /* Print everything else. */ + config_print_one(fp, cp, gv, ""); + } + + /* Display per-table configuration values. */ + if (ntables != 0) + for (i = 1; i <= ntables; ++i) + config_print_table(fp, tables[i]); /* Flush so we're up-to-date on error. */ (void)fflush(fp); @@ -1078,6 +1250,16 @@ config_file(const char *name) FILE *fp; char buf[256], *p, *t; + /* + * Turn off multi-table configuration for all configuration files, for backward compatibility. + * This doesn't stop multiple table configurations, using either "runs.tables" or an explicit + * mention of a table, it only prevents CONFIG files without a table reference from configuring + * tables. This should only affect putting some non-table-specific configurations into a file + * and running that file as a CONFIG, expecting a multi-table test, and means old-style CONFIG + * files don't suddenly turn into multiple table tests. + */ + g.multi_table_config = false; + if ((fp = fopen(name, "r")) == NULL) testutil_die(errno, "fopen: %s", name); @@ -1092,7 +1274,7 @@ config_file(const char *name) *p = '\0'; break; } - if (*p == '#') { /* Comment, skip the line */ + if (*p == '#') { /* Comment */ t = p; break; } @@ -1104,7 +1286,7 @@ config_file(const char *name) } if (*t == '\0' || *t == '#') continue; - config_single(t, true); + config_single(NULL, t, true); } fclose_and_clear(&fp); } @@ -1116,61 +1298,57 @@ config_file(const char *name) void config_clear(void) { - CONFIG *cp; + u_int i, j, slots; - /* Clear all allocated configuration data. */ - for (cp = c; cp->name != NULL; ++cp) - if (cp->vstr != NULL) { - free((void *)*cp->vstr); - *cp->vstr = NULL; - } - free(g.uri); - g.uri = NULL; -} + /* Clear all allocated configuration data in the tables array. */ + slots = ntables == 0 ? 1 : ntables; + for (i = 0; i < slots; ++i) { + free(tables[i]->val_base); -/* - * config_reset -- - * Clear per-run configuration values. - */ -static void -config_reset(void) -{ - CONFIG *cp; - - /* Clear temporary allocated configuration data. */ - for (cp = c; cp->name != NULL; ++cp) { - F_CLR(cp, C_TEMP); - if (!F_ISSET(cp, C_PERM) && cp->vstr != NULL) { - free((void *)*cp->vstr); - *cp->vstr = NULL; - } + for (j = 0; j < V_ELEMENT_COUNT; ++j) + free(tables[i]->v[j].vstr); + free(tables[i]); } - free(g.uri); - g.uri = NULL; } /* - * config_find - * Find a specific configuration entry. + * config_find -- + * Find a specific configuration entry. */ static CONFIG * config_find(const char *s, size_t len, bool fatal) { CONFIG *cp; - for (cp = c; cp->name != NULL; ++cp) + for (cp = configuration_list; cp->name != NULL; ++cp) if (strncmp(s, cp->name, len) == 0 && cp->name[len] == '\0') return (cp); /* Optionally ignore unknown keywords, it makes it easier to run old CONFIG files. */ if (fatal) - testutil_die(EINVAL, "%s: %s: unknown required configuration keyword\n", progname, s); + testutil_die(EINVAL, "%s: %s: unknown required configuration keyword", progname, s); - fprintf(stderr, "%s: %s: WARNING, ignoring unknown configuration keyword\n", progname, s); + WARN("%s: ignoring unknown configuration keyword", s); return (NULL); } /* + * config_off -- + * Turn a configuration value off. + */ +static void +config_off(TABLE *table, const char *s) +{ + CONFIG *cp; + char buf[100]; + + cp = config_find(s, strlen(s), true); + testutil_check( + __wt_snprintf(buf, sizeof(buf), "%s=%s", s, F_ISSET(cp, C_BOOL | C_STRING) ? "off" : "0")); + config_single(table, buf, false); +} + +/* * config_value -- * String to long helper function. */ @@ -1190,70 +1368,137 @@ config_value(const char *config, const char *p, int match) } /* + * config_table_extend -- + * Extend the tables array as necessary. + */ +static void +config_table_extend(u_int ntable) +{ + u_int i; + + if (g.backward_compatible) + testutil_die(0, "multiple tables not supported in backward compatibility mode"); + + if (ntable <= ntables) + return; + + /* + * Allocate any new tables structures. (We do it this way, rather than reallocating the whole + * tables array, because our caller doesn't know we're extending the list of tables, and is + * likely holding pointers into the current list of tables. Reallocating the whole array would + * require handling reallocation in our caller, and it's not worth the effort.) + * + * This might be the first extension, reset the base table's ID (for debugging, we should never + * be using a table with ID 0). + */ + for (i = 0; i <= ntable; ++i) { + if (tables[i] == NULL) + tables[i] = dcalloc(1, sizeof(TABLE)); + tables[i]->id = i; + } + ntables = ntable; +} + +/* * config_single -- * Set a single configuration structure value. */ void -config_single(const char *s, bool perm) +config_single(TABLE *table, const char *s, bool explicit) { enum { RANGE_FIXED, RANGE_NONE, RANGE_WEIGHTED } range; CONFIG *cp; + CONFIGV *v; uint32_t steps, v1, v2; + u_long ntable; u_int i; + const u_char *t; const char *equalp, *vp1, *vp2; + char *endptr; + /* + * Check for corrupted input. Format has a syntax checking mode and this simplifies that work by + * checking for any unexpected characters. It's complicated by wiredtiger.config, as that + * configuration option includes JSON characters. + */ + for (t = (const u_char *)s; *t != '\0'; ++t) + if (!__wt_isalnum(*t) && !__wt_isspace(*t) && strchr("\"'()-.:=[]_,", *t) == NULL) + testutil_die( + EINVAL, "%s: configuration contains unexpected character %#x", progname, (u_int)*t); + + /* Skip leading white space. */ while (__wt_isspace((u_char)*s)) ++s; + /* + * If configuring a single table, the table argument will be non-NULL. The configuration itself + * may include a table reference, in which case we extend the table as necessary and select the + * table. + */ + if (table == NULL) { + table = tables[0]; + if (strncmp(s, "table", strlen("table")) == 0) { + errno = 0; + ntable = strtoul(s + strlen("table"), &endptr, 10); + testutil_assert(errno == 0 && endptr[0] == '.'); + config_table_extend((uint32_t)ntable); + table = tables[ntable]; + + s = endptr + 1; + } + } + + /* Process backward compatibility configuration. */ config_compat(&s); if ((equalp = strchr(s, '=')) == NULL) - testutil_die(EINVAL, "%s: %s: illegal configuration value\n", progname, s); + testutil_die(EINVAL, "%s: %s: configuration missing \'=\' character", progname, s); + /* Find the configuration value, and assert it's not a table/global mismatch. */ if ((cp = config_find(s, (size_t)(equalp - s), false)) == NULL) return; + testutil_assert(F_ISSET(cp, C_TABLE) || table == tables[0]); + + /* Ignore tables settings in backward compatible runs. */ + if (g.backward_compatible && cp->off == V_GLOBAL_RUNS_TABLES) { + WARN("backward compatible run, ignoring %s setting", s); + return; + } - F_SET(cp, perm ? C_PERM : C_TEMP); ++equalp; + v = &table->v[cp->off]; if (F_ISSET(cp, C_STRING)) { /* - * Free the previous setting if a configuration has been passed in twice. + * Historically, both "none" and "off" were used for turning off string configurations, now + * we only use "off". */ - if (*cp->vstr != NULL) { - free(*cp->vstr); - *cp->vstr = NULL; - } - - if (strncmp(s, "backup.incremental", strlen("backup.incremental")) == 0) { - config_map_backup_incr(equalp, &g.c_backup_incr_flag); - *cp->vstr = dstrdup(equalp); - } else if (strncmp(s, "checkpoint", strlen("checkpoint")) == 0) { - config_map_checkpoint(equalp, &g.c_checkpoint_flag); - *cp->vstr = dstrdup(equalp); - } else if (strncmp(s, "disk.checksum", strlen("disk.checksum")) == 0) { - config_map_checksum(equalp, &g.c_checksum_flag); - *cp->vstr = dstrdup(equalp); - } else if (strncmp(s, "btree.compression", strlen("btree.compression")) == 0) { - config_map_compression(equalp, &g.c_compression_flag); - *cp->vstr = dstrdup(equalp); - } else if (strncmp(s, "runs.source", strlen("runs.source")) == 0 && + if (strcmp(equalp, "none") == 0) + equalp = "off"; + + if (strncmp(s, "backup.incremental", strlen("backup.incremental")) == 0) + config_map_backup_incr(equalp, &g.backup_incr_flag); + else if (strncmp(s, "checkpoint", strlen("checkpoint")) == 0) + config_map_checkpoint(equalp, &g.checkpoint_config); + else if (strncmp(s, "runs.source", strlen("runs.source")) == 0 && strncmp("file", equalp, strlen("file")) != 0 && strncmp("lsm", equalp, strlen("lsm")) != 0 && strncmp("table", equalp, strlen("table")) != 0) { - testutil_die(EINVAL, "Invalid data source option: %s\n", equalp); - } else if (strncmp(s, "disk.encryption", strlen("disk.encryption")) == 0) { - config_map_encryption(equalp, &g.c_encryption_flag); - *cp->vstr = dstrdup(equalp); + testutil_die(EINVAL, "Invalid data source option: %s", equalp); } else if (strncmp(s, "runs.type", strlen("runs.type")) == 0) { - config_map_file_type(equalp, &g.type); - *cp->vstr = dstrdup(config_file_type(g.type)); - } else if (strncmp(s, "logging.compression", strlen("logging.compression")) == 0) { - config_map_compression(equalp, &g.c_logging_compression_flag); - *cp->vstr = dstrdup(equalp); - } else - *cp->vstr = dstrdup(equalp); + /* Save any global configuration for later table configuration. */ + if (table == tables[0]) + testutil_check(__wt_snprintf(g.runs_type, sizeof(g.runs_type), "%s", equalp)); + + config_map_file_type(equalp, &table->type); + equalp = config_file_type(table->type); + } + + /* Free the previous setting if a configuration has been passed in twice. */ + free(v->vstr); + v->vstr = dstrdup(equalp); + v->set = explicit; return; } @@ -1268,7 +1513,8 @@ config_single(const char *s, bool perm) testutil_die(EINVAL, "%s: %s: value of boolean not 0 or 1", progname, s); } - *cp->v = v1; + v->v = v1; + v->set = explicit; return; } @@ -1287,19 +1533,35 @@ config_single(const char *s, bool perm) range = RANGE_WEIGHTED; } + /* + * Get the value and check the range; zero is optionally an out-of-band "don't set this + * variable" value. + */ v1 = config_value(s, vp1, range == RANGE_NONE ? '\0' : (range == RANGE_FIXED ? '-' : ':')); - if (v1 < cp->min || v1 > cp->maxset) - testutil_die(EINVAL, "%s: %s: value outside min/max values of %" PRIu32 "-%" PRIu32 "\n", - progname, s, cp->min, cp->maxset); + if (!(v1 == 0 && F_ISSET(cp, C_ZERO_NOTSET)) && (v1 < cp->min || v1 > cp->maxset)) { + /* + * Historically, btree.split_pct support ranges < 50; correct the value. + * + * Historically, btree.key_min allows ranges under the minimum; correct the value + */ + if (cp->off == V_TABLE_BTREE_SPLIT_PCT && v1 < 50) { + v1 = 50; + WARN("correcting btree.split_pct value to %" PRIu32, v1); + } else if (cp->off == V_TABLE_BTREE_KEY_MIN && v1 < KEY_LEN_CONFIG_MIN) { + v1 = KEY_LEN_CONFIG_MIN; + WARN("correcting btree.key_min value to %" PRIu32, v1); + } else + testutil_die(EINVAL, "%s: %s: value outside min/max values of %" PRIu32 "-%" PRIu32, + progname, s, cp->min, cp->maxset); + } if (range != RANGE_NONE) { v2 = config_value(s, vp2, '\0'); if (v2 < cp->min || v2 > cp->maxset) - testutil_die(EINVAL, - "%s: %s: value outside min/max values of %" PRIu32 "-%" PRIu32 "\n", progname, s, - cp->min, cp->maxset); + testutil_die(EINVAL, "%s: %s: value outside min/max values of %" PRIu32 "-%" PRIu32, + progname, s, cp->min, cp->maxset); if (v1 > v2) - testutil_die(EINVAL, "%s: %s: illegal numeric range\n", progname, s); + testutil_die(EINVAL, "%s: %s: illegal numeric range", progname, s); if (range == RANGE_FIXED) v1 = mmrand(NULL, (u_int)v1, (u_int)v2); @@ -1318,7 +1580,11 @@ config_single(const char *s, bool perm) } } - *cp->v = v1; + v->v = v1; + v->set = explicit; + + if (strncmp(s, "runs.tables", strlen("runs.tables")) == 0) + config_table_extend((uint32_t)v1); } /* @@ -1418,77 +1684,38 @@ config_map_checkpoint(const char *s, u_int *vp) } /* - * config_map_checksum -- - * Map a checksum configuration to a flag. - */ -static void -config_map_checksum(const char *s, u_int *vp) -{ - if (strcmp(s, "on") == 0) - *vp = CHECKSUM_ON; - else if (strcmp(s, "off") == 0) - *vp = CHECKSUM_OFF; - else if (strcmp(s, "uncompressed") == 0) - *vp = CHECKSUM_UNCOMPRESSED; - else if (strcmp(s, "unencrypted") == 0) - *vp = CHECKSUM_UNENCRYPTED; - else - testutil_die(EINVAL, "illegal checksum configuration: %s", s); -} - -/* - * config_map_compression -- - * Map a compression configuration to a flag. + * config_explicit -- + * Return if a configuration entry is explicitly set (as opposed to being randomly set). */ -static void -config_map_compression(const char *s, u_int *vp) -{ - if (strcmp(s, "none") == 0) - *vp = COMPRESS_NONE; - else if (strcmp(s, "lz4") == 0) - *vp = COMPRESS_LZ4; - else if (strcmp(s, "lz4-noraw") == 0) /* CONFIG compatibility */ - *vp = COMPRESS_LZ4; - else if (strcmp(s, "snappy") == 0) - *vp = COMPRESS_SNAPPY; - else if (strcmp(s, "zlib") == 0) - *vp = COMPRESS_ZLIB; - else if (strcmp(s, "zlib-noraw") == 0) /* CONFIG compatibility */ - *vp = COMPRESS_ZLIB; - else if (strcmp(s, "zstd") == 0) - *vp = COMPRESS_ZSTD; - else - testutil_die(EINVAL, "illegal compression configuration: %s", s); -} - -/* - * config_map_encryption -- - * Map a encryption configuration to a flag. - */ -static void -config_map_encryption(const char *s, u_int *vp) -{ - if (strcmp(s, "none") == 0) - *vp = ENCRYPT_NONE; - else if (strcmp(s, "rotn-7") == 0) - *vp = ENCRYPT_ROTN_7; - else if (strcmp(s, "sodium") == 0) - *vp = ENCRYPT_SODIUM; - else - testutil_die(EINVAL, "illegal encryption configuration: %s", s); -} - -/* - * config_is_perm - * Return if a specific configuration entry was permanently set. - */ -static int -config_is_perm(const char *s) +static bool +config_explicit(TABLE *table, const char *s) { CONFIG *cp; + u_int i; + /* Look up the configuration option. */ cp = config_find(s, strlen(s), true); - return (F_ISSET(cp, C_PERM) ? 1 : 0); + + /* + * If it's a global option, assert our caller didn't ask for a table value, and return if it's + * set in the base values. + */ + if (!F_ISSET(cp, C_TABLE)) { + testutil_assert(table == NULL); + return (tables[0]->v[cp->off].set); + } + + /* If checking a single table, the table argument is non-NULL. */ + if (table != NULL) + return (table->v[cp->off].set); + + /* Otherwise, check if it's set in any table. */ + if (ntables == 0) + return (tables[0]->v[cp->off].set); + for (i = 1; i < ntables; ++i) + if (tables[i]->v[cp->off].set) + return (true); + return (false); } /* diff --git a/src/third_party/wiredtiger/test/format/config.h b/src/third_party/wiredtiger/test/format/config.h index d12c30ed756..471fccd1365 100644 --- a/src/third_party/wiredtiger/test/format/config.h +++ b/src/third_party/wiredtiger/test/format/config.h @@ -1,373 +1,137 @@ -/*- - * Public Domain 2014-present MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ +/* DO NOT EDIT: automatically built by format/config.sh. */ + +#define C_TYPE_MATCH(cp, type) \ + (!F_ISSET(cp, (C_TYPE_FIX | C_TYPE_ROW | C_TYPE_VAR)) || \ + ((type) == FIX && F_ISSET(cp, C_TYPE_FIX)) || ((type) == ROW && F_ISSET(cp, C_TYPE_ROW)) || \ + ((type) == VAR && F_ISSET(cp, C_TYPE_VAR))) -/* - * Configuration for the wts program is an array of string-based parameters. This is the structure - * used to declare them. - */ typedef struct { const char *name; /* Configuration item */ const char *desc; /* Configuration description */ -/* Value is a boolean, yes if roll of 1-to-100 is <= CONFIG->min. */ -#define C_BOOL 0x01u - -/* Not a simple randomization, handle outside the main loop. */ -#define C_IGNORE 0x02u - -/* Value was set from command-line or file, ignore for all runs. */ -#define C_PERM 0x04u - -/* Value isn't random for this run, ignore just for this run. */ -#define C_TEMP 0x08u - -/* Value is a string. */ -#define C_STRING 0x20u - u_int flags; +#define C_BOOL 0x001u /* Boolean (true if roll of 1-to-100 is <= CONFIG->min) */ +#define C_IGNORE 0x002u /* Not a simple randomization, configured specially */ +#define C_STRING 0x004u /* String (rather than integral) */ +#define C_TABLE 0x008u /* Value is per table, not global */ +#define C_TYPE_FIX 0x010u /* Value is only relevant to FLCS */ +#define C_TYPE_LSM 0x020u /* Value is only relevant to LSM */ +#define C_TYPE_ROW 0x040u /* Value is only relevant to RS */ +#define C_TYPE_VAR 0x080u /* Value is only relevant to VLCS */ +#define C_ZERO_NOTSET 0x100u /* Ignore zero values */ + uint32_t flags; uint32_t min; /* Minimum value */ uint32_t maxrand; /* Maximum value randomly chosen */ uint32_t maxset; /* Maximum value explicitly set */ - uint32_t *v; /* Value for this run */ - char **vstr; /* Value for string options */ -} CONFIG; - -#define COMPRESSION_LIST " (none | lz4 | snappy | zlib | zstd)" - -static CONFIG c[] = { - /* 2% */ - {"assert.read_timestamp", "assert read_timestamp", C_BOOL, 2, 0, 0, &g.c_assert_read_timestamp, - NULL}, - - /* 2% */ - {"assert.write_timestamp", "set write_timestamp_usage and assert write_timestamp", C_BOOL, 2, 0, - 0, &g.c_assert_write_timestamp, NULL}, - - /* 20% */ - {"backup", "configure backups", C_BOOL, 20, 0, 0, &g.c_backups, NULL}, - - {"backup.incremental", "backup type (block | log | off)", C_IGNORE | C_STRING, 0, 0, 0, NULL, - &g.c_backup_incremental}, - - {"backup.incr_granularity", "incremental backup block granularity (KB)", 0x0, 4, 16384, 16384, - &g.c_backup_incr_granularity, NULL}, - - {"btree.bitcnt", "fixed-length column-store object size (number of bits)", 0x0, 1, 8, 8, - &g.c_bitcnt, NULL}, - - {"btree.compression", "compression type" COMPRESSION_LIST, C_IGNORE | C_STRING, 0, 0, 0, NULL, - &g.c_compression}, - - /* 20% */ - {"btree.dictionary", "configure dictionary compressed values", C_BOOL, 20, 0, 0, &g.c_dictionary, - NULL}, - - /* 20% */ - {"btree.huffman_value", "configure huffman encoded values", C_BOOL, 20, 0, 0, &g.c_huffman_value, - NULL}, - - /* 95% */ - {"btree.internal_key_truncation", "truncate internal keys", C_BOOL, 95, 0, 0, - &g.c_internal_key_truncation, NULL}, - - {"btree.internal_page_max", "btree internal node maximum size", 0x0, 9, 17, 27, - &g.c_intl_page_max, NULL}, - - {"btree.key_max", "maximum key size", 0x0, 20, 128, MEGABYTE(10), &g.c_key_max, NULL}, - - /* - * A minimum key size of 11 is necessary. Row-store keys have a leading 10-digit number and the - * 11 guarantees we never see a key that we can't convert to a numeric value without formatting - * it first because there's a trailing non-digit character in every key. - */ - {"btree.key_min", "minimum key size", 0x0, 11, 32, 256, &g.c_key_min, NULL}, - - {"btree.leaf_page_max", "btree leaf node maximum size", 0x0, 9, 17, 27, &g.c_leaf_page_max, NULL}, - - {"btree.memory_page_max", "maximum cache page size", 0x0, 1, 10, 128, &g.c_memory_page_max, NULL}, - - {"btree.prefix", "common key prefix", C_BOOL, 3, 0, 0, &g.c_prefix, NULL}, - - /* 80% */ - {"btree.prefix_compression", "configure prefix compressed keys", C_BOOL, 80, 0, 0, - &g.c_prefix_compression, NULL}, - - {"btree.prefix_compression_min", "minimum gain before prefix compression is used (bytes)", 0x0, 0, - 8, 256, &g.c_prefix_compression_min, NULL}, - - {"btree.repeat_data_pct", "duplicate values (percentage)", 0x0, 0, 90, 90, &g.c_repeat_data_pct, - NULL}, - - /* 10% */ - {"btree.reverse", "reverse order collation", C_BOOL, 10, 0, 0, &g.c_reverse, NULL}, - - {"btree.split_pct", "page split size as a percentage of the maximum page size", 0x0, 50, 100, 100, - &g.c_split_pct, NULL}, - - {"btree.value_max", "maximum value size", 0x0, 32, 4096, MEGABYTE(10), &g.c_value_max, NULL}, - - {"btree.value_min", "minimum value size", 0x0, 0, 20, 4096, &g.c_value_min, NULL}, - - {"cache", "cache size (MB)", 0x0, 1, 100, 100 * 1024, &g.c_cache, NULL}, - - {"cache.evict_max", "maximum number of eviction workers", 0x0, 0, 5, 100, &g.c_evict_max, NULL}, - - {"cache.minimum", "minimum cache size (MB)", C_IGNORE, 0, 0, 100 * 1024, &g.c_cache_minimum, - NULL}, - - {"checkpoint", "checkpoint type (on | off | wiredtiger)", C_IGNORE | C_STRING, 0, 0, 0, NULL, - &g.c_checkpoint}, - - {"checkpoint.log_size", "MB of log to wait if wiredtiger checkpoints configured", 0x0, 20, 200, - 1024, &g.c_checkpoint_log_size, NULL}, - - {"checkpoint.wait", "seconds to wait if wiredtiger checkpoints configured", 0x0, 5, 100, 3600, - &g.c_checkpoint_wait, NULL}, - - {"disk.checksum", "checksum type (on | off | uncompressed | unencrypted)", C_IGNORE | C_STRING, 0, - 0, 0, NULL, &g.c_checksum}, - - /* 5% */ - {"disk.data_extend", "configure data file extension", C_BOOL, 5, 0, 0, &g.c_data_extend, NULL}, - - /* 0% */ - {"disk.direct_io", "configure direct I/O for data objects", C_IGNORE | C_BOOL, 0, 0, 1, - &g.c_direct_io, NULL}, - - {"disk.encryption", "encryption type (none | rotn-7)", C_IGNORE | C_STRING, 0, 0, 0, NULL, - &g.c_encryption}, - - /* 10% */ - {"disk.firstfit", "configure first-fit allocation", C_BOOL, 10, 0, 0, &g.c_firstfit, NULL}, - - /* 90% */ - {"disk.mmap", "configure mmap operations (reads only)", C_BOOL, 90, 0, 0, &g.c_mmap, NULL}, - - /* 5% */ - {"disk.mmap_all", "configure mmap operations (read and write)", C_BOOL, 5, 0, 0, &g.c_mmap_all, - NULL}, - - /* 0% */ - {"format.abort", "drop core during timed run", C_BOOL, 0, 0, 0, &g.c_abort, NULL}, - - /* 75% */ - {"format.independent_thread_rng", "configure independent thread RNG space", C_BOOL, 75, 0, 0, - &g.c_independent_thread_rng, NULL}, - - {"format.major_timeout", "long-running operations timeout (minutes)", C_IGNORE, 0, 0, 1000, - &g.c_major_timeout, NULL}, - - /* - * 0% - * FIXME-WT-7418 and FIXME-WT-7510: Temporarily disable import until WT_ROLLBACK error and - * wt_copy_and_sync error is fixed. It should be (C_BOOL, 20, 0, 0). - */ - {"import", "import table from newly created database", C_BOOL, 0, 0, 0, &g.c_import, NULL}, - - /* 50% */ - {"logging", "configure logging", C_BOOL, 50, 0, 0, &g.c_logging, NULL}, - - /* 50% */ - {"logging.archive", "configure log file archival", C_BOOL, 50, 0, 0, &g.c_logging_archive, NULL}, - - {"logging.compression", "logging compression type" COMPRESSION_LIST, C_IGNORE | C_STRING, 0, 0, 0, - NULL, &g.c_logging_compression}, - - {"logging.file_max", "maximum log file size (KB)", 0x0, 100, 512000, 2097152, - &g.c_logging_file_max, NULL}, - - /* 50% */ - {"logging.prealloc", "configure log file pre-allocation", C_BOOL, 50, 0, 0, &g.c_logging_prealloc, - NULL}, - - /* 90% */ - {"lsm.auto_throttle", "throttle LSM inserts", C_BOOL, 90, 0, 0, &g.c_auto_throttle, NULL}, - - /* 95% */ - {"lsm.bloom", "configure bloom filters", C_BOOL, 95, 0, 0, &g.c_bloom, NULL}, - {"lsm.bloom_bit_count", "number of bits per item for bloom filters", 0x0, 4, 64, 1000, - &g.c_bloom_bit_count, NULL}, - - {"lsm.bloom_hash_count", "number of hash values per item for bloom filters", 0x0, 4, 32, 100, - &g.c_bloom_hash_count, NULL}, - - /* 10% */ - {"lsm.bloom_oldest", "configure bloom_oldest=true", C_BOOL, 10, 0, 0, &g.c_bloom_oldest, NULL}, - - {"lsm.chunk_size", "LSM chunk size (MB)", 0x0, 1, 10, 100, &g.c_chunk_size, NULL}, - - {"lsm.merge_max", "maximum number of chunks to include in an LSM merge operation", 0x0, 4, 20, - 100, &g.c_merge_max, NULL}, - - {"lsm.worker_threads", "number of LSM worker threads", 0x0, 3, 4, 20, &g.c_lsm_worker_threads, - NULL}, - - /* 10% */ - {"ops.alter", "configure table alterations", C_BOOL, 10, 0, 0, &g.c_alter, NULL}, - - /* 10% */ - {"ops.compaction", "configure compaction", C_BOOL, 10, 0, 0, &g.c_compact, NULL}, - - /* 50% */ - {"ops.hs_cursor", "configure history store cursor reads", C_BOOL, 50, 0, 0, &g.c_hs_cursor, NULL}, - - {"ops.pct.delete", "delete operations (percentage)", C_IGNORE, 0, 0, 100, &g.c_delete_pct, NULL}, - - {"ops.pct.insert", "insert operations (percentage)", C_IGNORE, 0, 0, 100, &g.c_insert_pct, NULL}, - - {"ops.pct.modify", "modify operations (percentage)", C_IGNORE, 0, 0, 100, &g.c_modify_pct, NULL}, - - {"ops.pct.read", "read operations (percentage)", C_IGNORE, 0, 0, 100, &g.c_read_pct, NULL}, - - {"ops.pct.write", "update operations (percentage)", C_IGNORE, 0, 0, 100, &g.c_write_pct, NULL}, - - /* 5% */ - {"ops.prepare", "configure transaction prepare", C_BOOL, 5, 0, 0, &g.c_prepare, NULL}, - - /* 10% */ - {"ops.random_cursor", "configure random cursor reads", C_BOOL, 10, 0, 0, &g.c_random_cursor, - NULL}, - - /* 100% */ - {"ops.salvage", "configure salvage", C_BOOL, 100, 1, 0, &g.c_salvage, NULL}, - - /* 100% */ - {"ops.truncate", "configure truncation", C_BOOL, 100, 0, 0, &g.c_truncate, NULL}, - - /* 100% */ - {"ops.verify", "configure verify", C_BOOL, 100, 1, 0, &g.c_verify, NULL}, - - {"quiet", "quiet run (same as -q)", C_IGNORE | C_BOOL, 0, 0, 1, &g.c_quiet, NULL}, - - {"runs", "number of runs", C_IGNORE, 0, 0, UINT_MAX, &g.c_runs, NULL}, - - {"runs.in_memory", "configure in-memory", C_IGNORE | C_BOOL, 0, 0, 1, &g.c_in_memory, NULL}, - - {"runs.ops", "operations per run", 0x0, 0, M(2), M(100), &g.c_ops, NULL}, - - {"runs.rows", "number of rows", 0x0, 10, M(1), M(100), &g.c_rows, NULL}, - - {"runs.source", "data source type (file | lsm | table)", C_IGNORE | C_STRING, 0, 0, 0, NULL, - &g.c_data_source}, - - {"runs.threads", "number of worker threads", 0x0, 1, 32, 128, &g.c_threads, NULL}, - - {"runs.timer", "run time (minutes)", C_IGNORE, 0, 0, UINT_MAX, &g.c_timer, NULL}, - - {"runs.type", "object type (fix | row | var)", C_IGNORE | C_STRING, 0, 0, 0, NULL, - &g.c_file_type}, - - {"runs.verify_failure_dump", "configure page dump on repeatable read error", C_IGNORE | C_BOOL, 0, - 0, 1, &g.c_verify_failure_dump, NULL}, - - /* 20% */ - {"statistics", "configure statistics", C_BOOL, 20, 0, 0, &g.c_statistics, NULL}, - - /* 5% */ - {"statistics.server", "configure statistics server thread", C_BOOL, 5, 0, 0, - &g.c_statistics_server, NULL}, - - /* 2% */ - {"stress.aggressive_sweep", "stress aggressive sweep", C_BOOL, 2, 0, 0, - &g.c_timing_stress_aggressive_sweep, NULL}, - - /* 2% */ - {"stress.checkpoint", "stress checkpoints", C_BOOL, 2, 0, 0, &g.c_timing_stress_checkpoint, NULL}, - - /* 2% */ - {"stress.checkpoint_reserved_txnid_delay", "stress checkpoint invisible transaction id delay", - C_BOOL, 2, 0, 0, &g.c_timing_stress_checkpoint_reserved_txnid_delay, NULL}, - - /* 2% */ - {"stress.checkpoint_prepare", "stress checkpoint prepare", C_BOOL, 2, 0, 0, - &g.c_timing_stress_checkpoint_prepare, NULL}, - - /* 30% */ - {"stress.failpoint_hs_delete_key_from_ts", "stress failpoint history store delete key from ts", - C_BOOL, 30, 0, 0, &g.c_timing_stress_failpoint_hs_delete_key_from_ts, NULL}, - - /* 30% */ - {"stress.failpoint_hs_insert_1", "stress failpoint history store insert (#1)", C_BOOL, 30, 0, 0, - &g.c_timing_stress_failpoint_hs_insert_1, NULL}, - - /* 30% */ - {"stress.failpoint_hs_insert_2", "stress failpoint history store insert (#2)", C_BOOL, 30, 0, 0, - &g.c_timing_stress_failpoint_hs_insert_2, NULL}, - - /* 2% */ - {"stress.hs_checkpoint_delay", "stress history store checkpoint delay", C_BOOL, 2, 0, 0, - &g.c_timing_stress_hs_checkpoint_delay, NULL}, - - /* 2% */ - {"stress.hs_search", "stress history store search", C_BOOL, 2, 0, 0, &g.c_timing_stress_hs_search, - NULL}, - - /* 2% */ - {"stress.hs_sweep", "stress history store sweep", C_BOOL, 2, 0, 0, &g.c_timing_stress_hs_sweep, - NULL}, - - /* 2% */ - {"stress.split_1", "stress splits (#1)", C_BOOL, 2, 0, 0, &g.c_timing_stress_split_1, NULL}, - - /* 2% */ - {"stress.split_2", "stress splits (#2)", C_BOOL, 2, 0, 0, &g.c_timing_stress_split_2, NULL}, - - /* 2% */ - {"stress.split_3", "stress splits (#3)", C_BOOL, 2, 0, 0, &g.c_timing_stress_split_3, NULL}, - - /* 2% */ - {"stress.split_4", "stress splits (#4)", C_BOOL, 2, 0, 0, &g.c_timing_stress_split_4, NULL}, - - /* 2% */ - {"stress.split_5", "stress splits (#5)", C_BOOL, 2, 0, 0, &g.c_timing_stress_split_5, NULL}, - - /* 2% */ - {"stress.split_6", "stress splits (#6)", C_BOOL, 2, 0, 0, &g.c_timing_stress_split_6, NULL}, - - /* 2% */ - {"stress.split_7", "stress splits (#7)", C_BOOL, 2, 0, 0, &g.c_timing_stress_split_7, NULL}, - - {"transaction.implicit", "implicit, without timestamps, transactions (percentage)", 0x0, 0, 100, - 100, &g.c_txn_implicit, NULL}, - - /* 70% */ - {"transaction.timestamps", "all transactions (or none), have timestamps", C_BOOL, 80, 0, 0, - &g.c_txn_timestamps, NULL}, - - {"wiredtiger.config", "wiredtiger_open API configuration string", C_IGNORE | C_STRING, 0, 0, 0, - NULL, &g.c_config_open}, - - /* 80% */ - {"wiredtiger.rwlock", "configure wiredtiger read/write mutexes", C_BOOL, 80, 0, 0, &g.c_wt_mutex, - NULL}, - - {"wiredtiger.leak_memory", "configure memory leaked on shutdown", C_BOOL, 0, 0, 0, - &g.c_leak_memory, NULL}, + u_int off; /* Value offset */ +} CONFIG; - {NULL, NULL, 0x0, 0, 0, 0, NULL, NULL}}; +#define V_MAX_TABLES_CONFIG 1000 + +#define V_GLOBAL_ASSERT_READ_TIMESTAMP 0 +#define V_GLOBAL_ASSERT_WRITE_TIMESTAMP 1 +#define V_GLOBAL_BACKUP 2 +#define V_GLOBAL_BACKUP_INCREMENTAL 3 +#define V_GLOBAL_BACKUP_INCR_GRANULARITY 4 +#define V_GLOBAL_BLOCK_CACHE 5 +#define V_GLOBAL_BLOCK_CACHE_CACHE_ON_CHECKPOINT 6 +#define V_GLOBAL_BLOCK_CACHE_CACHE_ON_WRITES 7 +#define V_GLOBAL_BLOCK_CACHE_SIZE 8 +#define V_TABLE_BTREE_BITCNT 9 +#define V_TABLE_BTREE_COMPRESSION 10 +#define V_TABLE_BTREE_DICTIONARY 11 +#define V_TABLE_BTREE_HUFFMAN_VALUE 12 +#define V_TABLE_BTREE_INTERNAL_KEY_TRUNCATION 13 +#define V_TABLE_BTREE_INTERNAL_PAGE_MAX 14 +#define V_TABLE_BTREE_KEY_MAX 15 +#define V_TABLE_BTREE_KEY_MIN 16 +#define V_TABLE_BTREE_LEAF_PAGE_MAX 17 +#define V_TABLE_BTREE_MEMORY_PAGE_MAX 18 +#define V_TABLE_BTREE_PREFIX_LEN 19 +#define V_TABLE_BTREE_PREFIX_COMPRESSION 20 +#define V_TABLE_BTREE_PREFIX_COMPRESSION_MIN 21 +#define V_TABLE_BTREE_REPEAT_DATA_PCT 22 +#define V_TABLE_BTREE_REVERSE 23 +#define V_TABLE_BTREE_SPLIT_PCT 24 +#define V_TABLE_BTREE_VALUE_MAX 25 +#define V_TABLE_BTREE_VALUE_MIN 26 +#define V_GLOBAL_CACHE 27 +#define V_GLOBAL_CACHE_EVICT_MAX 28 +#define V_GLOBAL_CACHE_MINIMUM 29 +#define V_GLOBAL_CHECKPOINT 30 +#define V_GLOBAL_CHECKPOINT_LOG_SIZE 31 +#define V_GLOBAL_CHECKPOINT_WAIT 32 +#define V_TABLE_DISK_CHECKSUM 33 +#define V_GLOBAL_DISK_DATA_EXTEND 34 +#define V_GLOBAL_DISK_DIRECT_IO 35 +#define V_GLOBAL_DISK_ENCRYPTION 36 +#define V_TABLE_DISK_FIRSTFIT 37 +#define V_GLOBAL_DISK_MMAP 38 +#define V_GLOBAL_DISK_MMAP_ALL 39 +#define V_GLOBAL_FORMAT_ABORT 40 +#define V_GLOBAL_FORMAT_INDEPENDENT_THREAD_RNG 41 +#define V_GLOBAL_FORMAT_MAJOR_TIMEOUT 42 +#define V_GLOBAL_IMPORT 43 +#define V_GLOBAL_LOGGING 44 +#define V_GLOBAL_LOGGING_ARCHIVE 45 +#define V_GLOBAL_LOGGING_COMPRESSION 46 +#define V_GLOBAL_LOGGING_FILE_MAX 47 +#define V_GLOBAL_LOGGING_PREALLOC 48 +#define V_TABLE_LSM_AUTO_THROTTLE 49 +#define V_TABLE_LSM_BLOOM 50 +#define V_TABLE_LSM_BLOOM_BIT_COUNT 51 +#define V_TABLE_LSM_BLOOM_HASH_COUNT 52 +#define V_TABLE_LSM_BLOOM_OLDEST 53 +#define V_TABLE_LSM_CHUNK_SIZE 54 +#define V_TABLE_LSM_MERGE_MAX 55 +#define V_GLOBAL_LSM_WORKER_THREADS 56 +#define V_GLOBAL_OPS_ALTER 57 +#define V_GLOBAL_OPS_COMPACTION 58 +#define V_GLOBAL_OPS_HS_CURSOR 59 +#define V_TABLE_OPS_PCT_DELETE 60 +#define V_TABLE_OPS_PCT_INSERT 61 +#define V_TABLE_OPS_PCT_MODIFY 62 +#define V_TABLE_OPS_PCT_READ 63 +#define V_TABLE_OPS_PCT_WRITE 64 +#define V_GLOBAL_OPS_PREPARE 65 +#define V_GLOBAL_OPS_RANDOM_CURSOR 66 +#define V_GLOBAL_OPS_SALVAGE 67 +#define V_TABLE_OPS_TRUNCATE 68 +#define V_GLOBAL_OPS_VERIFY 69 +#define V_GLOBAL_QUIET 70 +#define V_GLOBAL_RUNS_IN_MEMORY 71 +#define V_GLOBAL_RUNS_OPS 72 +#define V_TABLE_RUNS_ROWS 73 +#define V_TABLE_RUNS_SOURCE 74 +#define V_GLOBAL_RUNS_TABLES 75 +#define V_GLOBAL_RUNS_THREADS 76 +#define V_GLOBAL_RUNS_TIMER 77 +#define V_TABLE_RUNS_TYPE 78 +#define V_GLOBAL_RUNS_VERIFY_FAILURE_DUMP 79 +#define V_GLOBAL_STATISTICS 80 +#define V_GLOBAL_STATISTICS_SERVER 81 +#define V_GLOBAL_STRESS_AGGRESSIVE_SWEEP 82 +#define V_GLOBAL_STRESS_CHECKPOINT 83 +#define V_GLOBAL_STRESS_CHECKPOINT_RESERVED_TXNID_DELAY 84 +#define V_GLOBAL_STRESS_CHECKPOINT_PREPARE 85 +#define V_GLOBAL_STRESS_FAILPOINT_HS_DELETE_KEY_FROM_TS 86 +#define V_GLOBAL_STRESS_FAILPOINT_HS_INSERT_1 87 +#define V_GLOBAL_STRESS_FAILPOINT_HS_INSERT_2 88 +#define V_GLOBAL_STRESS_HS_CHECKPOINT_DELAY 89 +#define V_GLOBAL_STRESS_HS_SEARCH 90 +#define V_GLOBAL_STRESS_HS_SWEEP 91 +#define V_GLOBAL_STRESS_SPLIT_1 92 +#define V_GLOBAL_STRESS_SPLIT_2 93 +#define V_GLOBAL_STRESS_SPLIT_3 94 +#define V_GLOBAL_STRESS_SPLIT_4 95 +#define V_GLOBAL_STRESS_SPLIT_5 96 +#define V_GLOBAL_STRESS_SPLIT_6 97 +#define V_GLOBAL_STRESS_SPLIT_7 98 +#define V_GLOBAL_TRANSACTION_IMPLICIT 99 +#define V_GLOBAL_TRANSACTION_TIMESTAMPS 100 +#define V_GLOBAL_WIREDTIGER_CONFIG 101 +#define V_GLOBAL_WIREDTIGER_RWLOCK 102 +#define V_GLOBAL_WIREDTIGER_LEAK_MEMORY 103 + +#define V_ELEMENT_COUNT 104 diff --git a/src/third_party/wiredtiger/test/format/config.sh b/src/third_party/wiredtiger/test/format/config.sh new file mode 100644 index 00000000000..3dee4973714 --- /dev/null +++ b/src/third_party/wiredtiger/test/format/config.sh @@ -0,0 +1,296 @@ +#! /bin/sh + +# This script creates format's config.h and config_def.c files. To change format's configuration, +# modify this file and then run it as a script. + +fc="config_def.c" +fh="config.h" + +cat<<END_OF_HEADER_FILE_PREFIX>$fh +/* DO NOT EDIT: automatically built by format/config.sh. */ + +#define C_TYPE_MATCH(cp, type) \\ + (!F_ISSET(cp, (C_TYPE_FIX | C_TYPE_ROW | C_TYPE_VAR)) || \\ + ((type) == FIX && F_ISSET(cp, C_TYPE_FIX)) || ((type) == ROW && F_ISSET(cp, C_TYPE_ROW)) || \\ + ((type) == VAR && F_ISSET(cp, C_TYPE_VAR))) + +typedef struct { + const char *name; /* Configuration item */ + const char *desc; /* Configuration description */ + +#define C_BOOL 0x001u /* Boolean (true if roll of 1-to-100 is <= CONFIG->min) */ +#define C_IGNORE 0x002u /* Not a simple randomization, configured specially */ +#define C_STRING 0x004u /* String (rather than integral) */ +#define C_TABLE 0x008u /* Value is per table, not global */ +#define C_TYPE_FIX 0x010u /* Value is only relevant to FLCS */ +#define C_TYPE_LSM 0x020u /* Value is only relevant to LSM */ +#define C_TYPE_ROW 0x040u /* Value is only relevant to RS */ +#define C_TYPE_VAR 0x080u /* Value is only relevant to VLCS */ +#define C_ZERO_NOTSET 0x100u /* Ignore zero values */ + uint32_t flags; + + uint32_t min; /* Minimum value */ + uint32_t maxrand; /* Maximum value randomly chosen */ + uint32_t maxset; /* Maximum value explicitly set */ + + u_int off; /* Value offset */ +} CONFIG; + +#define V_MAX_TABLES_CONFIG 1000 + +END_OF_HEADER_FILE_PREFIX + +n=0 +while IFS= read -r line; do + case "$line" in + '{"'*) + tag=`echo "$line" | + sed -e 's/{"//' \ + -e 's/",.*//' \ + -e 's/\./_/g' | + tr '[:lower:]' '[:upper:]'` + prefix="GLOBAL" + if `echo "$line" | grep 'C_TABLE' > /dev/null`; then + prefix="TABLE" + fi + def="V_""$prefix""_""$tag" + echo "$line" | + sed -e "s/}/, $def},/" \ + -e 's/\(^.*",\) \(.*\)/ \1\n \2/' + + echo "#define $def $n" >> $fh + + n=`expr $n + 1` + ;; + *) + echo "$line" + esac +done<<END_OF_INPUT>$fc +/* DO NOT EDIT: automatically built by format/config.sh. */ + +#include "format.h" + +CONFIG configuration_list[] = { +{"assert.read_timestamp", "assert read_timestamp", C_BOOL, 2, 0, 0} + +{"assert.write_timestamp", "set write_timestamp_usage and assert write_timestamp", C_BOOL, 2, 0, 0} + +{"backup", "configure backups", C_BOOL, 20, 0, 0} + +{"backup.incremental", "backup type (off | block | log)", C_IGNORE | C_STRING, 0, 0, 0} + +{"backup.incr_granularity", "incremental backup block granularity (KB)", 0x0, 4, 16384, 16384} + +{"block_cache", "enable the block cache", C_BOOL, 10, 0, 0} + +{"block_cache.cache_on_checkpoint", "block cache: cache checkpoint writes", C_BOOL, 30, 0, 0} + +{"block_cache.cache_on_writes", "block cache: populate the cache on writes", C_BOOL, 60, 0, 0} + +{"block_cache.size", "block cache size (MB)", 0x0, 1, 100, 100 * 1024} + +{"btree.bitcnt", "fixed-length column-store object size (number of bits)", C_TABLE | C_TYPE_FIX, 1, 8, 8} + +{"btree.compression", "data compression (off | lz4 | snappy | zlib | zstd)", C_IGNORE | C_STRING | C_TABLE, 0, 0, 0} + +{"btree.dictionary", "configure dictionary compressed values", C_BOOL | C_TABLE | C_TYPE_ROW | C_TYPE_VAR, 20, 0, 0} + +{"btree.huffman_value", "configure huffman encoded values", C_BOOL | C_TABLE | C_TYPE_ROW | C_TYPE_VAR, 20, 0, 0} + +{"btree.internal_key_truncation", "truncate internal keys", C_BOOL | C_TABLE, 95, 0, 0} + +{"btree.internal_page_max", "btree internal node maximum size", C_TABLE, 9, 17, 27} + +{"btree.key_max", "maximum key size", C_TABLE | C_TYPE_ROW, 20, 128, MEGABYTE(10)} + +{"btree.key_min", "minimum key size", C_TABLE | C_TYPE_ROW, KEY_LEN_CONFIG_MIN, 32, 256} + +{"btree.leaf_page_max", "btree leaf node maximum size", C_TABLE, 9, 17, 27} + +{"btree.memory_page_max", "maximum cache page size", C_TABLE, 1, 10, 128} + +{"btree.prefix_len", "common key prefix", C_TABLE | C_TYPE_ROW | C_ZERO_NOTSET, PREFIX_LEN_CONFIG_MIN, PREFIX_LEN_CONFIG_MAX, PREFIX_LEN_CONFIG_MAX} + +{"btree.prefix_compression", "configure prefix compressed keys", C_BOOL | C_TABLE | C_TYPE_ROW, 80, 0, 0} + +{"btree.prefix_compression_min", "minimum gain before prefix compression is used (bytes)", C_TABLE | C_TYPE_ROW, 0, 8, 256} + +{"btree.repeat_data_pct", "duplicate values (percentage)", C_TABLE | C_TYPE_VAR, 0, 90, 90} + +{"btree.reverse", "reverse order collation", C_BOOL | C_TABLE | C_TYPE_ROW, 10, 0, 0} + +{"btree.split_pct", "page split size as a percentage of the maximum page size", C_TABLE, 50, 100, 100} + +{"btree.value_max", "maximum value size", C_TABLE | C_TYPE_ROW | C_TYPE_VAR, 32, 4096, MEGABYTE(10)} + +{"btree.value_min", "minimum value size", C_TABLE | C_TYPE_ROW | C_TYPE_VAR, 0, 20, 4096} + +{"cache", "cache size (MB)", 0x0, 1, 100, 100 * 1024} + +{"cache.evict_max", "maximum number of eviction workers", 0x0, 0, 5, 100} + +{"cache.minimum", "minimum cache size (MB)", C_IGNORE, 0, 0, 100 * 1024} + +{"checkpoint", "checkpoint type (on | off | wiredtiger)", C_IGNORE | C_STRING, 0, 0, 0} + +{"checkpoint.log_size", "MB of log to wait if wiredtiger checkpoints configured", 0x0, 20, 200, 1024} + +{"checkpoint.wait", "seconds to wait if wiredtiger checkpoints configured", 0x0, 5, 100, 3600} + +{"disk.checksum", "checksum type (on | off | uncompressed | unencrypted)", C_IGNORE | C_STRING | C_TABLE, 0, 0, 0} + +{"disk.data_extend", "configure data file extension", C_BOOL, 5, 0, 0} + +{"disk.direct_io", "configure direct I/O for data objects", C_BOOL | C_IGNORE, 0, 0, 1} + +{"disk.encryption", "encryption type (off | rotn-7)", C_IGNORE | C_STRING, 0, 0, 0} + +{"disk.firstfit", "configure first-fit allocation", C_BOOL | C_TABLE, 10, 0, 0} + +{"disk.mmap", "configure mmap operations (reads only)", C_BOOL, 90, 0, 0} + +{"disk.mmap_all", "configure mmap operations (read and write)", C_BOOL, 5, 0, 0} + +{"format.abort", "drop core during timed run", C_BOOL, 0, 0, 0} + +{"format.independent_thread_rng", "configure independent thread RNG space", C_BOOL, 75, 0, 0} + +{"format.major_timeout", "long-running operations timeout (minutes)", C_IGNORE, 0, 0, 1000} + +/* + * 0% + * FIXME-WT-7510: Temporarily disable import until WT_ROLLBACK error and wt_copy_and_sync error is + * fixed. It should be (C_BOOL, 20, 0, 0). + */ +{"import", "import table from newly created database", C_BOOL, 0, 0, 0} + +{"logging", "configure logging", C_BOOL, 50, 0, 0} + +{"logging.archive", "configure log file archival", C_BOOL, 50, 0, 0} + +{"logging.compression", "logging compression (off | lz4 | snappy | zlib | zstd)", C_IGNORE | C_STRING, 0, 0, 0} + +{"logging.file_max", "maximum log file size (KB)", 0x0, 100, 512000, 2097152} + +{"logging.prealloc", "configure log file pre-allocation", C_BOOL, 50, 0, 0} + +{"lsm.auto_throttle", "throttle LSM inserts", C_BOOL | C_TABLE | C_TYPE_LSM, 90, 0, 0} + +{"lsm.bloom", "configure bloom filters", C_BOOL | C_TABLE | C_TYPE_LSM, 95, 0, 0} + +{"lsm.bloom_bit_count", "number of bits per item for bloom filters", C_TABLE | C_TYPE_LSM, 4, 64, 1000} + +{"lsm.bloom_hash_count", "number of hash values per item for bloom filters", C_TABLE | C_TYPE_LSM, 4, 32, 100} + +{"lsm.bloom_oldest", "configure bloom_oldest=true", C_BOOL | C_TABLE | C_TYPE_LSM, 10, 0, 0} + +{"lsm.chunk_size", "LSM chunk size (MB)", C_TABLE | C_TYPE_LSM, 1, 10, 100} + +{"lsm.merge_max", "maximum number of chunks to include in an LSM merge operation", C_TABLE | C_TYPE_LSM, 4, 20, 100} + +{"lsm.worker_threads", "number of LSM worker threads", C_TYPE_LSM, 3, 4, 20} + +{"ops.alter", "configure table alterations", C_BOOL, 10, 0, 0} + +{"ops.compaction", "configure compaction", C_BOOL, 10, 0, 0} + +{"ops.hs_cursor", "configure history store cursor reads", C_BOOL, 50, 0, 0} + +{"ops.pct.delete", "delete operations (percentage)", C_IGNORE | C_TABLE, 0, 0, 100} + +{"ops.pct.insert", "insert operations (percentage)", C_IGNORE | C_TABLE, 0, 0, 100} + +{"ops.pct.modify", "modify operations (percentage)", C_IGNORE | C_TABLE, 0, 0, 100} + +{"ops.pct.read", "read operations (percentage)", C_IGNORE | C_TABLE, 0, 0, 100} + +{"ops.pct.write", "update operations (percentage)", C_IGNORE | C_TABLE, 0, 0, 100} + +{"ops.prepare", "configure transaction prepare", C_BOOL, 5, 0, 0} + +{"ops.random_cursor", "configure random cursor reads", C_BOOL, 10, 0, 0} + +{"ops.salvage", "configure salvage", C_BOOL, 100, 1, 0} + +{"ops.truncate", "configure truncation", C_BOOL | C_TABLE, 100, 0, 0} + +{"ops.verify", "configure verify", C_BOOL, 100, 1, 0} + +{"quiet", "quiet run (same as -q)", C_BOOL | C_IGNORE, 0, 0, 1} + +{"runs.in_memory", "configure in-memory", C_BOOL | C_IGNORE, 0, 0, 1} + +{"runs.ops", "operations per run", 0x0, 0, M(2), M(100)} + +{"runs.rows", "number of rows", C_TABLE, 10, M(1), M(100)} + +{"runs.source", "data source type (file | lsm | table)", C_IGNORE | C_STRING | C_TABLE, 0, 0, 0} + +{"runs.tables", "number of tables", 0x0, 1, 32, V_MAX_TABLES_CONFIG} + +{"runs.threads", "number of worker threads", 0x0, 1, 32, 128} + +{"runs.timer", "run time (minutes)", C_IGNORE, 0, 0, UINT_MAX} + +{"runs.type", "object type (fix | row | var)", C_IGNORE | C_STRING | C_TABLE, 0, 0, 0} + +{"runs.verify_failure_dump", "configure page dump on repeatable read error", C_BOOL | C_IGNORE, 0, 0, 1} + +{"statistics", "configure statistics", C_BOOL, 20, 0, 0} + +{"statistics.server", "configure statistics server thread", C_BOOL, 5, 0, 0} + +{"stress.aggressive_sweep", "stress aggressive sweep", C_BOOL, 2, 0, 0} + +{"stress.checkpoint", "stress checkpoints", C_BOOL, 2, 0, 0} + +{"stress.checkpoint_reserved_txnid_delay", "stress checkpoint invisible transaction id delay", C_BOOL, 2, 0, 0} + +{"stress.checkpoint_prepare", "stress checkpoint prepare", C_BOOL, 2, 0, 0} + +{"stress.failpoint_hs_delete_key_from_ts", "stress failpoint history store delete key from ts", C_BOOL, 30, 0, 0} + +{"stress.failpoint_hs_insert_1", "stress failpoint history store insert (#1)", C_BOOL, 30, 0, 0} + +{"stress.failpoint_hs_insert_2", "stress failpoint history store insert (#2)", C_BOOL, 30, 0, 0} + +{"stress.hs_checkpoint_delay", "stress history store checkpoint delay", C_BOOL, 2, 0, 0} + +{"stress.hs_search", "stress history store search", C_BOOL, 2, 0, 0} + +{"stress.hs_sweep", "stress history store sweep", C_BOOL, 2, 0, 0} + +{"stress.split_1", "stress splits (#1)", C_BOOL, 2, 0, 0} + +{"stress.split_2", "stress splits (#2)", C_BOOL, 2, 0, 0} + +{"stress.split_3", "stress splits (#3)", C_BOOL, 2, 0, 0} + +{"stress.split_4", "stress splits (#4)", C_BOOL, 2, 0, 0} + +{"stress.split_5", "stress splits (#5)", C_BOOL, 2, 0, 0} + +{"stress.split_6", "stress splits (#6)", C_BOOL, 2, 0, 0} + +{"stress.split_7", "stress splits (#7)", C_BOOL, 2, 0, 0} + +{"transaction.implicit", "implicit, without timestamps, transactions (percentage)", 0, 0, 100, 100} + +{"transaction.timestamps", "all transactions (or none), have timestamps", C_BOOL, 80, 0, 0} + +{"wiredtiger.config", "wiredtiger_open API configuration string", C_IGNORE | C_STRING, 0, 0, 0} + +{"wiredtiger.rwlock", "configure wiredtiger read/write mutexes", C_BOOL, 80, 0, 0} + +{"wiredtiger.leak_memory", "leak memory on wiredtiger shutdown", C_BOOL, 0, 0, 0} +END_OF_INPUT + +( +echo +echo ' {NULL, NULL, 0x0, 0, 0, 0, 0}' +echo '};') >> $fc + +( +echo +echo "#define V_ELEMENT_COUNT $n") >> $fh + diff --git a/src/third_party/wiredtiger/test/format/config_def.c b/src/third_party/wiredtiger/test/format/config_def.c new file mode 100644 index 00000000000..91aa3970823 --- /dev/null +++ b/src/third_party/wiredtiger/test/format/config_def.c @@ -0,0 +1,324 @@ +/* DO NOT EDIT: automatically built by format/config.sh. */ + +#include "format.h" + +CONFIG configuration_list[] = { + {"assert.read_timestamp", "assert read_timestamp", + C_BOOL, 2, 0, 0, V_GLOBAL_ASSERT_READ_TIMESTAMP}, + + {"assert.write_timestamp", "set write_timestamp_usage and assert write_timestamp", + C_BOOL, 2, 0, 0, V_GLOBAL_ASSERT_WRITE_TIMESTAMP}, + + {"backup", "configure backups", + C_BOOL, 20, 0, 0, V_GLOBAL_BACKUP}, + + {"backup.incremental", "backup type (off | block | log)", + C_IGNORE | C_STRING, 0, 0, 0, V_GLOBAL_BACKUP_INCREMENTAL}, + + {"backup.incr_granularity", "incremental backup block granularity (KB)", + 0x0, 4, 16384, 16384, V_GLOBAL_BACKUP_INCR_GRANULARITY}, + + {"block_cache", "enable the block cache", + C_BOOL, 10, 0, 0, V_GLOBAL_BLOCK_CACHE}, + + {"block_cache.cache_on_checkpoint", "block cache: cache checkpoint writes", + C_BOOL, 30, 0, 0, V_GLOBAL_BLOCK_CACHE_CACHE_ON_CHECKPOINT}, + + {"block_cache.cache_on_writes", "block cache: populate the cache on writes", + C_BOOL, 60, 0, 0, V_GLOBAL_BLOCK_CACHE_CACHE_ON_WRITES}, + + {"block_cache.size", "block cache size (MB)", + 0x0, 1, 100, 100 * 1024, V_GLOBAL_BLOCK_CACHE_SIZE}, + + {"btree.bitcnt", "fixed-length column-store object size (number of bits)", + C_TABLE | C_TYPE_FIX, 1, 8, 8, V_TABLE_BTREE_BITCNT}, + + {"btree.compression", "data compression (off | lz4 | snappy | zlib | zstd)", + C_IGNORE | C_STRING | C_TABLE, 0, 0, 0, V_TABLE_BTREE_COMPRESSION}, + + {"btree.dictionary", "configure dictionary compressed values", + C_BOOL | C_TABLE | C_TYPE_ROW | C_TYPE_VAR, 20, 0, 0, V_TABLE_BTREE_DICTIONARY}, + + {"btree.huffman_value", "configure huffman encoded values", + C_BOOL | C_TABLE | C_TYPE_ROW | C_TYPE_VAR, 20, 0, 0, V_TABLE_BTREE_HUFFMAN_VALUE}, + + {"btree.internal_key_truncation", "truncate internal keys", + C_BOOL | C_TABLE, 95, 0, 0, V_TABLE_BTREE_INTERNAL_KEY_TRUNCATION}, + + {"btree.internal_page_max", "btree internal node maximum size", + C_TABLE, 9, 17, 27, V_TABLE_BTREE_INTERNAL_PAGE_MAX}, + + {"btree.key_max", "maximum key size", + C_TABLE | C_TYPE_ROW, 20, 128, MEGABYTE(10), V_TABLE_BTREE_KEY_MAX}, + + {"btree.key_min", "minimum key size", + C_TABLE | C_TYPE_ROW, KEY_LEN_CONFIG_MIN, 32, 256, V_TABLE_BTREE_KEY_MIN}, + + {"btree.leaf_page_max", "btree leaf node maximum size", + C_TABLE, 9, 17, 27, V_TABLE_BTREE_LEAF_PAGE_MAX}, + + {"btree.memory_page_max", "maximum cache page size", + C_TABLE, 1, 10, 128, V_TABLE_BTREE_MEMORY_PAGE_MAX}, + + {"btree.prefix_len", "common key prefix", + C_TABLE | C_TYPE_ROW | C_ZERO_NOTSET, PREFIX_LEN_CONFIG_MIN, PREFIX_LEN_CONFIG_MAX, PREFIX_LEN_CONFIG_MAX, V_TABLE_BTREE_PREFIX_LEN}, + + {"btree.prefix_compression", "configure prefix compressed keys", + C_BOOL | C_TABLE | C_TYPE_ROW, 80, 0, 0, V_TABLE_BTREE_PREFIX_COMPRESSION}, + + {"btree.prefix_compression_min", "minimum gain before prefix compression is used (bytes)", + C_TABLE | C_TYPE_ROW, 0, 8, 256, V_TABLE_BTREE_PREFIX_COMPRESSION_MIN}, + + {"btree.repeat_data_pct", "duplicate values (percentage)", + C_TABLE | C_TYPE_VAR, 0, 90, 90, V_TABLE_BTREE_REPEAT_DATA_PCT}, + + {"btree.reverse", "reverse order collation", + C_BOOL | C_TABLE | C_TYPE_ROW, 10, 0, 0, V_TABLE_BTREE_REVERSE}, + + {"btree.split_pct", "page split size as a percentage of the maximum page size", + C_TABLE, 50, 100, 100, V_TABLE_BTREE_SPLIT_PCT}, + + {"btree.value_max", "maximum value size", + C_TABLE | C_TYPE_ROW | C_TYPE_VAR, 32, 4096, MEGABYTE(10), V_TABLE_BTREE_VALUE_MAX}, + + {"btree.value_min", "minimum value size", + C_TABLE | C_TYPE_ROW | C_TYPE_VAR, 0, 20, 4096, V_TABLE_BTREE_VALUE_MIN}, + + {"cache", "cache size (MB)", + 0x0, 1, 100, 100 * 1024, V_GLOBAL_CACHE}, + + {"cache.evict_max", "maximum number of eviction workers", + 0x0, 0, 5, 100, V_GLOBAL_CACHE_EVICT_MAX}, + + {"cache.minimum", "minimum cache size (MB)", + C_IGNORE, 0, 0, 100 * 1024, V_GLOBAL_CACHE_MINIMUM}, + + {"checkpoint", "checkpoint type (on | off | wiredtiger)", + C_IGNORE | C_STRING, 0, 0, 0, V_GLOBAL_CHECKPOINT}, + + {"checkpoint.log_size", "MB of log to wait if wiredtiger checkpoints configured", + 0x0, 20, 200, 1024, V_GLOBAL_CHECKPOINT_LOG_SIZE}, + + {"checkpoint.wait", "seconds to wait if wiredtiger checkpoints configured", + 0x0, 5, 100, 3600, V_GLOBAL_CHECKPOINT_WAIT}, + + {"disk.checksum", "checksum type (on | off | uncompressed | unencrypted)", + C_IGNORE | C_STRING | C_TABLE, 0, 0, 0, V_TABLE_DISK_CHECKSUM}, + + {"disk.data_extend", "configure data file extension", + C_BOOL, 5, 0, 0, V_GLOBAL_DISK_DATA_EXTEND}, + + {"disk.direct_io", "configure direct I/O for data objects", + C_BOOL | C_IGNORE, 0, 0, 1, V_GLOBAL_DISK_DIRECT_IO}, + + {"disk.encryption", "encryption type (off | rotn-7)", + C_IGNORE | C_STRING, 0, 0, 0, V_GLOBAL_DISK_ENCRYPTION}, + + {"disk.firstfit", "configure first-fit allocation", + C_BOOL | C_TABLE, 10, 0, 0, V_TABLE_DISK_FIRSTFIT}, + + {"disk.mmap", "configure mmap operations (reads only)", + C_BOOL, 90, 0, 0, V_GLOBAL_DISK_MMAP}, + + {"disk.mmap_all", "configure mmap operations (read and write)", + C_BOOL, 5, 0, 0, V_GLOBAL_DISK_MMAP_ALL}, + + {"format.abort", "drop core during timed run", + C_BOOL, 0, 0, 0, V_GLOBAL_FORMAT_ABORT}, + + {"format.independent_thread_rng", "configure independent thread RNG space", + C_BOOL, 75, 0, 0, V_GLOBAL_FORMAT_INDEPENDENT_THREAD_RNG}, + + {"format.major_timeout", "long-running operations timeout (minutes)", + C_IGNORE, 0, 0, 1000, V_GLOBAL_FORMAT_MAJOR_TIMEOUT}, + +/* + * 0% + * FIXME-WT-7510: Temporarily disable import until WT_ROLLBACK error and wt_copy_and_sync error is + * fixed. It should be (C_BOOL, 20, 0, 0). + */ + {"import", "import table from newly created database", + C_BOOL, 0, 0, 0, V_GLOBAL_IMPORT}, + + {"logging", "configure logging", + C_BOOL, 50, 0, 0, V_GLOBAL_LOGGING}, + + {"logging.archive", "configure log file archival", + C_BOOL, 50, 0, 0, V_GLOBAL_LOGGING_ARCHIVE}, + + {"logging.compression", "logging compression (off | lz4 | snappy | zlib | zstd)", + C_IGNORE | C_STRING, 0, 0, 0, V_GLOBAL_LOGGING_COMPRESSION}, + + {"logging.file_max", "maximum log file size (KB)", + 0x0, 100, 512000, 2097152, V_GLOBAL_LOGGING_FILE_MAX}, + + {"logging.prealloc", "configure log file pre-allocation", + C_BOOL, 50, 0, 0, V_GLOBAL_LOGGING_PREALLOC}, + + {"lsm.auto_throttle", "throttle LSM inserts", + C_BOOL | C_TABLE | C_TYPE_LSM, 90, 0, 0, V_TABLE_LSM_AUTO_THROTTLE}, + + {"lsm.bloom", "configure bloom filters", + C_BOOL | C_TABLE | C_TYPE_LSM, 95, 0, 0, V_TABLE_LSM_BLOOM}, + + {"lsm.bloom_bit_count", "number of bits per item for bloom filters", + C_TABLE | C_TYPE_LSM, 4, 64, 1000, V_TABLE_LSM_BLOOM_BIT_COUNT}, + + {"lsm.bloom_hash_count", "number of hash values per item for bloom filters", + C_TABLE | C_TYPE_LSM, 4, 32, 100, V_TABLE_LSM_BLOOM_HASH_COUNT}, + + {"lsm.bloom_oldest", "configure bloom_oldest=true", + C_BOOL | C_TABLE | C_TYPE_LSM, 10, 0, 0, V_TABLE_LSM_BLOOM_OLDEST}, + + {"lsm.chunk_size", "LSM chunk size (MB)", + C_TABLE | C_TYPE_LSM, 1, 10, 100, V_TABLE_LSM_CHUNK_SIZE}, + + {"lsm.merge_max", "maximum number of chunks to include in an LSM merge operation", + C_TABLE | C_TYPE_LSM, 4, 20, 100, V_TABLE_LSM_MERGE_MAX}, + + {"lsm.worker_threads", "number of LSM worker threads", + C_TYPE_LSM, 3, 4, 20, V_GLOBAL_LSM_WORKER_THREADS}, + + {"ops.alter", "configure table alterations", + C_BOOL, 10, 0, 0, V_GLOBAL_OPS_ALTER}, + + {"ops.compaction", "configure compaction", + C_BOOL, 10, 0, 0, V_GLOBAL_OPS_COMPACTION}, + + {"ops.hs_cursor", "configure history store cursor reads", + C_BOOL, 50, 0, 0, V_GLOBAL_OPS_HS_CURSOR}, + + {"ops.pct.delete", "delete operations (percentage)", + C_IGNORE | C_TABLE, 0, 0, 100, V_TABLE_OPS_PCT_DELETE}, + + {"ops.pct.insert", "insert operations (percentage)", + C_IGNORE | C_TABLE, 0, 0, 100, V_TABLE_OPS_PCT_INSERT}, + + {"ops.pct.modify", "modify operations (percentage)", + C_IGNORE | C_TABLE, 0, 0, 100, V_TABLE_OPS_PCT_MODIFY}, + + {"ops.pct.read", "read operations (percentage)", + C_IGNORE | C_TABLE, 0, 0, 100, V_TABLE_OPS_PCT_READ}, + + {"ops.pct.write", "update operations (percentage)", + C_IGNORE | C_TABLE, 0, 0, 100, V_TABLE_OPS_PCT_WRITE}, + + {"ops.prepare", "configure transaction prepare", + C_BOOL, 5, 0, 0, V_GLOBAL_OPS_PREPARE}, + + {"ops.random_cursor", "configure random cursor reads", + C_BOOL, 10, 0, 0, V_GLOBAL_OPS_RANDOM_CURSOR}, + + {"ops.salvage", "configure salvage", + C_BOOL, 100, 1, 0, V_GLOBAL_OPS_SALVAGE}, + + {"ops.truncate", "configure truncation", + C_BOOL | C_TABLE, 100, 0, 0, V_TABLE_OPS_TRUNCATE}, + + {"ops.verify", "configure verify", + C_BOOL, 100, 1, 0, V_GLOBAL_OPS_VERIFY}, + + {"quiet", "quiet run (same as -q)", + C_BOOL | C_IGNORE, 0, 0, 1, V_GLOBAL_QUIET}, + + {"runs.in_memory", "configure in-memory", + C_BOOL | C_IGNORE, 0, 0, 1, V_GLOBAL_RUNS_IN_MEMORY}, + + {"runs.ops", "operations per run", + 0x0, 0, M(2), M(100), V_GLOBAL_RUNS_OPS}, + + {"runs.rows", "number of rows", + C_TABLE, 10, M(1), M(100), V_TABLE_RUNS_ROWS}, + + {"runs.source", "data source type (file | lsm | table)", + C_IGNORE | C_STRING | C_TABLE, 0, 0, 0, V_TABLE_RUNS_SOURCE}, + + {"runs.tables", "number of tables", + 0x0, 1, 32, V_MAX_TABLES_CONFIG, V_GLOBAL_RUNS_TABLES}, + + {"runs.threads", "number of worker threads", + 0x0, 1, 32, 128, V_GLOBAL_RUNS_THREADS}, + + {"runs.timer", "run time (minutes)", + C_IGNORE, 0, 0, UINT_MAX, V_GLOBAL_RUNS_TIMER}, + + {"runs.type", "object type (fix | row | var)", + C_IGNORE | C_STRING | C_TABLE, 0, 0, 0, V_TABLE_RUNS_TYPE}, + + {"runs.verify_failure_dump", "configure page dump on repeatable read error", + C_BOOL | C_IGNORE, 0, 0, 1, V_GLOBAL_RUNS_VERIFY_FAILURE_DUMP}, + + {"statistics", "configure statistics", + C_BOOL, 20, 0, 0, V_GLOBAL_STATISTICS}, + + {"statistics.server", "configure statistics server thread", + C_BOOL, 5, 0, 0, V_GLOBAL_STATISTICS_SERVER}, + + {"stress.aggressive_sweep", "stress aggressive sweep", + C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_AGGRESSIVE_SWEEP}, + + {"stress.checkpoint", "stress checkpoints", + C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_CHECKPOINT}, + + {"stress.checkpoint_reserved_txnid_delay", "stress checkpoint invisible transaction id delay", + C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_CHECKPOINT_RESERVED_TXNID_DELAY}, + + {"stress.checkpoint_prepare", "stress checkpoint prepare", + C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_CHECKPOINT_PREPARE}, + + {"stress.failpoint_hs_delete_key_from_ts", "stress failpoint history store delete key from ts", + C_BOOL, 30, 0, 0, V_GLOBAL_STRESS_FAILPOINT_HS_DELETE_KEY_FROM_TS}, + + {"stress.failpoint_hs_insert_1", "stress failpoint history store insert (#1)", + C_BOOL, 30, 0, 0, V_GLOBAL_STRESS_FAILPOINT_HS_INSERT_1}, + + {"stress.failpoint_hs_insert_2", "stress failpoint history store insert (#2)", + C_BOOL, 30, 0, 0, V_GLOBAL_STRESS_FAILPOINT_HS_INSERT_2}, + + {"stress.hs_checkpoint_delay", "stress history store checkpoint delay", + C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_HS_CHECKPOINT_DELAY}, + + {"stress.hs_search", "stress history store search", + C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_HS_SEARCH}, + + {"stress.hs_sweep", "stress history store sweep", + C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_HS_SWEEP}, + + {"stress.split_1", "stress splits (#1)", + C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_SPLIT_1}, + + {"stress.split_2", "stress splits (#2)", + C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_SPLIT_2}, + + {"stress.split_3", "stress splits (#3)", + C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_SPLIT_3}, + + {"stress.split_4", "stress splits (#4)", + C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_SPLIT_4}, + + {"stress.split_5", "stress splits (#5)", + C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_SPLIT_5}, + + {"stress.split_6", "stress splits (#6)", + C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_SPLIT_6}, + + {"stress.split_7", "stress splits (#7)", + C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_SPLIT_7}, + + {"transaction.implicit", "implicit, without timestamps, transactions (percentage)", + 0, 0, 100, 100, V_GLOBAL_TRANSACTION_IMPLICIT}, + + {"transaction.timestamps", "all transactions (or none), have timestamps", + C_BOOL, 80, 0, 0, V_GLOBAL_TRANSACTION_TIMESTAMPS}, + + {"wiredtiger.config", "wiredtiger_open API configuration string", + C_IGNORE | C_STRING, 0, 0, 0, V_GLOBAL_WIREDTIGER_CONFIG}, + + {"wiredtiger.rwlock", "configure wiredtiger read/write mutexes", + C_BOOL, 80, 0, 0, V_GLOBAL_WIREDTIGER_RWLOCK}, + + {"wiredtiger.leak_memory", "leak memory on wiredtiger shutdown", + C_BOOL, 0, 0, 0, V_GLOBAL_WIREDTIGER_LEAK_MEMORY}, + + {NULL, NULL, 0x0, 0, 0, 0, 0} +}; diff --git a/src/third_party/wiredtiger/test/format/format.h b/src/third_party/wiredtiger/test/format/format.h index cce28406706..e6bdfc17e43 100644 --- a/src/third_party/wiredtiger/test/format/format.h +++ b/src/third_party/wiredtiger/test/format/format.h @@ -80,12 +80,12 @@ #undef MEGABYTE #define MEGABYTE(v) ((v)*WT_MEGABYTE) +/* Format isn't careful about path buffers, an easy to fix hard-coded length. */ +#define MAX_FORMAT_PATH 1024 + #define BACKUP_INFO_FILE "BACKUP_INFO" /* Format's backup information for restart */ #define BACKUP_INFO_FILE_TMP "BACKUP_INFO.TMP" /* Format's backup information for restart */ #define BACKUP_MAX_COPY MEGABYTE(64) /* Maximum size we'll read/write at a time */ -#define WT_NAME "wt" /* Object name */ - -#define DATASOURCE(v) (strcmp(v, g.c_data_source) == 0 ? 1 : 0) #define FORMAT_OPERATION_REPS 3 /* 3 thread operations sets */ @@ -105,45 +105,111 @@ typedef struct { #define LOCK_INITIALIZED(lock) ((lock)->lock_type != LOCK_NONE) +#include "config.h" +extern CONFIG configuration_list[]; + typedef struct { - char tidbuf[128]; /* thread ID in printable form */ + uint32_t v; /* integral value */ + char *vstr; /* string value */ + bool set; /* value explicitly set */ +} CONFIGV; + +typedef enum { FIX, ROW, VAR } table_type; +typedef struct { + u_int id; /* table ID */ + char uri[32]; /* table URI */ + table_type type; /* table type */ + char track_prefix[32]; /* table track message prefix */ + + uint32_t max_intl_page; /* page size configurations converted to bytes */ + uint32_t max_leaf_page; + uint32_t max_mem_page; + + uint32_t rows_current; /* current row count */ + + uint64_t truncate_cnt; /* truncation operation counter */ + + uint32_t key_rand_len[1031]; /* key: lengths */ + char *val_base; /* value: base/original */ + uint32_t val_dup_data_len; /* value: length of duplicate data items */ + + CONFIGV v[V_ELEMENT_COUNT]; /* table configuration */ +} TABLE; + +/* + * We read the configuration in a single pass, which means we don't know the table count until the + * end, and it can be extended at any time. Start out with a single table, which contains all of the + * global/default values, stored in the first slot of the tables array. If tables are added during + * configuration, they are separately allocated, but we continue to use the first (base) table slot + * for non-specific table or global configurations. In other words, the base information and the + * only table's information are both in tables' slot 0 to start. If additional tables are + * configured, the per-table information for each table is stored in tables slots 1-N. The number of + * tables starts at 0, and if any tables are configured, it's incremented: in other words, if the + * number of tables is 0, all of the information is in tables' slot 0. If the number of tables is + * greater than 1, all of the base information is in tables slot 0, and tables slot 1 holds table + * #1's specific information, slot #2 holds table #2's specific information and so on. This allows + * general and table-specific information to be configured in any order, and as part of the final + * table configuration, if there's more than a single table, the information in tables' slot 0 is + * propagated out to the additional table slots. + */ +extern TABLE *tables[V_MAX_TABLES_CONFIG + 1]; /* Table array */ +extern u_int ntables; + +/* + * Global and table-specific macros to retrieve configuration information. All of the tables contain + * all of the possible configuration entries, but the first table slot contains all of the global + * configuration information. The offset names a prefixed with "V_GLOBAL" and "V_TABLE" to reduce + * the chance of a coding error retrieving the wrong configuration item. If returning string values, + * convert NULL, where a configuration has never been set, to "off" for consistency. + */ +#define GV(off) (tables[0]->v[V_GLOBAL_##off].v) +#define GVS(off) \ + (tables[0]->v[V_GLOBAL_##off].vstr == NULL ? "off" : tables[0]->v[V_GLOBAL_##off].vstr) +#define TV(off) (table->v[V_TABLE_##off].v) +#define TVS(off) (table->v[V_TABLE_##off].vstr == NULL ? "off" : table->v[V_TABLE_##off].vstr) + +#define DATASOURCE(table, ds) (strcmp((table)->v[V_TABLE_RUNS_SOURCE].vstr, ds) == 0) +typedef struct { WT_CONNECTION *wts_conn; WT_CONNECTION *wts_conn_inmemory; WT_SESSION *wts_session; - char *uri; /* Object name */ - bool backward_compatible; /* Backward compatibility testing */ + bool configured; /* Configuration completed */ bool reopen; /* Reopen an existing database */ bool workers_finished; /* Operations completed */ char *home; /* Home directory */ char *home_config; /* Run CONFIG file path */ char *home_hsdump; /* HS dump filename */ - char *home_init; /* Initialize home command */ char *home_key; /* Key file filename */ - char *home_log; /* Operation log file path */ char *home_pagedump; /* Page dump filename */ - char *home_rand; /* RNG log file path */ char *home_stats; /* Statistics file path */ char *config_open; /* Command-line configuration */ - uint32_t run_cnt; /* Run counter */ - bool trace; /* trace operations */ bool trace_all; /* trace all operations */ bool trace_local; /* write trace to the primary database */ + char tidbuf[128]; /* thread ID in printable form */ WT_CONNECTION *trace_conn; /* optional tracing database */ WT_SESSION *trace_session; RWLOCK backup_lock; /* Backup running */ uint64_t backup_id; /* Block incremental id */ +#define INCREMENTAL_BLOCK 1 +#define INCREMENTAL_LOG 2 +#define INCREMENTAL_OFF 3 + u_int backup_incr_flag; /* Incremental backup configuration */ WT_RAND_STATE rnd; /* Global RNG state */ - uint32_t rts_no_check; /* track unsuccessful RTS checking */ + u_int rts_no_check; /* track unsuccessful RTS checking */ + + uint64_t timestamp; /* Counter for timestamps */ + uint64_t oldest_timestamp; /* Last timestamp used for oldest */ + uint64_t stable_timestamp; /* Last timestamp used for stable */ /* * Prepare will return an error if the prepare timestamp is less than any active read timestamp. @@ -153,6 +219,7 @@ typedef struct { * that requires locking out transactional ops that set a timestamp. */ RWLOCK ts_lock; + /* * Lock to prevent the stable timestamp from moving during the commit of prepared transactions. * Otherwise, it may panic if the stable timestamp is moved to greater than or equal to the @@ -160,163 +227,38 @@ typedef struct { */ RWLOCK prepare_commit_lock; - uint64_t timestamp; /* Counter for timestamps */ - uint64_t oldest_timestamp; /* Last timestamp used for oldest */ - uint64_t stable_timestamp; /* Last timestamp used for stable */ - - uint64_t truncate_cnt; /* Counter for truncation */ - /* - * Single-thread failure. Always use pthread lock rather than WT lock in case WT library is - * misbehaving. + * Single-thread failure. Not a WiredTiger library lock because it's set up before configuring + * anything. */ pthread_rwlock_t death_lock; + WT_CURSOR *page_dump_cursor; /* Snapshot isolation read failed, modifies failure handling. */ - uint32_t c_abort; /* Config values */ - uint32_t c_alter; - uint32_t c_assert_read_timestamp; - uint32_t c_assert_write_timestamp; - uint32_t c_auto_throttle; - char *c_backup_incremental; - uint32_t c_backup_incr_granularity; - uint32_t c_backups; - uint32_t c_bitcnt; - uint32_t c_bloom; - uint32_t c_bloom_bit_count; - uint32_t c_bloom_hash_count; - uint32_t c_bloom_oldest; - uint32_t c_cache; - uint32_t c_cache_minimum; - char *c_checkpoint; - uint32_t c_checkpoint_log_size; - uint32_t c_checkpoint_wait; - char *c_checksum; - uint32_t c_chunk_size; - uint32_t c_compact; - char *c_compression; - char *c_config_open; - uint32_t c_data_extend; - char *c_data_source; - uint32_t c_delete_pct; - uint32_t c_dictionary; - uint32_t c_direct_io; - char *c_encryption; - uint32_t c_evict_max; - char *c_file_type; - uint32_t c_firstfit; - uint32_t c_hs_cursor; - uint32_t c_huffman_value; - uint32_t c_import; - uint32_t c_in_memory; - uint32_t c_independent_thread_rng; - uint32_t c_insert_pct; - uint32_t c_internal_key_truncation; - uint32_t c_intl_page_max; - uint32_t c_key_max; - uint32_t c_key_min; - uint32_t c_leaf_page_max; - uint32_t c_leak_memory; - uint32_t c_logging; - uint32_t c_logging_archive; - char *c_logging_compression; - uint32_t c_logging_file_max; - uint32_t c_logging_prealloc; - uint32_t c_lsm_worker_threads; - uint32_t c_major_timeout; - uint32_t c_memory_page_max; - uint32_t c_merge_max; - uint32_t c_mmap; - uint32_t c_mmap_all; - uint32_t c_modify_pct; - uint32_t c_ops; - uint32_t c_prefix; - uint32_t c_prefix_compression; - uint32_t c_prefix_compression_min; - uint32_t c_prepare; - uint32_t c_quiet; - uint32_t c_random_cursor; - uint32_t c_read_pct; - uint32_t c_repeat_data_pct; - uint32_t c_reverse; - uint32_t c_rows; - uint32_t c_runs; - uint32_t c_salvage; - uint32_t c_split_pct; - uint32_t c_statistics; - uint32_t c_statistics_server; - uint32_t c_threads; - uint32_t c_timer; - uint32_t c_timing_stress_aggressive_sweep; - uint32_t c_timing_stress_checkpoint; - uint32_t c_timing_stress_checkpoint_reserved_txnid_delay; - uint32_t c_timing_stress_failpoint_hs_delete_key_from_ts; - uint32_t c_timing_stress_failpoint_hs_insert_1; - uint32_t c_timing_stress_failpoint_hs_insert_2; - uint32_t c_timing_stress_hs_checkpoint_delay; - uint32_t c_timing_stress_hs_search; - uint32_t c_timing_stress_hs_sweep; - uint32_t c_timing_stress_checkpoint_prepare; - uint32_t c_timing_stress_split_1; - uint32_t c_timing_stress_split_2; - uint32_t c_timing_stress_split_3; - uint32_t c_timing_stress_split_4; - uint32_t c_timing_stress_split_5; - uint32_t c_timing_stress_split_6; - uint32_t c_timing_stress_split_7; - uint32_t c_truncate; - uint32_t c_txn_implicit; - uint32_t c_txn_timestamps; - uint32_t c_value_max; - uint32_t c_value_min; - uint32_t c_verify; - uint32_t c_verify_failure_dump; - uint32_t c_write_pct; - uint32_t c_wt_mutex; - -#define FIX 1 -#define ROW 2 -#define VAR 3 - u_int type; /* File type's flag value */ + /* Any runs.type configuration. */ + char runs_type[64]; -#define INCREMENTAL_BLOCK 1 -#define INCREMENTAL_LOG 2 -#define INCREMENTAL_OFF 3 - u_int c_backup_incr_flag; /* Incremental backup flag value */ + /* + * The minimum key size: A minimum key size of 11 is necessary, row-store keys have a leading + * 10-digit number and the 11 guarantees we never see a key we can't immediately convert to a + * numeric value without modification (there's a trailing non-digit character after every key). + * + * Range of common key prefix selection and the maximum table prefix length. + */ +#define KEY_LEN_CONFIG_MIN 11 +#define PREFIX_LEN_CONFIG_MIN 15 +#define PREFIX_LEN_CONFIG_MAX 80 + uint32_t prefix_len_max; + + bool column_store_config; /* At least one column-store table configured */ + bool lsm_config; /* At least one LSM data source configured */ + bool multi_table_config; /* If configuring multiple tables */ + bool transaction_timestamps_config; /* If transaction timestamps configured on any table */ #define CHECKPOINT_OFF 1 #define CHECKPOINT_ON 2 #define CHECKPOINT_WIREDTIGER 3 - u_int c_checkpoint_flag; /* Checkpoint flag value */ - -#define CHECKSUM_OFF 1 -#define CHECKSUM_ON 2 -#define CHECKSUM_UNCOMPRESSED 3 -#define CHECKSUM_UNENCRYPTED 4 - u_int c_checksum_flag; /* Checksum flag value */ - -#define COMPRESS_NONE 1 -#define COMPRESS_LZ4 2 -#define COMPRESS_SNAPPY 3 -#define COMPRESS_ZLIB 4 -#define COMPRESS_ZSTD 5 - u_int c_compression_flag; /* Compression flag value */ - u_int c_logging_compression_flag; /* Log compression flag value */ - -#define ENCRYPT_NONE 1 -#define ENCRYPT_ROTN_7 2 -#define ENCRYPT_SODIUM 3 - u_int c_encryption_flag; /* Encryption flag value */ - -/* The page must be a multiple of the allocation size, and 512 always works. */ -#define BLOCK_ALLOCATION_SIZE 512 - uint32_t intl_page_max; /* Maximum page sizes */ - uint32_t leaf_page_max; - - uint64_t rows; /* Total rows */ - - uint32_t prefix_len; /* Common key prefix length */ - uint32_t key_rand_len[1031]; /* Key lengths */ + u_int checkpoint_config; /* Checkpoint configuration */ } GLOBAL; extern GLOBAL g; @@ -330,6 +272,7 @@ typedef struct { thread_op op; /* Operation */ uint64_t opid; /* Operation ID */ + uint32_t id; /* Table ID */ uint64_t keyno; /* Row number */ uint64_t ts; /* Read/commit timestamp */ @@ -373,7 +316,14 @@ typedef struct { uint64_t update; WT_SESSION *session; /* WiredTiger session */ - WT_CURSOR *cursor; /* WiredTiger cursor */ + WT_CURSOR **cursors; /* WiredTiger cursors, maps one-to-one to tables */ + WT_CURSOR *cursor; /* Current cursor */ + TABLE *table; /* Current table */ + + struct col_insert { + uint32_t insert_list[256]; /* Inserted column-store records, maps one-to-one to tables */ + u_int insert_list_cnt; + } * col_insert; WT_SESSION *trace; /* WiredTiger operations tracing session */ @@ -398,9 +348,6 @@ typedef struct { #define snap_first s->snap_state_first #define snap_list s->snap_state_list - uint64_t insert_list[256]; /* column-store inserted records */ - u_int insert_list_cnt; - WT_ITEM vprint; /* Temporary buffer for printable values */ WT_ITEM moda, modb; /* Temporary buffer for modify operations */ @@ -426,17 +373,16 @@ void config_clear(void); void config_compat(const char **); void config_error(void); void config_file(const char *); -void config_final(void); void config_print(bool); void config_run(void); -void config_single(const char *, bool); +void config_single(TABLE *, const char *, bool); void create_database(const char *home, WT_CONNECTION **connp); void fclose_and_clear(FILE **); bool fp_readv(FILE *, char *, uint32_t *); -void key_gen_common(WT_ITEM *, uint64_t, const char *); +void key_gen_common(TABLE *, WT_ITEM *, uint64_t, const char *); void key_gen_init(WT_ITEM *); void key_gen_teardown(WT_ITEM *); -void key_init(void); +void key_init(TABLE *, void *); void lock_destroy(WT_SESSION *, RWLOCK *); void lock_init(WT_SESSION *, RWLOCK *); void operations(u_int, bool); @@ -445,39 +391,44 @@ void set_alarm(u_int); void set_core_off(void); void set_oldest_timestamp(void); void snap_init(TINFO *); -void snap_teardown(TINFO *); void snap_op_init(TINFO *, uint64_t, bool); -void snap_repeat_rollback(WT_CURSOR *, TINFO **, size_t); -void snap_repeat_single(WT_CURSOR *, TINFO *); -int snap_repeat_txn(WT_CURSOR *, TINFO *); +void snap_repeat_rollback(TINFO **, size_t); +void snap_repeat_single(TINFO *); +int snap_repeat_txn(TINFO *); void snap_repeat_update(TINFO *, bool); +void snap_teardown(TINFO *); void snap_track(TINFO *, thread_op); void timestamp_init(void); void timestamp_once(WT_SESSION *, bool, bool); void timestamp_teardown(WT_SESSION *); -int trace_config(const char *); +void trace_config(const char *); void trace_init(void); void trace_ops_init(TINFO *); void trace_teardown(void); -void track(const char *, uint64_t, TINFO *); -void val_gen(WT_RAND_STATE *, WT_ITEM *, uint64_t); +void track(const char *, uint64_t); +void track_ops(TINFO *); +void val_gen(TABLE *, WT_RAND_STATE *, WT_ITEM *, uint64_t); void val_gen_init(WT_ITEM *); void val_gen_teardown(WT_ITEM *); -void val_init(void); +void val_init(TABLE *, void *); void wts_checkpoints(void); void wts_close(WT_CONNECTION **, WT_SESSION **); -void wts_create(const char *); +void wts_create_database(void); +void wts_create_home(void); void wts_dump(const char *, bool); -void wts_load(void); +void wts_load(TABLE *, void *); void wts_open(const char *, WT_CONNECTION **, WT_SESSION **, bool); -void wts_read_scan(void); +void wts_read_scan(TABLE *, void *); void wts_reopen(void); -void wts_salvage(void); +void wts_salvage(TABLE *, void *); void wts_stats(void); -void wts_verify(WT_CONNECTION *, const char *); +void wts_verify(TABLE *, void *); +/* Backward compatibility to older versions of the WiredTiger library. */ #if !defined(CUR2S) #define CUR2S(c) ((WT_SESSION_IMPL *)((WT_CURSOR *)c)->session) #endif +#define WARN(fmt, ...) fprintf(stderr, "%s: WARNING: " fmt "\n", progname, __VA_ARGS__); + #include "format.i" diff --git a/src/third_party/wiredtiger/test/format/format.i b/src/third_party/wiredtiger/test/format/format.i index f0645ffb956..75cde988b0d 100644 --- a/src/third_party/wiredtiger/test/format/format.i +++ b/src/third_party/wiredtiger/test/format/format.i @@ -147,6 +147,138 @@ random_sleep(WT_RAND_STATE *rnd, u_int max_seconds) } } +/* + * tables_apply - + * Call an underlying function on all tables. + */ +static inline void +tables_apply(void (*func)(TABLE *, void *), void *arg) +{ + u_int i; + + if (ntables == 0) + func(tables[0], arg); + else + for (i = 1; i <= ntables; ++i) + func(tables[i], arg); +} + +/* + * table_maxv -- + * Return the maximum value for a table configuration variable. + */ +static inline uint32_t +table_maxv(u_int off) +{ + uint32_t v; + u_int i; + + if (ntables == 0) + return (tables[0]->v[off].v); + + for (v = 0, i = 1; i <= ntables; ++i) + v = WT_MAX(v, tables[i]->v[off].v); + return (v); +} + +/* + * table_sumv -- + * Return the summed value for a table configuration variable. + */ +static inline uint32_t +table_sumv(u_int off) +{ + uint32_t v; + u_int i; + + if (ntables == 0) + return (tables[0]->v[off].v); + + for (v = 0, i = 1; i <= ntables; ++i) + v += tables[i]->v[off].v; + return (v); +} + +/* + * table_select -- + * Randomly select a table. + */ +static inline TABLE * +table_select(TINFO *tinfo) +{ + if (ntables == 0) + return (tables[0]); + + return (tables[mmrand(tinfo == NULL ? NULL : &tinfo->rnd, 1, ntables)]); +} + +/* + * table_select_type -- + * Randomly select a table of a specific type. + */ +static inline TABLE * +table_select_type(table_type type) +{ + u_int i; + + if (ntables == 0) + return (tables[0]->type == type ? tables[0] : NULL); + + for (i = mmrand(NULL, 1, ntables);; ++i) { + if (i > ntables) + i = 1; + if (tables[i]->type == type) + break; + } + return (tables[i]); +} + +/* + * wiredtiger_open_cursor -- + * Open a WiredTiger cursor. + */ +static inline void +wiredtiger_open_cursor( + WT_SESSION *session, const char *uri, const char *config, WT_CURSOR **cursorp) +{ + WT_DECL_RET; + + *cursorp = NULL; + + /* WT_SESSION.open_cursor can return EBUSY if concurrent with a metadata operation, retry. */ + while ((ret = session->open_cursor(session, uri, NULL, config, cursorp)) == EBUSY) + __wt_yield(); + testutil_checkfmt(ret, "%s", uri); +} + +/* + * table_cursor -- + * Return the cursor for a table, opening a new one if necessary. + */ +static inline WT_CURSOR * +table_cursor(TINFO *tinfo, u_int id) +{ + TABLE *table; + const char *config; + + testutil_assert(id > 0); + + /* The table ID is 1-based, the cursor array is 0-based. */ + table = tables[ntables == 0 ? 0 : id]; + --id; + + if (tinfo->cursors[id] == NULL) { + /* Configure "append", in the case of column stores, we append when inserting new rows. */ + config = table->type == ROW ? NULL : "append"; + wiredtiger_open_cursor(tinfo->session, table->uri, config, &tinfo->cursors[id]); + } + return (tinfo->cursors[id]); +} + +/* + * wiredtiger_begin_transaction -- + * Start a WiredTiger transaction. + */ static inline void wiredtiger_begin_transaction(WT_SESSION *session, const char *config) { @@ -166,9 +298,9 @@ wiredtiger_begin_transaction(WT_SESSION *session, const char *config) * Generate a key for lookup. */ static inline void -key_gen(WT_ITEM *key, uint64_t keyno) +key_gen(TABLE *table, WT_ITEM *key, uint64_t keyno) { - key_gen_common(key, keyno, "00"); + key_gen_common(table, key, keyno, "00"); } /* @@ -176,12 +308,12 @@ key_gen(WT_ITEM *key, uint64_t keyno) * Generate a key for insertion. */ static inline void -key_gen_insert(WT_RAND_STATE *rnd, WT_ITEM *key, uint64_t keyno) +key_gen_insert(TABLE *table, WT_RAND_STATE *rnd, WT_ITEM *key, uint64_t keyno) { static const char *const suffix[15] = { "01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12", "13", "14", "15"}; - key_gen_common(key, keyno, suffix[mmrand(rnd, 0, 14)]); + key_gen_common(table, key, keyno, suffix[mmrand(rnd, 0, 14)]); } /* @@ -193,11 +325,9 @@ lock_try_writelock(WT_SESSION *session, RWLOCK *lock) { testutil_assert(LOCK_INITIALIZED(lock)); - if (lock->lock_type == LOCK_WT) { + if (lock->lock_type == LOCK_WT) return (__wt_try_writelock((WT_SESSION_IMPL *)session, &lock->l.wt)); - } else { - return (pthread_rwlock_trywrlock(&lock->l.pthread)); - } + return (pthread_rwlock_trywrlock(&lock->l.pthread)); } /* @@ -209,11 +339,10 @@ lock_writelock(WT_SESSION *session, RWLOCK *lock) { testutil_assert(LOCK_INITIALIZED(lock)); - if (lock->lock_type == LOCK_WT) { + if (lock->lock_type == LOCK_WT) __wt_writelock((WT_SESSION_IMPL *)session, &lock->l.wt); - } else { + else testutil_check(pthread_rwlock_wrlock(&lock->l.pthread)); - } } /* @@ -225,11 +354,10 @@ lock_writeunlock(WT_SESSION *session, RWLOCK *lock) { testutil_assert(LOCK_INITIALIZED(lock)); - if (lock->lock_type == LOCK_WT) { + if (lock->lock_type == LOCK_WT) __wt_writeunlock((WT_SESSION_IMPL *)session, &lock->l.wt); - } else { + else testutil_check(pthread_rwlock_unlock(&lock->l.pthread)); - } } /* @@ -241,11 +369,10 @@ lock_readlock(WT_SESSION *session, RWLOCK *lock) { testutil_assert(LOCK_INITIALIZED(lock)); - if (lock->lock_type == LOCK_WT) { + if (lock->lock_type == LOCK_WT) __wt_readlock((WT_SESSION_IMPL *)session, &lock->l.wt); - } else { + else testutil_check(pthread_rwlock_rdlock(&lock->l.pthread)); - } } /* @@ -257,11 +384,10 @@ lock_readunlock(WT_SESSION *session, RWLOCK *lock) { testutil_assert(LOCK_INITIALIZED(lock)); - if (lock->lock_type == LOCK_WT) { + if (lock->lock_type == LOCK_WT) __wt_readunlock((WT_SESSION_IMPL *)session, &lock->l.wt); - } else { + else testutil_check(pthread_rwlock_unlock(&lock->l.pthread)); - } } #define trace_msg(fmt, ...) \ @@ -275,16 +401,16 @@ lock_readunlock(WT_SESSION *session, RWLOCK *lock) (uintmax_t)__ts.tv_nsec / WT_THOUSAND, g.tidbuf, __VA_ARGS__)); \ } \ } while (0) -#define trace_op(tinfo, fmt, ...) \ - do { \ - if (g.trace) { \ - struct timespec __ts; \ - WT_SESSION *__s = (tinfo)->trace; \ - __wt_epoch((WT_SESSION_IMPL *)__s, &__ts); \ - testutil_check( \ - __s->log_printf(__s, "[%" PRIuMAX ":%" PRIuMAX "][%s] " fmt, (uintmax_t)__ts.tv_sec, \ - (uintmax_t)__ts.tv_nsec / WT_THOUSAND, tinfo->tidbuf, __VA_ARGS__)); \ - } \ +#define trace_op(tinfo, fmt, ...) \ + do { \ + if (g.trace) { \ + struct timespec __ts; \ + WT_SESSION *__s = (tinfo)->trace; \ + __wt_epoch((WT_SESSION_IMPL *)__s, &__ts); \ + testutil_check(__s->log_printf(__s, "[%" PRIuMAX ":%" PRIuMAX "][%s]:%s " fmt, \ + (uintmax_t)__ts.tv_sec, (uintmax_t)__ts.tv_nsec / WT_THOUSAND, (tinfo)->tidbuf, \ + (tinfo)->table->uri, __VA_ARGS__)); \ + } \ } while (0) /* diff --git a/src/third_party/wiredtiger/test/format/format.sh b/src/third_party/wiredtiger/test/format/format.sh index 461875e33bb..2b71673d061 100755 --- a/src/third_party/wiredtiger/test/format/format.sh +++ b/src/third_party/wiredtiger/test/format/format.sh @@ -17,12 +17,14 @@ onintr() trap 'onintr' 2 usage() { - echo "usage: $0 [-aEFRSv] [-b format-binary] [-c config] [-e env-var]" - echo " [-h home] [-j parallel-jobs] [-n total-jobs] [-r live-record-binary] [-t minutes] [format-configuration]" + echo "usage: $0 [-aEFRSv] [-b format-binary] [-c config] [-D directory]" + echo " [-e env-var] [-h home] [-j parallel-jobs] [-n total-jobs] [-r live-record-binary]" + echo " [-t minutes] [format-configuration]" echo echo " -a add configuration for abort/recovery testing (defaults to off)" echo " -b binary format binary (defaults to "./t")" echo " -c config format configuration file (defaults to CONFIG.stress)" + echo " -D directory directory of format configuration files (named \"CONFIG.*\")" echo " -E skip known errors (defaults to off)" echo " -e envvar Environment variable setting (default to none)" echo " -F quit on first failure (defaults to off)" @@ -45,17 +47,16 @@ smoke_base_2="$smoke_base_1 leaf_page_max=9 internal_page_max=9" smoke_list=( # Three access methods. "$smoke_base_1 file_type=row" - # Temporarily disabled + # Temporarily disabled: FIXME FLCS # "$smoke_base_1 file_type=fix" - # "$smoke_base_1 file_type=var" + "$smoke_base_1 file_type=var" # Huffman value encoding. "$smoke_base_1 file_type=row huffman_value=1" - # Temporarily disabled - # "$smoke_base_1 file_type=var huffman_value=1" + "$smoke_base_1 file_type=var huffman_value=1" # LSM - # Temporarily disabled + # Temporarily disabled: FIXME LSM # "$smoke_base_1 file_type=row runs.source=lsm" # Force the statistics server. @@ -64,10 +65,14 @@ smoke_list=( # Overflow testing. "$smoke_base_2 file_type=row key_min=256" "$smoke_base_2 file_type=row key_min=256 value_min=256" - # Temporarily disabled - # "$smoke_base_2 file_type=var value_min=256" + "$smoke_base_2 file_type=var value_min=256" ) + smoke_next=0 +smoke_test=0 + +directory_next=0 +directory_total=0 abort_test=0 build="" @@ -80,7 +85,6 @@ live_record_binary="" minutes=0 parallel_jobs=8 skip_errors=0 -smoke_test=0 stress_split_test=0 total_jobs=0 verbose=0 @@ -96,6 +100,17 @@ while :; do -c) config="$2" shift ; shift ;; + -D) + # Format changes directories, get absolute paths to the CONFIG files. + dir="$2" + [[ "$dir" == /* ]] || dir="$PWD/$dir" + directory_list=($dir/CONFIG.*) + directory_total=${#directory_list[@]} + [[ -f "${directory_list[0]}" ]] || { + echo "$name: no CONFIG files found in $2" + exit 1 + } + shift ; shift ;; -E) skip_errors=1 shift ;; @@ -171,10 +186,36 @@ verbose "$name: run starting at $(date)" } home=$(cd $home > /dev/null || exit 1 && echo $PWD) +# From the Bash FAQ, shuffle an array. +shuffle() { + local i tmp size max rand + + size=${#directory_list[*]} + for ((i=size-1; i>0; i--)); do + # RANDOM % (i+1) is biased because of the limited range of $RANDOM + # Compensate by using a range which is a multiple of the rand modulus. + + max=$(( 32768 / (i+1) * (i+1) )) + while (( (rand=RANDOM) >= max )); do :; done + rand=$(( rand % (i+1) )) + tmp=${directory_list[i]} directory_list[i]=${directory_list[rand]} directory_list[rand]=$tmp + done +} + +# If we have a directory of CONFIGs, shuffle it. The directory has to be an absolute path so there +# is no additional path checking to do. +config_found=0 +[[ $directory_total -ne 0 ]] && { + shuffle + config_found=1 +} + # Config is possibly relative to our current directory and we're about to change directories. # Get an absolute path for config if it's local. -config_found=0 -[[ -f "$config" ]] && config_found=1 && config="$PWD/$config" +[[ $config_found -eq 0 ]] && [[ -f "$config" ]] && { + [[ "$config" == /* ]] || config="$PWD/$config" + config_found=1 +} # Move to the format.sh directory (assumed to be in a WiredTiger build tree). cd $(dirname $0) || exit 1 @@ -183,14 +224,16 @@ cd $(dirname $0) || exit 1 # lives in the same directory of the WiredTiger build tree as format.sh. We're about to change # directories if we don't find the format binary here, get an absolute path for config if it's # local. -[[ $config_found -eq 0 ]] && [[ -f "$config" ]] && config="$PWD/$config" +[[ $config_found -eq 0 ]] && [[ -f "$config" ]] && { + config="$PWD/$config" + config_found=1 +} # Find the last part of format_binary, which is format binary file. Builds are normally in the # WiredTiger source tree, in which case it's in the same directory as format.sh, else it's in # the build_posix tree. If the build is in the build_posix tree, move there, we have to run in # the directory where the format binary lives because the format binary "knows" the wt utility # is two directory levels above it. - [[ -x ${format_binary##* } ]] || { build_posix_directory="../../build_posix/test/format" [[ ! -d $build_posix_directory ]] || cd $build_posix_directory || exit 1 @@ -209,9 +252,11 @@ wt_binary="../../wt" # We tested for the CONFIG file in the original directory, then in the WiredTiger source directory, # the last place to check is in the WiredTiger build directory. Fail if we don't find it. -[[ -f "$config" ]] || { - echo "$name: configuration file \"$config\" not found" - exit 1 +[[ $config_found -eq 0 ]] && { + [[ -f "$config" ]] || { + echo "$name: configuration file \"$config\" not found" + exit 1 + } } verbose "$name configuration: $format_binary [-c $config]\ @@ -392,18 +437,36 @@ resolve() echo "$name: original directory copied into $dir.RECOVER" echo) >> $log - # Everything is a table unless explicitly a file. - uri="table:wt" - grep 'runs.source=file' $dir/CONFIG > /dev/null && uri="file:wt" + # Verify the objects. In current format, it's a list of files named with a + # leading F or tables named with a leading T. Historically, it was a file + # or table named "wt". + verify_failed=0 + for i in $(ls $dir | sed -e 's/.*\///'); do + case $i in + F*) uri="file:$i";; + T*) uri="table:${i%.wt}";; + wt) uri="file:wt";; + wt.wt) uri="table:wt";; + *) continue;; + esac + + # Use the wt utility to recover & verify the object. + echo "verify: $wt_binary -m -R -h $dir verify $uri" >> $log + if $($wt_binary -m -R -h $dir verify $uri >> $log 2>&1); then + continue + fi + + verify_failed=1 + break + done - # Use the wt utility to recover & verify the object. - if $($wt_binary -m -R -h $dir verify $uri >> $log 2>&1); then - rm -rf $dir $dir.RECOVER $log - success=$(($success + 1)) - verbose "$name: job in $dir successfully completed" + if [[ $verify_failed -eq 0 ]]; then + rm -rf $dir $dir.RECOVER $log + success=$(($success + 1)) + verbose "$name: job in $dir successfully completed" else - echo "$name: job in $dir failed abort/recovery testing" - report_failure $dir + echo "$name: job in $dir failed abort/recovery testing" + report_failure $dir fi continue } @@ -479,6 +542,10 @@ format() args=${smoke_list[$smoke_next]} smoke_next=$(($smoke_next + 1)) fi + if [[ $directory_total -ne 0 ]]; then + config="${directory_list[$directory_next]}" + directory_next=$(($directory_next + 1)) + fi if [[ $abort_test -ne 0 ]]; then args+=" format.abort=1" fi @@ -498,7 +565,7 @@ format() live_record_binary="$live_record_binary --save-on=error" fi - cmd="$live_record_binary $format_binary -c "$config" -h "$dir" -1 $args quiet=1" + cmd="$live_record_binary $format_binary -c "$config" -h "$dir" $args quiet=1" echo "$name: $cmd" # Disassociate the command from the shell script so we can exit and let the command @@ -536,6 +603,9 @@ while :; do # Check if we're only running the smoke-tests and we're done. [[ $smoke_test -ne 0 ]] && [[ $smoke_next -ge ${#smoke_list[@]} ]] && quit=1 + # Check if we're running CONFIGs from a directory and we're done. + [[ $directory_total -ne 0 ]] && [[ $directory_next -ge $directory_total ]] && quit=1 + # Check if the total number of jobs has been reached. [[ $total_jobs -ne 0 ]] && [[ $count_jobs -ge $total_jobs ]] && quit=1 diff --git a/src/third_party/wiredtiger/test/format/import.c b/src/third_party/wiredtiger/test/format/import.c index 0b85515b806..1d22782ca3e 100644 --- a/src/third_party/wiredtiger/test/format/import.c +++ b/src/third_party/wiredtiger/test/format/import.c @@ -104,18 +104,14 @@ import(void *arg) copy_file_into_directory(import_session, "import.wt"); /* Perform import with either repair or file metadata. */ - memset(buf, 0, sizeof(buf)); import_value = mmrand(NULL, 0, 1); - if (import_value == 0) { + if (import_value == 0) testutil_check(__wt_snprintf(buf, sizeof(buf), "import=(enabled,repair=true)")); - if ((ret = session->create(session, IMPORT_URI, buf)) != 0) - testutil_die(ret, "session.import", ret); - } else { + else testutil_check(__wt_snprintf(buf, sizeof(buf), "%s,import=(enabled,repair=false,file_metadata=(%s))", table_config, file_config)); - if ((ret = session->create(session, IMPORT_URI, buf)) != 0) - testutil_die(ret, "session.import", ret); - } + if ((ret = session->create(session, IMPORT_URI, buf)) != 0) + testutil_die(ret, "session.import", ret); verify_import(session); diff --git a/src/third_party/wiredtiger/test/format/kv.c b/src/third_party/wiredtiger/test/format/kv.c index 32788b86ffb..7760b74c3ea 100644 --- a/src/third_party/wiredtiger/test/format/kv.c +++ b/src/third_party/wiredtiger/test/format/kv.c @@ -33,11 +33,19 @@ * Initialize the keys for a run. */ void -key_init(void) +key_init(TABLE *table, void *arg) { FILE *fp; size_t i; uint32_t max; + char buf[MAX_FORMAT_PATH]; + + (void)arg; /* unused argument */ + + if (ntables == 0) + testutil_check(__wt_snprintf(buf, sizeof(buf), "%s", g.home_key)); + else + testutil_check(__wt_snprintf(buf, sizeof(buf), "%s.%u", g.home_key, table->id)); /* * The key is a variable length item with a leading 10-digit value. Since we have to be able @@ -48,10 +56,10 @@ key_init(void) * Read in the values during reopen. */ if (g.reopen) { - if ((fp = fopen(g.home_key, "r")) == NULL) - testutil_die(errno, "%s", g.home_key); - for (i = 0; i < WT_ELEMENTS(g.key_rand_len); ++i) - fp_readv(fp, g.home_key, &g.key_rand_len[i]); + if ((fp = fopen(buf, "r")) == NULL) + testutil_die(errno, "%s", buf); + for (i = 0; i < WT_ELEMENTS(table->key_rand_len); ++i) + fp_readv(fp, buf, &table->key_rand_len[i]); fclose_and_clear(&fp); return; } @@ -62,23 +70,19 @@ key_init(void) * Focus on relatively small items, admitting the possibility of larger items. Pick a size close * to the minimum most of the time, only create a larger item 1 in 20 times. */ - for (i = 0; i < WT_ELEMENTS(g.key_rand_len); ++i) { - max = g.c_key_max; - if (i % 20 != 0 && max > g.c_key_min + 20) - max = g.c_key_min + 20; - g.key_rand_len[i] = mmrand(NULL, g.c_key_min, max); + for (i = 0; i < WT_ELEMENTS(table->key_rand_len); ++i) { + max = TV(BTREE_KEY_MAX); + if (i % 20 != 0 && max > TV(BTREE_KEY_MIN) + 20) + max = TV(BTREE_KEY_MIN) + 20; + table->key_rand_len[i] = mmrand(NULL, TV(BTREE_KEY_MIN), max); } /* Write out the values for a subsequent reopen. */ - if ((fp = fopen(g.home_key, "w")) == NULL) - testutil_die(errno, "%s", g.home_key); - for (i = 0; i < WT_ELEMENTS(g.key_rand_len); ++i) - fprintf(fp, "%" PRIu32 "\n", g.key_rand_len[i]); + if ((fp = fopen(buf, "w")) == NULL) + testutil_die(errno, "%s", buf); + for (i = 0; i < WT_ELEMENTS(table->key_rand_len); ++i) + fprintf(fp, "%" PRIu32 "\n", table->key_rand_len[i]); fclose_and_clear(&fp); - - /* Fill in the common key prefix length (which is added to the key min/max). */ - if (g.c_prefix != 0) - g.prefix_len = mmrand(NULL, 15, 80); } /* @@ -91,7 +95,7 @@ key_gen_init(WT_ITEM *key) size_t i, len; char *p; - len = WT_MAX(KILOBYTE(100), g.c_key_max + g.prefix_len); + len = WT_MAX(KILOBYTE(100), table_maxv(V_TABLE_BTREE_KEY_MAX) + g.prefix_len_max + 10); p = dmalloc(len); for (i = 0; i < len; ++i) p[i] = "abcdefghijklmnopqrstuvwxyz"[i % 26]; @@ -113,20 +117,22 @@ key_gen_teardown(WT_ITEM *key) memset(key, 0, sizeof(*key)); } +#define COMMON_PREFIX_CHAR 'C' + /* * key_gen_common -- * Row-store key generation code shared between normal and insert key generation. */ void -key_gen_common(WT_ITEM *key, uint64_t keyno, const char *const suffix) +key_gen_common(TABLE *table, WT_ITEM *key, uint64_t keyno, const char *const suffix) { + size_t i; uint64_t n; + uint32_t prefix_len; char *p; const char *bucket; - testutil_assert(g.type == ROW); - - p = key->mem; + testutil_assert(table->type == ROW); /* * The workload we're trying to mimic with a prefix is a long common prefix followed by a record @@ -138,16 +144,32 @@ key_gen_common(WT_ITEM *key, uint64_t keyno, const char *const suffix) * lexicographically, meaning the common key prefix will grow and shrink by a few bytes as the * number increments, which is a good thing for testing. */ - if (g.prefix_len > 0) { - bucket = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; - for (n = keyno; n > 0; n >>= 1) { - if (*bucket == 'z') - break; - ++bucket; + p = key->mem; + prefix_len = TV(BTREE_PREFIX_LEN); + if (g.prefix_len_max != 0) { + /* + * Not all tables have prefixes and prefixes may be of different lengths. If any table has a + * prefix, check if we need to reset the leading bytes in the key to their original values. + * It's an ugly test, but it avoids rewriting the key in a performance path. The variable is + * the largest prefix in the run, and the hard-coded 20 gets us past the key appended to + * that prefix. + */ + if (p[1] == COMMON_PREFIX_CHAR) { + for (i = 0; i < g.prefix_len_max + 20; ++i) + p[i] = "abcdefghijklmnopqrstuvwxyz"[i % 26]; + p = key->mem; + } + if (prefix_len != 0) { + bucket = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; + for (n = keyno; n > 0; n >>= 1) { + if (*bucket == 'z') + break; + ++bucket; + } + p[0] = *bucket; + memset(p + 1, COMMON_PREFIX_CHAR, prefix_len - 1); + p += prefix_len; } - p[0] = *bucket; - memset(p + 1, 'C', g.prefix_len - 1); - p += g.prefix_len; } /* @@ -166,19 +188,19 @@ key_gen_common(WT_ITEM *key, uint64_t keyno, const char *const suffix) * the cache. Handle that here, use a really big key 1 in 2500 times. */ key->data = key->mem; - key->size = g.prefix_len; - key->size += keyno % 2500 == 0 && g.c_key_max < KILOBYTE(80) ? + key->size = prefix_len; + key->size += keyno % 2500 == 0 && TV(BTREE_KEY_MAX) < KILOBYTE(80) ? KILOBYTE(80) : - g.key_rand_len[keyno % WT_ELEMENTS(g.key_rand_len)]; + table->key_rand_len[keyno % WT_ELEMENTS(table->key_rand_len)]; testutil_assert(key->size <= key->memsize); } -static char *val_base; /* Base/original value */ -static uint32_t val_dup_data_len; /* Length of duplicate data items */ -static uint32_t val_len; /* Length of data items */ - +/* + * val_len -- + * Select and return the length for a value. + */ static inline uint32_t -value_len(WT_RAND_STATE *rnd, uint64_t keyno, uint32_t min, uint32_t max) +val_len(WT_RAND_STATE *rnd, uint64_t keyno, uint32_t min, uint32_t max) { /* * Focus on relatively small items, admitting the possibility of larger items. Pick a size close @@ -193,15 +215,22 @@ value_len(WT_RAND_STATE *rnd, uint64_t keyno, uint32_t min, uint32_t max) return (mmrand(rnd, min, max)); } +/* + * val_init -- + * Initialize the value structures for a table. + */ void -val_init(void) +val_init(TABLE *table, void *arg) { size_t i; + uint32_t len; + + (void)arg; /* unused argument */ /* Discard any previous value initialization. */ - free(val_base); - val_base = NULL; - val_dup_data_len = val_len = 0; + free(table->val_base); + table->val_base = NULL; + table->val_dup_data_len = 0; /* * Set initial buffer contents to recognizable text. @@ -209,23 +238,35 @@ val_init(void) * Add a few extra bytes in order to guarantee we can always offset into the buffer by a few * extra bytes, used to generate different data for column-store run-length encoded files. */ - val_len = WT_MAX(KILOBYTE(100), g.c_value_max) + 20; - val_base = dmalloc(val_len); - for (i = 0; i < val_len; ++i) - val_base[i] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"[i % 26]; + len = WT_MAX(KILOBYTE(100), table_maxv(V_TABLE_BTREE_VALUE_MAX)) + 20; + table->val_base = dmalloc(len); + for (i = 0; i < len; ++i) + table->val_base[i] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"[i % 26]; - val_dup_data_len = value_len(NULL, (uint64_t)mmrand(NULL, 1, 20), g.c_value_min, g.c_value_max); + table->val_dup_data_len = + val_len(NULL, (uint64_t)mmrand(NULL, 1, 20), TV(BTREE_VALUE_MIN), TV(BTREE_VALUE_MAX)); } +/* + * val_gen_init -- + * Initialize a single value structure. + */ void val_gen_init(WT_ITEM *value) { - value->mem = dmalloc(val_len); - value->memsize = val_len; + uint32_t len; + + len = WT_MAX(KILOBYTE(100), table_maxv(V_TABLE_BTREE_VALUE_MAX)) + 20; + value->mem = dmalloc(len); + value->memsize = len; value->data = value->mem; value->size = 0; } +/* + * val_gen_teardown -- + * Discard a single value structure. + */ void val_gen_teardown(WT_ITEM *value) { @@ -233,8 +274,12 @@ val_gen_teardown(WT_ITEM *value) memset(value, 0, sizeof(*value)); } +/* + * val_gen -- + * Generate a new value. + */ void -val_gen(WT_RAND_STATE *rnd, WT_ITEM *value, uint64_t keyno) +val_gen(TABLE *table, WT_RAND_STATE *rnd, WT_ITEM *value, uint64_t keyno) { char *p; @@ -244,8 +289,8 @@ val_gen(WT_RAND_STATE *rnd, WT_ITEM *value, uint64_t keyno) /* * Fixed-length records: take the low N bits from the last digit of the record number. */ - if (g.type == FIX) { - switch (g.c_bitcnt) { + if (table->type == FIX) { + switch (TV(BTREE_BITCNT)) { case 8: p[0] = (char)mmrand(rnd, 1, 0xff); break; @@ -289,14 +334,14 @@ val_gen(WT_RAND_STATE *rnd, WT_ITEM *value, uint64_t keyno) * Data items have unique leading numbers by default and random lengths; variable-length * column-stores use a duplicate data value to test RLE. */ - if (g.type == VAR && mmrand(rnd, 1, 100) < g.c_repeat_data_pct) { - value->size = val_dup_data_len; - memcpy(p, val_base, value->size); + if (table->type == VAR && mmrand(rnd, 1, 100) < TV(BTREE_REPEAT_DATA_PCT)) { + value->size = table->val_dup_data_len; + memcpy(p, table->val_base, value->size); (void)strcpy(p, "DUPLICATEV"); p[10] = '/'; } else { - value->size = value_len(rnd, keyno, g.c_value_min, g.c_value_max); - memcpy(p, val_base, value->size); + value->size = val_len(rnd, keyno, TV(BTREE_VALUE_MIN), TV(BTREE_VALUE_MAX)); + memcpy(p, table->val_base, value->size); u64_to_string_zf(keyno, p, 11); p[10] = '/'; } diff --git a/src/third_party/wiredtiger/test/format/ops.c b/src/third_party/wiredtiger/test/format/ops.c index 4c24ba9fd45..6d079eb12aa 100644 --- a/src/third_party/wiredtiger/test/format/ops.c +++ b/src/third_party/wiredtiger/test/format/ops.c @@ -28,23 +28,23 @@ #include "format.h" -static int col_insert(TINFO *, WT_CURSOR *); -static void col_insert_resolve(TINFO *); -static int col_modify(TINFO *, WT_CURSOR *, bool); -static int col_remove(TINFO *, WT_CURSOR *, bool); -static int col_reserve(TINFO *, WT_CURSOR *, bool); -static int col_truncate(TINFO *, WT_CURSOR *); -static int col_update(TINFO *, WT_CURSOR *, bool); -static int nextprev(TINFO *, WT_CURSOR *, bool); +static int col_insert(TINFO *); +static void col_insert_resolve(TABLE *, void *); +static int col_modify(TINFO *, bool); +static int col_remove(TINFO *, bool); +static int col_reserve(TINFO *, bool); +static int col_truncate(TINFO *); +static int col_update(TINFO *, bool); +static int nextprev(TINFO *, bool); static WT_THREAD_RET ops(void *); -static int read_row(TINFO *, WT_CURSOR *); -static int read_row_worker(WT_CURSOR *, TINFO *, uint64_t, WT_ITEM *, WT_ITEM *, bool); -static int row_insert(TINFO *, WT_CURSOR *, bool); -static int row_modify(TINFO *, WT_CURSOR *, bool); -static int row_remove(TINFO *, WT_CURSOR *, bool); -static int row_reserve(TINFO *, WT_CURSOR *, bool); -static int row_truncate(TINFO *, WT_CURSOR *); -static int row_update(TINFO *, WT_CURSOR *, bool); +static int read_row(TINFO *); +static int read_row_worker(TINFO *, TABLE *, WT_CURSOR *, uint64_t, WT_ITEM *, WT_ITEM *, bool); +static int row_insert(TINFO *, bool); +static int row_modify(TINFO *, bool); +static int row_remove(TINFO *, bool); +static int row_reserve(TINFO *, bool); +static int row_truncate(TINFO *); +static int row_update(TINFO *, bool); static char modify_repl[256]; @@ -113,13 +113,16 @@ tinfo_init(void) /* Allocate the thread structures separately to minimize false sharing. */ if (tinfo_list == NULL) { - tinfo_list = dcalloc((size_t)g.c_threads + 1, sizeof(TINFO *)); - for (i = 0; i < g.c_threads; ++i) { + tinfo_list = dcalloc((size_t)GV(RUNS_THREADS) + 1, sizeof(TINFO *)); + for (i = 0; i < GV(RUNS_THREADS); ++i) { tinfo_list[i] = dcalloc(1, sizeof(TINFO)); tinfo = tinfo_list[i]; tinfo->id = (int)i + 1; + tinfo->cursors = dcalloc(WT_MAX(ntables, 1), sizeof(tinfo->cursors[0])); + tinfo->col_insert = dcalloc(WT_MAX(ntables, 1), sizeof(tinfo->col_insert[0])); + /* Set up the default key and value buffers. */ tinfo->key = &tinfo->_key; key_gen_init(tinfo->key); @@ -133,7 +136,7 @@ tinfo_init(void) } /* Cleanup for each new run. */ - for (i = 0; i < g.c_threads; ++i) { + for (i = 0; i < GV(RUNS_THREADS); ++i) { tinfo = tinfo_list[i]; tinfo->ops = 0; @@ -147,10 +150,8 @@ tinfo_init(void) tinfo->update = 0; tinfo->session = NULL; - tinfo->cursor = NULL; - - memset(tinfo->insert_list, 0, sizeof(tinfo->insert_list)); - tinfo->insert_list_cnt = 0; + memset(tinfo->cursors, 0, WT_MAX(ntables, 1) * sizeof(tinfo->cursors[0])); + memset(tinfo->col_insert, 0, WT_MAX(ntables, 1) * sizeof(tinfo->col_insert[0])); tinfo->state = TINFO_RUNNING; tinfo->quit = false; @@ -167,19 +168,16 @@ tinfo_teardown(void) TINFO *tinfo; u_int i; - for (i = 0; i < g.c_threads; ++i) { + for (i = 0; i < GV(RUNS_THREADS); ++i) { tinfo = tinfo_list[i]; + free(tinfo->cursors); + free(tinfo->col_insert); + __wt_buf_free(NULL, &tinfo->vprint); __wt_buf_free(NULL, &tinfo->moda); __wt_buf_free(NULL, &tinfo->modb); - /* - * Assert records were not removed unless configured to do so, otherwise subsequent runs can - * incorrectly report scan errors. - */ - testutil_assert(g.c_delete_pct != 0 || tinfo->remove == 0); - snap_teardown(tinfo); key_gen_teardown(tinfo->key); val_gen_teardown(tinfo->value); @@ -192,25 +190,21 @@ tinfo_teardown(void) } /* - * tinfo_rollback_to_stable -- + * rollback_to_stable -- * Do a rollback to stable and verify operations. */ static void -tinfo_rollback_to_stable(WT_SESSION *session) +rollback_to_stable(void) { - WT_CURSOR *cursor; - - /* Rollback-to-stable only makes sense for timestamps and on-disk stores. */ - if (g.c_txn_timestamps == 0 || g.c_in_memory != 0) + /* Rollback-to-stable only makes sense for timestamps. */ + if (!g.transaction_timestamps_config) return; trace_msg("%-10s ts=%" PRIu64, "rts", g.stable_timestamp); testutil_check(g.wts_conn->rollback_to_stable(g.wts_conn, NULL)); /* Check the saved snap operations for consistency. */ - testutil_check(session->open_cursor(session, g.uri, NULL, NULL, &cursor)); - snap_repeat_rollback(cursor, tinfo_list, g.c_threads); - testutil_check(cursor->close(cursor)); + snap_repeat_rollback(tinfo_list, GV(RUNS_THREADS)); } /* @@ -252,12 +246,12 @@ operations(u_int ops_seconds, bool lastrun) * Calculate how many fourth-of-a-second sleeps until the timer expires. If the timer expires * and threads don't return in 15 minutes, assume there is something hung, and force the quit. */ - if (g.c_ops == 0) + if (GV(RUNS_OPS) == 0) thread_ops = -1; else { - if (g.c_ops < g.c_threads) - g.c_ops = g.c_threads; - thread_ops = g.c_ops / g.c_threads; + if (GV(RUNS_OPS) < GV(RUNS_THREADS)) + GV(RUNS_OPS) = GV(RUNS_THREADS); + thread_ops = GV(RUNS_OPS) / GV(RUNS_THREADS); } if (ops_seconds == 0) fourths = quit_fourths = -1; @@ -273,35 +267,35 @@ operations(u_int ops_seconds, bool lastrun) tinfo_init(); trace_msg("%s", "=============== thread ops start"); - for (i = 0; i < g.c_threads; ++i) { + for (i = 0; i < GV(RUNS_THREADS); ++i) { tinfo = tinfo_list[i]; testutil_check(__wt_thread_create(NULL, &tinfo->tid, ops, tinfo)); } /* Start optional special-purpose threads. */ - if (g.c_alter) + if (GV(OPS_ALTER)) testutil_check(__wt_thread_create(NULL, &alter_tid, alter, NULL)); - if (g.c_backups) + if (GV(BACKUP)) testutil_check(__wt_thread_create(NULL, &backup_tid, backup, NULL)); - if (g.c_compact) + if (GV(OPS_COMPACTION)) testutil_check(__wt_thread_create(NULL, &compact_tid, compact, NULL)); - if (g.c_hs_cursor) + if (GV(OPS_HS_CURSOR)) testutil_check(__wt_thread_create(NULL, &hs_tid, hs_cursor, NULL)); - if (g.c_import) + if (GV(IMPORT)) testutil_check(__wt_thread_create(NULL, &import_tid, import, NULL)); - if (g.c_random_cursor) + if (GV(OPS_RANDOM_CURSOR)) testutil_check(__wt_thread_create(NULL, &random_tid, random_kv, NULL)); - if (g.c_txn_timestamps) + if (g.transaction_timestamps_config) testutil_check(__wt_thread_create(NULL, ×tamp_tid, timestamp, tinfo_list)); - if (g.c_checkpoint_flag == CHECKPOINT_ON) + if (g.checkpoint_config == CHECKPOINT_ON) testutil_check(__wt_thread_create(NULL, &checkpoint_tid, checkpoint, NULL)); /* Spin on the threads, calculating the totals. */ for (;;) { /* Clear out the totals each pass. */ memset(&total, 0, sizeof(total)); - for (i = 0, running = false; i < g.c_threads; ++i) { + for (i = 0, running = false; i < GV(RUNS_THREADS); ++i) { tinfo = tinfo_list[i]; total.commit += tinfo->commit; total.insert += tinfo->insert; @@ -332,12 +326,12 @@ operations(u_int ops_seconds, bool lastrun) /* * On the last execution, optionally drop core for recovery testing. */ - if (lastrun && g.c_abort) + if (lastrun && GV(FORMAT_ABORT)) random_failure(); tinfo->quit = true; } } - track("ops", 0ULL, &total); + track_ops(&total); if (!running) break; __wt_sleep(0, 250000); /* 1/4th of a second */ @@ -363,21 +357,21 @@ operations(u_int ops_seconds, bool lastrun) /* Wait for the special-purpose threads. */ g.workers_finished = true; - if (g.c_alter) + if (GV(OPS_ALTER)) testutil_check(__wt_thread_join(NULL, &alter_tid)); - if (g.c_backups) + if (GV(BACKUP)) testutil_check(__wt_thread_join(NULL, &backup_tid)); - if (g.c_checkpoint_flag == CHECKPOINT_ON) + if (g.checkpoint_config == CHECKPOINT_ON) testutil_check(__wt_thread_join(NULL, &checkpoint_tid)); - if (g.c_compact) + if (GV(OPS_COMPACTION)) testutil_check(__wt_thread_join(NULL, &compact_tid)); - if (g.c_hs_cursor) + if (GV(OPS_HS_CURSOR)) testutil_check(__wt_thread_join(NULL, &hs_tid)); - if (g.c_import) + if (GV(IMPORT)) testutil_check(__wt_thread_join(NULL, &import_tid)); - if (g.c_random_cursor) + if (GV(OPS_RANDOM_CURSOR)) testutil_check(__wt_thread_join(NULL, &random_tid)); - if (g.c_txn_timestamps) + if (g.transaction_timestamps_config) testutil_check(__wt_thread_join(NULL, ×tamp_tid)); g.workers_finished = false; @@ -390,7 +384,7 @@ operations(u_int ops_seconds, bool lastrun) * close/re-open pair. Note we are not advancing the oldest timestamp, otherwise we wouldn't be * able to replay operations from after rollback-to-stable completes. */ - tinfo_rollback_to_stable(session); + rollback_to_stable(); if (lastrun) { tinfo_teardown(); @@ -496,12 +490,12 @@ commit_transaction(TINFO *tinfo, bool prepared) uint64_t ts; char buf[64]; - ++tinfo->commit; - session = tinfo->session; + ++tinfo->commit; + ts = 0; /* -Wconditional-uninitialized */ - if (g.c_txn_timestamps) { + if (g.transaction_timestamps_config) { if (prepared) lock_readlock(session, &g.prepare_commit_lock); @@ -628,47 +622,35 @@ prepare_transaction(TINFO *tinfo) } while (0) /* - * ops_open_session -- - * Create a new session/cursor pair for the thread. + * ops_session_open -- + * Create a new session for the thread. */ static void -ops_open_session(TINFO *tinfo) +ops_session_open(TINFO *tinfo) { WT_CONNECTION *conn; - WT_CURSOR *cursor; - WT_DECL_RET; WT_SESSION *session; conn = g.wts_conn; - /* Close any open session/cursor. */ + /* Close any open session (which closes all open cursors as well). */ if ((session = tinfo->session) != NULL) testutil_check(session->close(session, NULL)); + tinfo->session = NULL; + memset(tinfo->cursors, 0, WT_MAX(ntables, 1) * sizeof(tinfo->cursors[0])); - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - - /* - * Configure "append", in the case of column stores, we append when inserting new rows. - * - * WT_SESSION.open_cursor can return EBUSY if concurrent with a metadata operation, retry. - */ - while ((ret = session->open_cursor(session, g.uri, NULL, "append", &cursor)) == EBUSY) - __wt_yield(); - testutil_checkfmt(ret, "%s", g.uri); - - tinfo->session = session; - tinfo->cursor = cursor; + testutil_check(conn->open_session(conn, NULL, NULL, &tinfo->session)); } /* Isolation configuration. */ typedef enum { + ISOLATION_IMPLICIT, ISOLATION_READ_COMMITTED, ISOLATION_READ_UNCOMMITTED, ISOLATION_SNAPSHOT } iso_level_t; -/* When in an explicit snapshot isolation transaction, track operations for later - * repetition. */ +/* When in an explicit snapshot isolation transaction, track operations for later repetition. */ #define SNAP_TRACK(tinfo, op) \ do { \ if (intxn && iso_level == ISOLATION_SNAPSHOT) \ @@ -682,14 +664,15 @@ typedef enum { static WT_THREAD_RET ops(void *arg) { + struct col_insert *cip; TINFO *tinfo; - WT_CURSOR *cursor; + TABLE *table; WT_DECL_RET; WT_SESSION *session; iso_level_t iso_level; thread_op op; - uint64_t max_rows, reset_op, session_op, truncate_op; - uint32_t range, rnd; + uint64_t reset_op, session_op, truncate_op; + uint32_t max_rows, range, rnd; u_int i, j; const char *iso_config; bool greater_than, intxn, next, positioned, prepared; @@ -703,29 +686,28 @@ ops(void *arg) * pound on the same key/value pairs, that is, by making them traverse the same RNG space. 75% * of the time we run in independent RNG space. */ - if (g.c_independent_thread_rng) + if (GV(FORMAT_INDEPENDENT_THREAD_RNG)) __wt_random_init_seed(NULL, &tinfo->rnd); else __wt_random_init(&tinfo->rnd); iso_level = ISOLATION_SNAPSHOT; /* -Wconditional-uninitialized */ - /* Set the first operation where we'll create sessions and cursors. */ - cursor = NULL; + /* Set the first operation where we'll create a new session and cursors. */ session = NULL; session_op = 0; /* Set the first operation where we'll reset the session. */ reset_op = mmrand(&tinfo->rnd, 100, 10000); /* Set the first operation where we'll truncate a range. */ - truncate_op = g.c_truncate == 0 ? UINT64_MAX : mmrand(&tinfo->rnd, 100, 10000); + truncate_op = mmrand(&tinfo->rnd, 100, 10000); /* Initialize operation trace. */ trace_ops_init(tinfo); for (intxn = false; !tinfo->quit; ++tinfo->ops) { /* Periodically open up a new session and cursors. */ - if (tinfo->ops > session_op || session == NULL || cursor == NULL) { + if (tinfo->ops > session_op || session == NULL) { /* * We can't swap sessions/cursors if in a transaction, resolve any running transaction. */ @@ -734,13 +716,11 @@ ops(void *arg) intxn = false; } - ops_open_session(tinfo); + ops_session_open(tinfo); + session = tinfo->session; /* Pick the next session/cursor close/open. */ session_op += mmrand(&tinfo->rnd, 100, 5000); - - session = tinfo->session; - cursor = tinfo->cursor; } /* @@ -752,20 +732,27 @@ ops(void *arg) testutil_check(session->reset(session)); /* Pick the next reset operation. */ - reset_op += mmrand(&tinfo->rnd, 20000, 50000); + reset_op += mmrand(&tinfo->rnd, 40000, 60000); } + /* Select a table, acquire a cursor. */ + tinfo->table = table = table_select(tinfo); + tinfo->cursor = table_cursor(tinfo, table->id); + /* - * If not in a transaction and in a timestamp world, occasionally repeat a timestamped - * operation. + * If not in a transaction and in a timestamp world, occasionally repeat timestamped + * operations. */ - if (!intxn && g.c_txn_timestamps && mmrand(&tinfo->rnd, 1, 15) == 1) { + if (!intxn && g.transaction_timestamps_config && mmrand(&tinfo->rnd, 1, 15) == 1) { ++tinfo->search; - snap_repeat_single(cursor, tinfo); + snap_repeat_single(tinfo); } - /* If not in a transaction and in a timestamp world, start a transaction. */ - if (!intxn && g.c_txn_timestamps) { + /* + * If not in a transaction and in a timestamp world, start a transaction (which is always at + * snapshot-isolation). + */ + if (!intxn && g.transaction_timestamps_config) { iso_level = ISOLATION_SNAPSHOT; begin_transaction_ts(tinfo); intxn = true; @@ -773,47 +760,52 @@ ops(void *arg) /* * If not in a transaction and not in a timestamp world, start a transaction some percentage - * of the time. + * of the time, otherwise it's an implicit transaction. */ - if (!intxn && mmrand(&tinfo->rnd, 1, 100) < g.c_txn_implicit) { - iso_level = ISOLATION_SNAPSHOT; - iso_config = "isolation=snapshot"; + if (!intxn && !g.transaction_timestamps_config) { + iso_level = ISOLATION_IMPLICIT; + + if (mmrand(&tinfo->rnd, 1, 100) < GV(TRANSACTION_IMPLICIT)) { + iso_level = ISOLATION_SNAPSHOT; + iso_config = "isolation=snapshot"; + + /* Occasionally do reads at an isolation level lower than snapshot. */ + switch (mmrand(NULL, 1, 20)) { + case 1: + iso_level = ISOLATION_READ_COMMITTED; /* 5% */ + iso_config = "isolation=read-committed"; + break; + case 2: + iso_level = ISOLATION_READ_UNCOMMITTED; /* 5% */ + iso_config = "isolation=read-uncommitted"; + break; + } - /* Occasionally do reads at an isolation level lower than snapshot. */ - switch (mmrand(NULL, 1, 20)) { - case 1: - iso_level = ISOLATION_READ_COMMITTED; /* 5% */ - iso_config = "isolation=read-committed"; - break; - case 2: - iso_level = ISOLATION_READ_UNCOMMITTED; /* 5% */ - iso_config = "isolation=read-uncommitted"; - break; + begin_transaction(tinfo, iso_config); + intxn = true; } - - begin_transaction(tinfo, iso_config); - intxn = true; } /* - * Select an operation: all updates must be in snapshot isolation, modify must be in an - * explicit transaction. + * Select an operation: updates cannot happen at lower isolation levels and modify must be + * in an explicit transaction. */ op = READ; - if (iso_level == ISOLATION_SNAPSHOT) { + if (iso_level == ISOLATION_IMPLICIT || iso_level == ISOLATION_SNAPSHOT) { i = mmrand(&tinfo->rnd, 1, 100); - if (i < g.c_delete_pct && tinfo->ops > truncate_op) { + if (TV(OPS_TRUNCATE) && i < TV(OPS_PCT_DELETE) && tinfo->ops > truncate_op) { op = TRUNCATE; /* Pick the next truncate operation. */ truncate_op += mmrand(&tinfo->rnd, 20000, 100000); - } else if (i < g.c_delete_pct) + } else if (i < TV(OPS_PCT_DELETE)) op = REMOVE; - else if (i < g.c_delete_pct + g.c_insert_pct) + else if (i < TV(OPS_PCT_DELETE) + TV(OPS_PCT_INSERT)) op = INSERT; - else if (intxn && i < g.c_delete_pct + g.c_insert_pct + g.c_modify_pct) + else if (intxn && i < TV(OPS_PCT_DELETE) + TV(OPS_PCT_INSERT) + TV(OPS_PCT_MODIFY)) op = MODIFY; - else if (i < g.c_delete_pct + g.c_insert_pct + g.c_modify_pct + g.c_write_pct) + else if (i < + TV(OPS_PCT_DELETE) + TV(OPS_PCT_INSERT) + TV(OPS_PCT_MODIFY) + TV(OPS_PCT_WRITE)) op = UPDATE; } @@ -821,7 +813,7 @@ ops(void *arg) * Select a row. Column-store extends the object, explicitly read the maximum row count and * then use a local variable so the value won't change inside the loop. */ - max_rows = (volatile uint64_t)g.rows; + WT_ORDERED_READ(max_rows, table->rows_current); tinfo->keyno = mmrand(&tinfo->rnd, 1, (u_int)max_rows); /* @@ -832,7 +824,7 @@ ops(void *arg) positioned = false; if (op != READ && mmrand(&tinfo->rnd, 1, 5) == 1) { ++tinfo->search; - ret = read_row(tinfo, cursor); + ret = read_row(tinfo); if (ret == 0) { positioned = true; SNAP_TRACK(tinfo, READ); @@ -841,17 +833,18 @@ ops(void *arg) } /* - * Optionally reserve a row, it's an update so it requires snapshot isolation. Reserving a - * row before a read isn't all that sensible, but not unexpected, either. + * If we're in a transaction, optionally reserve a row: it's an update so cannot be done at + * lower isolation levels. Reserving a row in an implicit transaction will work, but doesn't + * make sense. Reserving a row before a read isn't sensible either, but it's not unexpected. */ if (intxn && iso_level == ISOLATION_SNAPSHOT && mmrand(&tinfo->rnd, 0, 20) == 1) { - switch (g.type) { + switch (table->type) { case ROW: - ret = row_reserve(tinfo, cursor, positioned); + ret = row_reserve(tinfo, positioned); break; case FIX: case VAR: - ret = col_reserve(tinfo, cursor, positioned); + ret = col_reserve(tinfo, positioned); break; } if (ret == 0) { @@ -864,9 +857,9 @@ ops(void *arg) /* Perform the operation. */ switch (op) { case INSERT: - switch (g.type) { + switch (table->type) { case ROW: - ret = row_insert(tinfo, cursor, positioned); + ret = row_insert(tinfo, positioned); break; case FIX: case VAR: @@ -874,10 +867,11 @@ ops(void *arg) * We can only append so many new records, once we reach that limit, update a record * instead of inserting. */ - if (tinfo->insert_list_cnt >= WT_ELEMENTS(tinfo->insert_list)) + cip = &tinfo->col_insert[table->id - 1]; + if (cip->insert_list_cnt >= WT_ELEMENTS(cip->insert_list)) goto update_instead_of_chosen_op; - ret = col_insert(tinfo, cursor); + ret = col_insert(tinfo); break; } @@ -891,12 +885,16 @@ ops(void *arg) break; case MODIFY: ++tinfo->update; - switch (g.type) { + switch (table->type) { + case FIX: + testutil_die( + 0, "%s", "fixed-length column-store does not support modify operations"); + /* NOTREACHED */ case ROW: - ret = row_modify(tinfo, cursor, positioned); + ret = row_modify(tinfo, positioned); break; case VAR: - ret = col_modify(tinfo, cursor, positioned); + ret = col_modify(tinfo, positioned); break; } if (ret == 0) { @@ -907,7 +905,7 @@ ops(void *arg) break; case READ: ++tinfo->search; - ret = read_row(tinfo, cursor); + ret = read_row(tinfo); if (ret == 0) { positioned = true; SNAP_TRACK(tinfo, READ); @@ -915,13 +913,13 @@ ops(void *arg) READ_OP_FAILED(true); break; case REMOVE: - switch (g.type) { + switch (table->type) { case ROW: - ret = row_remove(tinfo, cursor, positioned); + ret = row_remove(tinfo, positioned); break; case FIX: case VAR: - ret = col_remove(tinfo, cursor, positioned); + ret = col_remove(tinfo, positioned); break; } if (ret == 0) { @@ -936,11 +934,11 @@ ops(void *arg) break; case TRUNCATE: /* - * A maximum of 2 truncation operations at a time, more than that can lead to serious - * thrashing. + * A maximum of 2 truncation operations at a time in an object, more than that can lead + * to serious thrashing. */ - if (__wt_atomic_addv64(&g.truncate_cnt, 1) > 2) { - (void)__wt_atomic_subv64(&g.truncate_cnt, 1); + if (__wt_atomic_addv64(&table->truncate_cnt, 1) > 2) { + (void)__wt_atomic_subv64(&table->truncate_cnt, 1); goto update_instead_of_chosen_op; } @@ -960,7 +958,7 @@ ops(void *arg) range = max_rows < 20 ? 0 : mmrand(&tinfo->rnd, 0, (u_int)max_rows / 20); tinfo->last = tinfo->keyno; if (greater_than) { - if (g.c_reverse) { + if (TV(BTREE_REVERSE)) { if (tinfo->keyno <= range) tinfo->last = 0; else @@ -971,7 +969,7 @@ ops(void *arg) tinfo->last = 0; } } else { - if (g.c_reverse) { + if (TV(BTREE_REVERSE)) { tinfo->keyno += range; if (tinfo->keyno > max_rows) tinfo->keyno = 0; @@ -982,16 +980,16 @@ ops(void *arg) tinfo->keyno -= range; } } - switch (g.type) { + switch (table->type) { case ROW: - ret = row_truncate(tinfo, cursor); + ret = row_truncate(tinfo); break; case FIX: case VAR: - ret = col_truncate(tinfo, cursor); + ret = col_truncate(tinfo); break; } - (void)__wt_atomic_subv64(&g.truncate_cnt, 1); + (void)__wt_atomic_subv64(&table->truncate_cnt, 1); /* Truncate never leaves the cursor positioned. */ positioned = false; @@ -1004,13 +1002,13 @@ ops(void *arg) case UPDATE: update_instead_of_chosen_op: ++tinfo->update; - switch (g.type) { + switch (table->type) { case ROW: - ret = row_update(tinfo, cursor, positioned); + ret = row_update(tinfo, positioned); break; case FIX: case VAR: - ret = col_update(tinfo, cursor, positioned); + ret = col_update(tinfo, positioned); break; } if (ret == 0) { @@ -1022,8 +1020,8 @@ update_instead_of_chosen_op: } /* If we have pending inserts, try and update the total rows. */ - if (tinfo->insert_list_cnt > 0) - col_insert_resolve(tinfo); + if (g.column_store_config) + tables_apply(col_insert_resolve, tinfo); /* * The cursor is positioned if we did any operation other than insert, do a small number of @@ -1033,7 +1031,7 @@ update_instead_of_chosen_op: next = mmrand(&tinfo->rnd, 0, 1) == 1; j = mmrand(&tinfo->rnd, 1, 100); for (i = 0; i < j; ++i) { - if ((ret = nextprev(tinfo, cursor, next)) == 0) + if ((ret = nextprev(tinfo, next)) == 0) continue; READ_OP_FAILED(true); @@ -1042,7 +1040,7 @@ update_instead_of_chosen_op: } /* Reset the cursor: there is no reason to keep pages pinned. */ - testutil_check(cursor->reset(cursor)); + testutil_check(tinfo->cursor->reset(tinfo->cursor)); /* * No post-operation work is needed outside of a transaction. If in a transaction, add more @@ -1052,21 +1050,24 @@ update_instead_of_chosen_op: continue; /* - * Ending a transaction. If the transaction was configured for snapshot isolation, repeat - * the operations and confirm the results are unchanged. + * Ending a transaction. If an explicit transaction was configured for snapshot isolation, + * repeat the operations and confirm the results are unchanged. */ if (intxn && iso_level == ISOLATION_SNAPSHOT) { __wt_yield(); /* Encourage races */ - ret = snap_repeat_txn(cursor, tinfo); + ret = snap_repeat_txn(tinfo); testutil_assert(ret == 0 || ret == WT_ROLLBACK || ret == WT_CACHE_FULL); if (ret == WT_ROLLBACK || ret == WT_CACHE_FULL) goto rollback; } - /* If prepare configured, prepare the transaction 10% of the time. */ + /* + * If prepare configured, prepare the transaction 10% of the time. Note prepare requires a + * timestamped world, which means we're in a snapshot-isolation transaction by definition. + */ prepared = false; - if (g.c_prepare && mmrand(&tinfo->rnd, 1, 10) == 1) { + if (GV(OPS_PREPARE) && mmrand(&tinfo->rnd, 1, 10) == 1) { if ((ret = prepare_transaction(tinfo)) != 0) WRITE_OP_FAILED(false); @@ -1096,11 +1097,10 @@ rollback: intxn = false; } - if (session != NULL) { + if (session != NULL) testutil_check(session->close(session, NULL)); - tinfo->cursor = NULL; - tinfo->session = NULL; - } + tinfo->session = NULL; + memset(tinfo->cursors, 0, WT_MAX(ntables, 1) * sizeof(tinfo->cursors[0])); tinfo->state = TINFO_COMPLETE; return (WT_THREAD_RET_VALUE); @@ -1111,7 +1111,7 @@ rollback: * Read and verify a subset of the elements in a file. */ void -wts_read_scan(void) +wts_read_scan(TABLE *table, void *arg) { WT_CONNECTION *conn; WT_CURSOR *cursor; @@ -1119,14 +1119,15 @@ wts_read_scan(void) WT_ITEM key, value; WT_SESSION *session; uint64_t keyno; + uint32_t max_rows; - conn = g.wts_conn; + conn = (WT_CONNECTION *)arg; /* * We're not configuring transactions or read timestamps, if there's a diagnostic check, skip * the scan. */ - if (g.c_assert_read_timestamp) + if (GV(ASSERT_READ_TIMESTAMP)) return; /* Set up the default key/value buffers. */ @@ -1135,20 +1136,16 @@ wts_read_scan(void) /* Open a session and cursor pair. */ testutil_check(conn->open_session(conn, NULL, NULL, &session)); - /* - * open_cursor can return EBUSY if concurrent with a metadata operation, retry in that case. - */ - while ((ret = session->open_cursor(session, g.uri, NULL, NULL, &cursor)) == EBUSY) - __wt_yield(); - testutil_check(ret); + wiredtiger_open_cursor(session, table->uri, NULL, &cursor); /* Check a random subset of the records using the key. */ - for (keyno = 0; keyno < g.rows;) { + WT_ORDERED_READ(max_rows, table->rows_current); + for (keyno = 0; keyno < max_rows;) { keyno += mmrand(NULL, 1, 1000); - if (keyno > g.rows) - keyno = g.rows; + if (keyno > max_rows) + keyno = max_rows; - switch (ret = read_row_worker(cursor, NULL, keyno, &key, &value, false)) { + switch (ret = read_row_worker(NULL, table, cursor, keyno, &key, &value, false)) { case 0: case WT_NOTFOUND: case WT_ROLLBACK: @@ -1171,20 +1168,20 @@ wts_read_scan(void) * Read and verify a single element in a row- or column-store file. */ static int -read_row_worker( - WT_CURSOR *cursor, TINFO *tinfo, uint64_t keyno, WT_ITEM *key, WT_ITEM *value, bool sn) +read_row_worker(TINFO *tinfo, TABLE *table, WT_CURSOR *cursor, uint64_t keyno, WT_ITEM *key, + WT_ITEM *value, bool sn) { uint8_t bitfield; int exact, ret; /* Retrieve the key/value pair by key. */ - switch (g.type) { + switch (table->type) { case FIX: case VAR: cursor->set_key(cursor, keyno); break; case ROW: - key_gen(key, keyno); + key_gen(table, key, keyno); cursor->set_key(cursor, key); break; } @@ -1197,7 +1194,7 @@ read_row_worker( ret = read_op(cursor, SEARCH, NULL); switch (ret) { case 0: - if (g.type == FIX) { + if (table->type == FIX) { testutil_check(cursor->get_value(cursor, &bitfield)); *(uint8_t *)(value->data) = bitfield; value->size = 1; @@ -1211,7 +1208,7 @@ read_row_worker( * The WiredTiger cursor has lost its position though, so we return not-found, the cursor * movement can't continue. */ - if (g.type == FIX) { + if (table->type == FIX) { *(uint8_t *)(value->data) = 0; value->size = 1; } @@ -1222,7 +1219,7 @@ read_row_worker( /* Log the operation */ if (ret == 0) - switch (g.type) { + switch (table->type) { case FIX: if (tinfo == NULL && g.trace_all) trace_msg("read %" PRIu64 " {0x%02x}", keyno, ((char *)value->data)[0]); @@ -1247,11 +1244,11 @@ read_row_worker( * Read and verify a single element in a row- or column-store file. */ static int -read_row(TINFO *tinfo, WT_CURSOR *cursor) +read_row(TINFO *tinfo) { /* 25% of the time we call search-near. */ - return (read_row_worker( - cursor, tinfo, tinfo->keyno, tinfo->key, tinfo->value, mmrand(&tinfo->rnd, 0, 3) == 1)); + return (read_row_worker(tinfo, tinfo->table, tinfo->cursor, tinfo->keyno, tinfo->key, + tinfo->value, mmrand(&tinfo->rnd, 0, 3) == 1)); } /* @@ -1259,22 +1256,26 @@ read_row(TINFO *tinfo, WT_CURSOR *cursor) * Read and verify the next/prev element in a row- or column-store file. */ static int -nextprev(TINFO *tinfo, WT_CURSOR *cursor, bool next) +nextprev(TINFO *tinfo, bool next) { + TABLE *table; + WT_CURSOR *cursor; WT_DECL_RET; WT_ITEM key, value; - uint64_t keyno, keyno_prev; + uint64_t keyno; uint8_t bitfield; int cmp; const char *which; - bool incrementing, record_gaps; + bool incrementing; + table = tinfo->table; + cursor = tinfo->cursor; keyno = 0; which = next ? "next" : "prev"; switch (ret = read_op(cursor, next ? NEXT : PREV, NULL)) { case 0: - switch (g.type) { + switch (table->type) { case FIX: if ((ret = cursor->get_key(cursor, &keyno)) == 0 && (ret = cursor->get_value(cursor, &bitfield)) == 0) { @@ -1294,64 +1295,26 @@ nextprev(TINFO *tinfo, WT_CURSOR *cursor, bool next) if (ret != 0) testutil_die(ret, "nextprev: get_key/get_value"); - /* Check that keys are never returned out-of-order. */ - /* - * XXX WT-3889 LSM has a bug that prevents cursor order checks from working, skip the test - * for now. - */ - if (DATASOURCE("lsm")) - break; - /* - * Compare the returned key with the previously returned key, and assert the order is - * correct. If not deleting keys, and the rows aren't in the column-store insert name space, - * also assert we don't skip groups of records (that's a page-split bug symptom). Note a - * previous run that performed salvage might have corrupted a chunk of space such that - * records were removed. If this is a reopen of an existing database, assume salvage might - * have happened. + * Check that keys are never returned out-of-order by comparing the returned key with the + * previously returned key, and assert the order is correct. */ - record_gaps = g.c_delete_pct != 0 || g.reopen; - switch (g.type) { + switch (table->type) { case FIX: case VAR: - if (tinfo->keyno > g.c_rows || keyno > g.c_rows) - record_gaps = true; - if (!next) { - if (tinfo->keyno < keyno || (!record_gaps && keyno != tinfo->keyno - 1)) - goto order_error_col; - } else if (tinfo->keyno > keyno || (!record_gaps && keyno != tinfo->keyno + 1)) - goto order_error_col; - if (0) { -order_error_col: + if ((next && tinfo->keyno > keyno) || (!next && tinfo->keyno < keyno)) testutil_die( 0, "%s returned %" PRIu64 " then %" PRIu64, which, tinfo->keyno, keyno); - } - tinfo->keyno = keyno; break; case ROW: - incrementing = (next && !g.c_reverse) || (!next && g.c_reverse); + incrementing = (next && !TV(BTREE_REVERSE)) || (!next && TV(BTREE_REVERSE)); cmp = memcmp(tinfo->key->data, key.data, WT_MIN(tinfo->key->size, key.size)); if (incrementing) { if (cmp > 0 || (cmp == 0 && tinfo->key->size < key.size)) goto order_error_row; } else if (cmp < 0 || (cmp == 0 && tinfo->key->size > key.size)) goto order_error_row; - if (!record_gaps) { - /* - * Convert the keys to record numbers and then compare less-than-or-equal. (It's not - * less-than, row-store inserts new rows in-between rows by appending a new suffix - * to the row's key.) Keys are strings with terminating '/' values, so absent key - * corruption, we can simply do the underlying string conversion on the key string. - */ - keyno_prev = strtoul((char *)tinfo->key->data + g.prefix_len, NULL, 10); - keyno = strtoul((char *)key.data + g.prefix_len, NULL, 10); - if (incrementing) { - if (keyno_prev != keyno && keyno_prev + 1 != keyno) - goto order_error_row; - } else if (keyno_prev != keyno && keyno_prev - 1 != keyno) - goto order_error_row; - } if (0) { order_error_row: #ifdef HAVE_DIAGNOSTIC @@ -1366,13 +1329,12 @@ order_error_row: } break; case WT_NOTFOUND: - break; default: return (ret); } - if (g.trace_all && ret == 0) - switch (g.type) { + if (g.trace_all) + switch (table->type) { case FIX: trace_op(tinfo, "%s %" PRIu64 " {0x%02x}", which, keyno, ((char *)value.data)[0]); break; @@ -1393,12 +1355,15 @@ order_error_row: * Reserve a row in a row-store file. */ static int -row_reserve(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) +row_reserve(TINFO *tinfo, bool positioned) { + WT_CURSOR *cursor; WT_DECL_RET; + cursor = tinfo->cursor; + if (!positioned) { - key_gen(tinfo->key, tinfo->keyno); + key_gen(tinfo->table, tinfo->key, tinfo->keyno); cursor->set_key(cursor, tinfo->key); } @@ -1416,10 +1381,13 @@ row_reserve(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) * Reserve a row in a column-store file. */ static int -col_reserve(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) +col_reserve(TINFO *tinfo, bool positioned) { + WT_CURSOR *cursor; WT_DECL_RET; + cursor = tinfo->cursor; + if (!positioned) cursor->set_key(cursor, tinfo->keyno); @@ -1491,11 +1459,14 @@ modify(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) * Modify a row in a row-store file. */ static int -row_modify(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) +row_modify(TINFO *tinfo, bool positioned) { + WT_CURSOR *cursor; + + cursor = tinfo->cursor; if (!positioned) { - key_gen(tinfo->key, tinfo->keyno); + key_gen(tinfo->table, tinfo->key, tinfo->keyno); cursor->set_key(cursor, tinfo->key); } @@ -1512,8 +1483,12 @@ row_modify(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) * Modify a row in a column-store file. */ static int -col_modify(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) +col_modify(TINFO *tinfo, bool positioned) { + WT_CURSOR *cursor; + + cursor = tinfo->cursor; + if (!positioned) cursor->set_key(cursor, tinfo->keyno); @@ -1529,12 +1504,13 @@ col_modify(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) * Truncate rows in a row-store file. */ static int -row_truncate(TINFO *tinfo, WT_CURSOR *cursor) +row_truncate(TINFO *tinfo) { - WT_CURSOR *c2; + WT_CURSOR *cursor, *c2; WT_DECL_RET; WT_SESSION *session; + cursor = tinfo->cursor; session = cursor->session; /* @@ -1544,28 +1520,26 @@ row_truncate(TINFO *tinfo, WT_CURSOR *cursor) c2 = NULL; if (tinfo->keyno == 0) { - key_gen(tinfo->key, tinfo->last); + key_gen(tinfo->table, tinfo->key, tinfo->last); cursor->set_key(cursor, tinfo->key); - ret = session->truncate(session, NULL, NULL, cursor, NULL); + WT_RET(session->truncate(session, NULL, NULL, cursor, NULL)); } else if (tinfo->last == 0) { - key_gen(tinfo->key, tinfo->keyno); + key_gen(tinfo->table, tinfo->key, tinfo->keyno); cursor->set_key(cursor, tinfo->key); - ret = session->truncate(session, NULL, cursor, NULL, NULL); + WT_RET(session->truncate(session, NULL, cursor, NULL, NULL)); } else { - key_gen(tinfo->key, tinfo->keyno); + key_gen(tinfo->table, tinfo->key, tinfo->keyno); cursor->set_key(cursor, tinfo->key); - testutil_check(session->open_cursor(session, g.uri, NULL, NULL, &c2)); - key_gen(tinfo->lastkey, tinfo->last); + testutil_check(session->open_cursor(session, tinfo->table->uri, NULL, NULL, &c2)); + key_gen(tinfo->table, tinfo->lastkey, tinfo->last); cursor->set_key(c2, tinfo->lastkey); ret = session->truncate(session, NULL, cursor, c2, NULL); testutil_check(c2->close(c2)); + WT_RET(ret); } - if (ret != 0) - return (ret); - trace_op(tinfo, "truncate %" PRIu64 ", %" PRIu64, "truncate", tinfo->keyno, tinfo->last); return (0); @@ -1576,12 +1550,13 @@ row_truncate(TINFO *tinfo, WT_CURSOR *cursor) * Truncate rows in a column-store file. */ static int -col_truncate(TINFO *tinfo, WT_CURSOR *cursor) +col_truncate(TINFO *tinfo) { - WT_CURSOR *c2; + WT_CURSOR *cursor, *c2; WT_DECL_RET; WT_SESSION *session; + cursor = tinfo->cursor; session = cursor->session; /* @@ -1599,7 +1574,7 @@ col_truncate(TINFO *tinfo, WT_CURSOR *cursor) } else { cursor->set_key(cursor, tinfo->keyno); - testutil_check(session->open_cursor(session, g.uri, NULL, NULL, &c2)); + testutil_check(session->open_cursor(session, tinfo->table->uri, NULL, NULL, &c2)); cursor->set_key(c2, tinfo->last); ret = session->truncate(session, NULL, cursor, c2, NULL); @@ -1618,15 +1593,18 @@ col_truncate(TINFO *tinfo, WT_CURSOR *cursor) * Update a row in a row-store file. */ static int -row_update(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) +row_update(TINFO *tinfo, bool positioned) { + WT_CURSOR *cursor; WT_DECL_RET; + cursor = tinfo->cursor; + if (!positioned) { - key_gen(tinfo->key, tinfo->keyno); + key_gen(tinfo->table, tinfo->key, tinfo->keyno); cursor->set_key(cursor, tinfo->key); } - val_gen(&tinfo->rnd, tinfo->value, tinfo->keyno); + val_gen(tinfo->table, &tinfo->rnd, tinfo->value, tinfo->keyno); cursor->set_value(cursor, tinfo->value); if ((ret = cursor->update(cursor)) != 0) @@ -1643,14 +1621,19 @@ row_update(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) * Update a row in a column-store file. */ static int -col_update(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) +col_update(TINFO *tinfo, bool positioned) { + TABLE *table; + WT_CURSOR *cursor; WT_DECL_RET; + table = tinfo->table; + cursor = tinfo->cursor; + if (!positioned) cursor->set_key(cursor, tinfo->keyno); - val_gen(&tinfo->rnd, tinfo->value, tinfo->keyno); - if (g.type == FIX) + val_gen(table, &tinfo->rnd, tinfo->value, tinfo->keyno); + if (table->type == FIX) cursor->set_value(cursor, *(uint8_t *)tinfo->value->data); else cursor->set_value(cursor, tinfo->value); @@ -1658,7 +1641,7 @@ col_update(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) if ((ret = cursor->update(cursor)) != 0) return (ret); - if (g.type == FIX) + if (table->type == FIX) trace_op(tinfo, "update %" PRIu64 " {0x%02" PRIx8 "}", tinfo->keyno, ((uint8_t *)tinfo->value->data)[0]); else @@ -1672,19 +1655,22 @@ col_update(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) * Insert a row in a row-store file. */ static int -row_insert(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) +row_insert(TINFO *tinfo, bool positioned) { + WT_CURSOR *cursor; WT_DECL_RET; + cursor = tinfo->cursor; + /* * If we positioned the cursor already, it's a test of an update using the insert method. * Otherwise, generate a unique key and insert. */ if (!positioned) { - key_gen_insert(&tinfo->rnd, tinfo->key, tinfo->keyno); + key_gen_insert(tinfo->table, &tinfo->rnd, tinfo->key, tinfo->keyno); cursor->set_key(cursor, tinfo->key); } - val_gen(&tinfo->rnd, tinfo->value, tinfo->keyno); + val_gen(tinfo->table, &tinfo->rnd, tinfo->value, tinfo->keyno); cursor->set_value(cursor, tinfo->value); if ((ret = cursor->insert(cursor)) != 0) @@ -1702,11 +1688,18 @@ row_insert(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) * Resolve newly inserted records. */ static void -col_insert_resolve(TINFO *tinfo) +col_insert_resolve(TABLE *table, void *arg) { - uint64_t v, *p; + struct col_insert *cip; + TINFO *tinfo; + uint32_t max_rows, *p; u_int i; + tinfo = arg; + cip = &tinfo->col_insert[table->id - 1]; + if (cip->insert_list_cnt == 0) + return; + /* * We don't want to ignore column-store records we insert, which requires we update the "last * row" so other threads consider them. Threads allocating record numbers can race with other @@ -1722,17 +1715,17 @@ col_insert_resolve(TINFO *tinfo) * Process the existing records and advance the last row count until we can't go further. */ do { - WT_ORDERED_READ(v, g.rows); - for (i = 0, p = tinfo->insert_list; i < WT_ELEMENTS(tinfo->insert_list); ++i, ++p) { - if (*p == v + 1) { - testutil_assert(__wt_atomic_casv64(&g.rows, v, v + 1)); + WT_ORDERED_READ(max_rows, table->rows_current); + for (i = 0, p = cip->insert_list; i < WT_ELEMENTS(cip->insert_list); ++i, ++p) { + if (*p == max_rows + 1) { + testutil_assert(__wt_atomic_casv32(&table->rows_current, max_rows, max_rows + 1)); *p = 0; - --tinfo->insert_list_cnt; + --cip->insert_list_cnt; break; } - testutil_assert(*p == 0 || *p > v); + testutil_assert(*p == 0 || *p > max_rows); } - } while (tinfo->insert_list_cnt > 0 && i < WT_ELEMENTS(tinfo->insert_list)); + } while (cip->insert_list_cnt > 0 && i < WT_ELEMENTS(cip->insert_list)); } /* @@ -1742,16 +1735,18 @@ col_insert_resolve(TINFO *tinfo) static void col_insert_add(TINFO *tinfo) { + struct col_insert *cip; u_int i; - /* Add the inserted record to the array. */ - for (i = 0; i < WT_ELEMENTS(tinfo->insert_list); ++i) - if (tinfo->insert_list[i] == 0) { - tinfo->insert_list[i] = tinfo->keyno; - ++tinfo->insert_list_cnt; + /* Add the inserted record to the insert array. */ + cip = &tinfo->col_insert[tinfo->table->id - 1]; + for (i = 0; i < WT_ELEMENTS(cip->insert_list); ++i) + if (cip->insert_list[i] == 0) { + cip->insert_list[i] = (uint32_t)tinfo->keyno; + ++cip->insert_list_cnt; break; } - testutil_assert(i < WT_ELEMENTS(tinfo->insert_list)); + testutil_assert(i < WT_ELEMENTS(cip->insert_list)); } /* @@ -1759,12 +1754,19 @@ col_insert_add(TINFO *tinfo) * Insert an element in a column-store file. */ static int -col_insert(TINFO *tinfo, WT_CURSOR *cursor) +col_insert(TINFO *tinfo) { + TABLE *table; + WT_CURSOR *cursor; WT_DECL_RET; + uint64_t max_rows; - val_gen(&tinfo->rnd, tinfo->value, g.rows + 1); - if (g.type == FIX) + table = tinfo->table; + cursor = tinfo->cursor; + + WT_ORDERED_READ(max_rows, table->rows_current); + val_gen(table, &tinfo->rnd, tinfo->value, max_rows + 1); + if (table->type == FIX) cursor->set_value(cursor, *(uint8_t *)tinfo->value->data); else cursor->set_value(cursor, tinfo->value); @@ -1777,7 +1779,7 @@ col_insert(TINFO *tinfo, WT_CURSOR *cursor) col_insert_add(tinfo); /* Extend the object. */ - if (g.type == FIX) + if (table->type == FIX) trace_op(tinfo, "insert %" PRIu64 " {0x%02" PRIx8 "}", tinfo->keyno, ((uint8_t *)tinfo->value->data)[0]); else @@ -1791,12 +1793,15 @@ col_insert(TINFO *tinfo, WT_CURSOR *cursor) * Remove an row from a row-store file. */ static int -row_remove(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) +row_remove(TINFO *tinfo, bool positioned) { + WT_CURSOR *cursor; WT_DECL_RET; + cursor = tinfo->cursor; + if (!positioned) { - key_gen(tinfo->key, tinfo->keyno); + key_gen(tinfo->table, tinfo->key, tinfo->keyno); cursor->set_key(cursor, tinfo->key); } @@ -1817,10 +1822,13 @@ row_remove(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) * Remove a row from a column-store file. */ static int -col_remove(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) +col_remove(TINFO *tinfo, bool positioned) { + WT_CURSOR *cursor; WT_DECL_RET; + cursor = tinfo->cursor; + if (!positioned) cursor->set_key(cursor, tinfo->keyno); diff --git a/src/third_party/wiredtiger/test/format/random.c b/src/third_party/wiredtiger/test/format/random.c index f5726ff8ea3..fba5e062079 100644 --- a/src/third_party/wiredtiger/test/format/random.c +++ b/src/third_party/wiredtiger/test/format/random.c @@ -35,6 +35,7 @@ WT_THREAD_RET random_kv(void *arg) { + TABLE *table; WT_CONNECTION *conn; WT_CURSOR *cursor; WT_DECL_RET; @@ -47,13 +48,19 @@ random_kv(void *arg) (void)(arg); /* Unused parameter */ - conn = g.wts_conn; - - /* Random cursor ops are only supported on row-store. */ - if (g.type != ROW) + /* Random cursor ops are only supported on row-store, make sure there's a row-store table. */ + if (ntables == 0 && tables[0]->type != ROW) return (WT_THREAD_RET_VALUE); + else { + for (i = 1; i < ntables; ++i) + if (tables[i]->type == ROW) + break; + if (i == ntables) + return (WT_THREAD_RET_VALUE); + } /* Open a session. */ + conn = g.wts_conn; testutil_check(conn->open_session(conn, NULL, NULL, &session)); for (simple = false;;) { @@ -61,12 +68,9 @@ random_kv(void *arg) config = simple ? "next_random=true" : "next_random=true,next_random_sample_size=37"; simple = !simple; - /* - * open_cursor can return EBUSY if concurrent with a metadata operation, retry in that case. - */ - while ((ret = session->open_cursor(session, g.uri, NULL, config, &cursor)) == EBUSY) - __wt_yield(); - testutil_check(ret); + /* Select a table and open a cursor. */ + table = table_select_type(ROW); + wiredtiger_open_cursor(session, table->uri, config, &cursor); /* This is just a smoke-test, get some key/value pairs. */ for (i = mmrand(NULL, 0, 1000); i > 0; --i) { diff --git a/src/third_party/wiredtiger/test/format/salvage.c b/src/third_party/wiredtiger/test/format/salvage.c index a383c121862..26589e18a3f 100644 --- a/src/third_party/wiredtiger/test/format/salvage.c +++ b/src/third_party/wiredtiger/test/format/salvage.c @@ -29,124 +29,126 @@ #include "format.h" /* + * uri_path -- + * Return the path to an object file, and optionally, the object name. + */ +static void +uri_path(TABLE *table, char **object_namep, char *buf, size_t len) +{ + char *p; + + /* + * It's a little tricky: if the data source is a file, we're looking for the table URI, if the + * data source is a table, we're looking for the table URI with a trailing ".wt". + */ + p = strchr(table->uri, ':'); + testutil_assert(p != NULL); + ++p; + + testutil_check(__wt_snprintf(buf, len, "%s/%s", g.home, p)); + if (object_namep != NULL) + *object_namep = strrchr(buf, '/') + 1; + if (!access(buf, F_OK)) + return; + testutil_check(__wt_snprintf(buf, len, "%s/%s.wt", g.home, p)); + if (object_namep != NULL) + *object_namep = strrchr(buf, '/') + 1; + if (!access(buf, F_OK)) + return; + testutil_die(0, "%s: unable to find file for salvage", table->uri); +} + +/* * corrupt -- * Corrupt the file in a random way. */ -static int -corrupt(void) +static void +corrupt(TABLE *table) { struct stat sb; FILE *fp; wt_off_t offset; size_t len, nw; - int fd, ret; - char copycmd[2 * 1024], path[1024]; + int fd; + char buf[MAX_FORMAT_PATH * 2], *object_name, path[MAX_FORMAT_PATH]; const char *smash; - /* - * If it's a single Btree file (not LSM), open the file, and corrupt roughly 2% of the file at a - * random spot, including the beginning of the file and overlapping the end. - * - * It's a little tricky: if the data source is a file, we're looking for "wt", if the data - * source is a table, we're looking for "wt.wt". - */ - testutil_check(__wt_snprintf(path, sizeof(path), "%s/%s", g.home, WT_NAME)); - if ((fd = open(path, O_RDWR)) != -1) { - testutil_check(__wt_snprintf(copycmd, sizeof(copycmd), - "cp %s/%s %s/SALVAGE.copy/%s.corrupted", g.home, WT_NAME, g.home, WT_NAME)); - goto found; - } - testutil_check(__wt_snprintf(path, sizeof(path), "%s/%s.wt", g.home, WT_NAME)); - if ((fd = open(path, O_RDWR)) != -1) { - testutil_check(__wt_snprintf(copycmd, sizeof(copycmd), - "cp %s/%s.wt %s/SALVAGE.copy/%s.wt.corrupted", g.home, WT_NAME, g.home, WT_NAME)); - goto found; - } - return (0); + uri_path(table, &object_name, path, sizeof(path)); -found: - if (fstat(fd, &sb) == -1) - testutil_die(errno, "salvage-corrupt: fstat"); + fd = open(path, O_RDWR); + testutil_assert(fd != -1); - offset = mmrand(NULL, 0, (u_int)sb.st_size); - len = (size_t)(20 + (sb.st_size / 100) * 2); - testutil_check(__wt_snprintf(path, sizeof(path), "%s/SALVAGE.corrupt", g.home)); - if ((fp = fopen(path, "w")) == NULL) - testutil_die(errno, "salvage-corrupt: open: %s", path); + /* + * Corrupt a chunk of the file at a random spot, including the first bytes of the file and + * possibly overlapping the end. The length of the corruption is roughly 2% of the file, not + * exceeding a megabyte (so we aren't just corrupting the whole file). + */ + testutil_check(fstat(fd, &sb)); + offset = mmrand(NULL, 0, (u_int)sb.st_size - 1024); + len = (size_t)(sb.st_size * 2) / 100; + len += 4 * 1024; + len = WT_MIN(len, WT_MEGABYTE); + + /* Log the corruption offset and length. */ + testutil_check(__wt_snprintf(buf, sizeof(buf), "%s/SALVAGE.corrupt", g.home)); + testutil_assert((fp = fopen(buf, "w")) != NULL); (void)fprintf(fp, "salvage-corrupt: offset %" PRIuMAX ", length %" WT_SIZET_FMT "\n", (uintmax_t)offset, len); fclose_and_clear(&fp); - if (lseek(fd, offset, SEEK_SET) == -1) - testutil_die(errno, "salvage-corrupt: lseek"); - + testutil_assert(lseek(fd, offset, SEEK_SET) != -1); smash = "!!! memory corrupted by format to test salvage "; for (; len > 0; len -= nw) { nw = (size_t)(len > strlen(smash) ? strlen(smash) : len); - if (write(fd, smash, nw) == -1) - testutil_die(errno, "salvage-corrupt: write"); + testutil_assert(write(fd, smash, nw) != -1); } - if (close(fd) == -1) - testutil_die(errno, "salvage-corrupt: close"); + testutil_check(close(fd)); - /* - * Save a copy of the corrupted file so we can replay the salvage step as necessary. - */ - if ((ret = system(copycmd)) != 0) - testutil_die(ret, "salvage corrupt copy step failed"); - - return (1); + /* Save a copy of the corrupted file so we can replay the salvage step as necessary. */ + testutil_check(__wt_snprintf( + buf, sizeof(buf), "cp %s %s/SALVAGE.copy/%s.corrupted", path, g.home, object_name)); + testutil_check(system(buf)); } -/* - * Salvage command, save the interesting files so we can replay the salvage command as necessary. - * - * Redirect the "cd" command to /dev/null so chatty cd implementations don't add the new working - * directory to our output. - */ -#define SALVAGE_COPY_CMD \ - "cd %s > /dev/null && " \ - "rm -rf SALVAGE.copy && mkdir SALVAGE.copy && " \ - "cp WiredTiger* wt* SALVAGE.copy/" +/* Salvage command, save the interesting files so we can replay the salvage command as necessary. */ +#define SALVAGE_COPY_CMD \ + "rm -rf %s/SALVAGE.copy && mkdir %s/SALVAGE.copy && cp %s/WiredTiger* %s %s/SALVAGE.copy/" /* * wts_salvage -- * Salvage testing. */ void -wts_salvage(void) +wts_salvage(TABLE *table, void *arg) { WT_CONNECTION *conn; - WT_DECL_RET; WT_SESSION *session; - size_t len; - char *cmd; + char buf[MAX_FORMAT_PATH * 5], path[MAX_FORMAT_PATH]; - if (g.c_salvage == 0) - return; + (void)arg; /* unused argument */ - track("salvage", 0ULL, NULL); + if (GV(OPS_SALVAGE) == 0 || DATASOURCE(table, "lsm")) + return; /* Save a copy of the interesting files so we can replay the salvage step as necessary. */ - len = strlen(g.home) + strlen(SALVAGE_COPY_CMD) + 1; - cmd = dmalloc(len); - testutil_check(__wt_snprintf(cmd, len, SALVAGE_COPY_CMD, g.home)); - if ((ret = system(cmd)) != 0) - testutil_die(ret, "salvage copy (\"%s\"), failed", cmd); - free(cmd); + uri_path(table, NULL, path, sizeof(path)); + testutil_check( + __wt_snprintf(buf, sizeof(buf), SALVAGE_COPY_CMD, g.home, g.home, g.home, path, g.home)); + testutil_check(system(buf)); /* Salvage, then verify. */ wts_open(g.home, &conn, &session, true); - testutil_check(session->salvage(session, g.uri, "force=true")); - wts_verify(conn, "post-salvage verify"); + session->app_private = table->track_prefix; + testutil_check(session->salvage(session, table->uri, "force=true")); + wts_verify(table, conn); wts_close(&conn, &session); /* Corrupt the file randomly, salvage, then verify. */ - if (corrupt()) { - wts_open(g.home, &conn, &session, false); - testutil_check(session->salvage(session, g.uri, "force=true")); - wts_verify(conn, "post-corrupt-salvage verify"); - wts_close(&conn, &session); - } + corrupt(table); + wts_open(g.home, &conn, &session, false); + testutil_check(session->salvage(session, table->uri, "force=true")); + wts_verify(table, conn); + + wts_close(&conn, &session); } diff --git a/src/third_party/wiredtiger/test/format/smoke.sh b/src/third_party/wiredtiger/test/format/smoke.sh index 151d1be37b3..af63df24a0a 100755 --- a/src/third_party/wiredtiger/test/format/smoke.sh +++ b/src/third_party/wiredtiger/test/format/smoke.sh @@ -3,14 +3,16 @@ set -e # Smoke-test format as part of running "make check". -args="-1 -c . " -args="$args btree.compression=none " +args="-c . " +args="$args btree.compression=off " args="$args cache.minimum=40 " -args="$args logging_compression=none" -args="$args runs.ops=500000 " +args="$args logging_compression=off" args="$args runs.rows=100000 " args="$args runs.source=table " -args="$args runs.threads=4 " +args="$args runs.tables=3 " +args="$args runs.threads=6 " +args="$args runs.timer=1 " +args="$args transaction.timestamps=1 " # Temporarily disable LSM and FLCS. # $TEST_WRAPPER ./t $args runs.type=fix diff --git a/src/third_party/wiredtiger/test/format/snap.c b/src/third_party/wiredtiger/test/format/snap.c index e7709754523..298b82ec9a5 100644 --- a/src/third_party/wiredtiger/test/format/snap.c +++ b/src/third_party/wiredtiger/test/format/snap.c @@ -45,7 +45,7 @@ snap_init(TINFO *tinfo) * we can the secondary snap list to see the state of keys/values seen and updated at the time * of the rollback. */ - if (g.c_txn_timestamps) { + if (g.transaction_timestamps_config) { tinfo->s = &tinfo->snap_states[1]; tinfo->snap_list = dcalloc(SNAP_LIST_SIZE, sizeof(SNAP_OPS)); tinfo->snap_end = &tinfo->snap_list[SNAP_LIST_SIZE]; @@ -77,7 +77,7 @@ snap_teardown(TINFO *tinfo) } /* - * snap_clear -- + * snap_clear_one -- * Clear a single snap entry. */ static void @@ -110,7 +110,7 @@ snap_op_init(TINFO *tinfo, uint64_t read_ts, bool repeatable_reads) ++tinfo->opid; - if (g.c_txn_timestamps) { + if (g.transaction_timestamps_config) { /* * If the stable timestamp has changed and we've advanced beyond it, preserve the current * snapshot history up to this point, we'll use it verify rollback_to_stable. Switch our @@ -150,13 +150,14 @@ snap_track(TINFO *tinfo, thread_op op) snap = tinfo->snap_current; snap->op = op; snap->opid = tinfo->opid; + snap->id = tinfo->table->id; snap->keyno = tinfo->keyno; snap->ts = WT_TS_NONE; snap->repeatable = false; snap->last = op == TRUNCATE ? tinfo->last : 0; snap->ksize = snap->vsize = 0; - if (op == INSERT && g.type == ROW) { + if (op == INSERT && tinfo->table->type == ROW) { ip = tinfo->key; if (snap->kmemsize < ip->size) { snap->kdata = drealloc(snap->kdata, ip->size); @@ -196,11 +197,11 @@ snap_track(TINFO *tinfo, thread_op op) * Display a single data/size pair, with a tag. */ static void -print_item_data(const char *tag, const uint8_t *data, size_t size) +print_item_data(TABLE *table, const char *tag, const uint8_t *data, size_t size) { WT_ITEM tmp; - if (g.type == FIX) { + if (table->type == FIX) { fprintf(stderr, "%s {0x%02x}\n", tag, data[0]); return; } @@ -216,8 +217,10 @@ print_item_data(const char *tag, const uint8_t *data, size_t size) * Repeat a read and verify the contents. */ static int -snap_verify(WT_CURSOR *cursor, TINFO *tinfo, SNAP_OPS *snap) +snap_verify(TINFO *tinfo, SNAP_OPS *snap) { + TABLE *table; + WT_CURSOR *cursor; WT_DECL_RET; WT_ITEM *key, *value; uint64_t keyno; @@ -225,26 +228,28 @@ snap_verify(WT_CURSOR *cursor, TINFO *tinfo, SNAP_OPS *snap) testutil_assert(snap->op != TRUNCATE); + table = tables[ntables == 0 ? 0 : snap->id]; + cursor = table_cursor(tinfo, snap->id); + keyno = snap->keyno; key = tinfo->key; value = tinfo->value; - keyno = snap->keyno; /* * Retrieve the key/value pair by key. Row-store inserts have a unique generated key we saved, * else generate the key from the key number. */ - if (snap->op == INSERT && g.type == ROW) { + if (snap->op == INSERT && table->type == ROW) { key->data = snap->kdata; key->size = snap->ksize; cursor->set_key(cursor, key); } else { - switch (g.type) { + switch (table->type) { case FIX: case VAR: cursor->set_key(cursor, keyno); break; case ROW: - key_gen(key, keyno); + key_gen(table, key, keyno); cursor->set_key(cursor, key); break; } @@ -252,7 +257,7 @@ snap_verify(WT_CURSOR *cursor, TINFO *tinfo, SNAP_OPS *snap) switch (ret = read_op(cursor, SEARCH, NULL)) { case 0: - if (g.type == FIX) { + if (table->type == FIX) { testutil_check(cursor->get_value(cursor, &bitfield)); *(uint8_t *)(value->data) = bitfield; value->size = 1; @@ -279,15 +284,22 @@ snap_verify(WT_CURSOR *cursor, TINFO *tinfo, SNAP_OPS *snap) * In fixed length stores, zero values at the end of the key space are returned as not-found, * and not-found row reads are saved as zero values. Map back-and-forth for simplicity. */ - if (g.type == FIX) { + if (table->type == FIX) { if (ret == WT_NOTFOUND && snap->vsize == 1 && *(uint8_t *)snap->vdata == 0) return (0); if (snap->op == REMOVE && value->size == 1 && *(uint8_t *)value->data == 0) return (0); } - /* Things went pear-shaped. */ - switch (g.type) { + /* + * Things went pear-shaped. + * + * Dump the WiredTiger handle ID, it's useful in selecting trace records from the log. We have + * an open cursor on the handle, so while this is pretty ugly, I don't think it's unsafe. + */ + fprintf(stderr, "%s: WiredTiger trace ID: %u\n", table->uri, + (u_int)((WT_BTREE *)((WT_CURSOR_BTREE *)cursor)->dhandle->handle)->id); + switch (table->type) { case FIX: fprintf(stderr, "snapshot-isolation: %" PRIu64 " search: expected {0x%02x}, found {0x%02x}\n", keyno, @@ -301,11 +313,11 @@ snap_verify(WT_CURSOR *cursor, TINFO *tinfo, SNAP_OPS *snap) if (snap->op == REMOVE) fprintf(stderr, "expected {deleted}\n"); else - print_item_data("expected", snap->vdata, snap->vsize); + print_item_data(table, "expected", snap->vdata, snap->vsize); if (ret == WT_NOTFOUND) fprintf(stderr, " found {deleted}\n"); else - print_item_data(" found", value->data, value->size); + print_item_data(table, " found", value->data, value->size); break; case VAR: fprintf(stderr, "snapshot-isolation %" PRIu64 " search mismatch\n", keyno); @@ -313,11 +325,11 @@ snap_verify(WT_CURSOR *cursor, TINFO *tinfo, SNAP_OPS *snap) if (snap->op == REMOVE) fprintf(stderr, "expected {deleted}\n"); else - print_item_data("expected", snap->vdata, snap->vsize); + print_item_data(table, "expected", snap->vdata, snap->vsize); if (ret == WT_NOTFOUND) fprintf(stderr, " found {deleted}\n"); else - print_item_data(" found", value->data, value->size); + print_item_data(table, " found", value->data, value->size); break; } @@ -344,31 +356,40 @@ snap_ts_clear(TINFO *tinfo, uint64_t ts) } /* - * snap_repeat_ok_match -- - * Compare two operations and see if they modified the same record. + * snap_repeat_match -- + * Compare two operations and return if they modified the same record. */ static bool -snap_repeat_ok_match(SNAP_OPS *current, SNAP_OPS *a) +snap_repeat_match(SNAP_OPS *current, SNAP_OPS *a) { + TABLE *table; + bool reverse; + /* Reads are never a problem, there's no modification. */ if (a->op == READ) - return (true); + return (false); - /* Check for a matching single record modification. */ - if (a->keyno == current->keyno) + /* Check if the operations were in the same table. */ + if (a->id != current->id) return (false); + /* Check for a matching single record insert, modify, remove or update. */ + if (a->keyno == current->keyno) + return (true); + /* Truncates are slightly harder, make sure the ranges don't overlap. */ + table = tables[ntables == 0 ? 0 : a->id]; + reverse = TV(BTREE_REVERSE) != 0; if (a->op == TRUNCATE) { - if (g.c_reverse && (a->keyno == 0 || a->keyno >= current->keyno) && + if (reverse && (a->keyno == 0 || a->keyno >= current->keyno) && (a->last == 0 || a->last <= current->keyno)) - return (false); - if (!g.c_reverse && (a->keyno == 0 || a->keyno <= current->keyno) && + return (true); + if (!reverse && (a->keyno == 0 || a->keyno <= current->keyno) && (a->last == 0 || a->last >= current->keyno)) - return (false); + return (true); } - return (true); + return (false); } /* @@ -401,7 +422,7 @@ snap_repeat_ok_commit(TINFO *tinfo, SNAP_OPS *current) if (p->opid != tinfo->opid) break; - if (!snap_repeat_ok_match(current, p)) + if (snap_repeat_match(current, p)) return (false); } @@ -414,7 +435,7 @@ snap_repeat_ok_commit(TINFO *tinfo, SNAP_OPS *current) if (p->opid != tinfo->opid) break; - if (!snap_repeat_ok_match(current, p)) + if (snap_repeat_match(current, p)) return (false); } return (true); @@ -444,7 +465,7 @@ snap_repeat_ok_rollback(TINFO *tinfo, SNAP_OPS *current) if (p->opid != tinfo->opid) break; - if (!snap_repeat_ok_match(current, p)) + if (snap_repeat_match(current, p)) return (false); } return (true); @@ -455,7 +476,7 @@ snap_repeat_ok_rollback(TINFO *tinfo, SNAP_OPS *current) * Repeat each operation done within a snapshot isolation transaction. */ int -snap_repeat_txn(WT_CURSOR *cursor, TINFO *tinfo) +snap_repeat_txn(TINFO *tinfo) { SNAP_OPS *current; @@ -478,7 +499,7 @@ snap_repeat_txn(WT_CURSOR *cursor, TINFO *tinfo) * other threads of control committing in our past, until the transaction resolves. */ if (snap_repeat_ok_commit(tinfo, current)) - WT_RET(snap_verify(cursor, tinfo, current)); + WT_RET(snap_verify(tinfo, current)); } return (0); @@ -532,7 +553,7 @@ snap_repeat_update(TINFO *tinfo, bool committed) * Repeat one operation. */ static void -snap_repeat(WT_CURSOR *cursor, TINFO *tinfo, SNAP_OPS *snap) +snap_repeat(TINFO *tinfo, SNAP_OPS *snap) { WT_DECL_RET; WT_SESSION *session; @@ -540,7 +561,7 @@ snap_repeat(WT_CURSOR *cursor, TINFO *tinfo, SNAP_OPS *snap) u_int max_retry; char buf[64]; - session = cursor->session; + session = tinfo->session; trace_op(tinfo, "repeat %" PRIu64 " ts=%" PRIu64 " {%s}", snap->keyno, snap->ts, trace_bytes(tinfo, snap->vdata, snap->vsize)); @@ -563,7 +584,7 @@ snap_repeat(WT_CURSOR *cursor, TINFO *tinfo, SNAP_OPS *snap) * matter to us). Persist after rollback, as a repeatable read we should succeed, yield to * let eviction catch up. */ - if ((ret = snap_verify(cursor, tinfo, snap)) == 0) + if ((ret = snap_verify(tinfo, snap)) == 0) break; testutil_assert(ret == WT_ROLLBACK); @@ -579,7 +600,7 @@ snap_repeat(WT_CURSOR *cursor, TINFO *tinfo, SNAP_OPS *snap) * Repeat an historic operation. */ void -snap_repeat_single(WT_CURSOR *cursor, TINFO *tinfo) +snap_repeat_single(TINFO *tinfo) { SNAP_OPS *snap; u_int v; @@ -602,7 +623,7 @@ snap_repeat_single(WT_CURSOR *cursor, TINFO *tinfo) if (count == 0) return; - snap_repeat(cursor, tinfo, snap); + snap_repeat(tinfo, snap); } /* @@ -610,20 +631,27 @@ snap_repeat_single(WT_CURSOR *cursor, TINFO *tinfo) * Repeat all known operations after a rollback. */ void -snap_repeat_rollback(WT_CURSOR *cursor, TINFO **tinfo_array, size_t tinfo_count) +snap_repeat_rollback(TINFO **tinfo_array, size_t tinfo_count) { SNAP_OPS *snap; SNAP_STATE *state; TINFO *tinfo, **tinfop; + WT_SESSION *push_session, *session; uint32_t count; size_t i, statenum; - char buf[100]; + char buf[64]; count = 0; + session = NULL; - track("rollback_to_stable: checking", 0ULL, NULL); + track("rollback_to_stable: checking", 0ULL); for (i = 0, tinfop = tinfo_array; i < tinfo_count; ++i, ++tinfop) { tinfo = *tinfop; + if ((push_session = tinfo->session) == NULL) { + if (session == NULL) + testutil_check(g.wts_conn->open_session(g.wts_conn, NULL, NULL, &session)); + tinfo->session = session; + } /* * For this thread, walk through both sets of snaps ("states"), looking for entries that are @@ -636,28 +664,33 @@ snap_repeat_rollback(WT_CURSOR *cursor, TINFO **tinfo_array, size_t tinfo_count) state = &tinfo->snap_states[statenum]; for (snap = state->snap_state_list; snap < state->snap_state_end; ++snap) { if (snap->repeatable && snap->ts <= g.stable_timestamp) { - snap_repeat(cursor, tinfo, snap); + snap_repeat(tinfo, snap); ++count; if (count % 100 == 0) { testutil_check(__wt_snprintf( buf, sizeof(buf), "rollback_to_stable: %" PRIu32 " ops repeated", count)); - track(buf, 0ULL, NULL); + track(buf, 0ULL); } } snap_clear_one(snap); } } + + tinfo->session = push_session; } /* Show the final result and check that we're accomplishing some checking. */ testutil_check( __wt_snprintf(buf, sizeof(buf), "rollback_to_stable: %" PRIu32 " ops repeated", count)); - track(buf, 0ULL, NULL); + track(buf, 0ULL); if (count == 0) { #define WARN_RTS_NO_CHECK 5 if (++g.rts_no_check >= WARN_RTS_NO_CHECK) - fprintf(stderr, - "Warning: %" PRIu32 " consecutive runs with no rollback_to_stable checking\n", count); + fprintf( + stderr, "Warning: %u consecutive runs with no rollback_to_stable checking\n", count); } else g.rts_no_check = 0; + + if (session == NULL) + testutil_check(session->close(session, NULL)); } diff --git a/src/third_party/wiredtiger/test/format/t.c b/src/third_party/wiredtiger/test/format/t.c index 3656635e873..aa7baad4c7c 100644 --- a/src/third_party/wiredtiger/test/format/t.c +++ b/src/third_party/wiredtiger/test/format/t.c @@ -30,17 +30,22 @@ GLOBAL g; +TABLE *tables[V_MAX_TABLES_CONFIG + 1]; /* Table array */ +u_int ntables; + static void format_die(void); static void usage(void) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); extern int __wt_optind; extern char *__wt_optarg; +static void signal_handler(int signo) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); +static void signal_timer(int signo) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); + /* * signal_handler -- * Generic signal handler, report the signal and exit. */ -static void signal_handler(int signo) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static void signal_handler(int signo) { @@ -53,7 +58,6 @@ signal_handler(int signo) * signal_timer -- * Alarm signal handler. */ -static void signal_timer(int signo) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static void signal_timer(int signo) { @@ -61,7 +65,7 @@ signal_timer(int signo) * Direct I/O configurations can result in really long run times depending on how the test * machine is configured. If a direct I/O run timed out, don't bother dropping core. */ - if (g.c_direct_io) { + if (GV(DISK_DIRECT_IO)) { fprintf(stderr, "format direct I/O configuration timed out\n"); fprintf(stderr, "format caught signal %d, exiting with error\n", signo); fflush(stderr); @@ -118,9 +122,6 @@ format_process_env(void) (void)signal(SIGTERM, signal_handler); #endif - /* Initialize lock to ensure single threading during failure handling */ - testutil_check(pthread_rwlock_init(&g.death_lock, NULL)); - #if 0 /* Configure the GNU malloc for debugging. */ (void)setenv("MALLOC_CHECK_", "2", 1); @@ -135,15 +136,17 @@ format_process_env(void) * TIMED_MAJOR_OP -- * Set a timer and perform a major operation (for example, verify or salvage). */ -#define TIMED_MAJOR_OP(call) \ - do { \ - if (g.c_major_timeout != 0) \ - set_alarm(g.c_major_timeout * 60); \ - call; \ - if (g.c_major_timeout != 0) \ - set_alarm(0); \ +#define TIMED_MAJOR_OP(call) \ + do { \ + if (GV(FORMAT_MAJOR_TIMEOUT) != 0) \ + set_alarm(GV(FORMAT_MAJOR_TIMEOUT) * 60); \ + call; \ + if (GV(FORMAT_MAJOR_TIMEOUT) != 0) \ + set_alarm(0); \ } while (0) +static bool syntax_check; /* Only checking configuration syntax. */ + int main(int argc, char *argv[]) { @@ -151,7 +154,7 @@ main(int argc, char *argv[]) u_int ops_seconds; int ch, reps; const char *config, *home; - bool one_flag, quiet_flag; + bool quiet_flag; custom_die = format_die; /* Local death handler. */ @@ -159,21 +162,23 @@ main(int argc, char *argv[]) (void)testutil_set_progname(argv); - /* The monitoring program looks for this line in the log file, push it out quickly. */ - printf("%s: process %" PRIdMAX " running\n", progname, (intmax_t)getpid()); - fflush(stdout); - format_process_env(); - __wt_random_init_seed(NULL, &g.rnd); /* Initialize the RNG. */ + /* + * If built in a branch that doesn't support all current options, configure for backward + * compatibility. + */ +#if WIREDTIGER_VERSION_MAJOR < 10 + g.backward_compatible = true; +#endif /* Set values from the command line. */ home = NULL; - one_flag = quiet_flag = false; - while ((ch = __wt_getopt(progname, argc, argv, "1BC:c:h:qRrT:t")) != EOF) + quiet_flag = syntax_check = false; + while ((ch = __wt_getopt(progname, argc, argv, "1BC:c:h:qRSrT:t")) != EOF) switch (ch) { - case '1': /* One run and quit */ - one_flag = true; + case '1': + /* Ignored for backward compatibility. */ break; case 'B': /* Backward compatibility */ g.backward_compatible = true; @@ -193,12 +198,12 @@ main(int argc, char *argv[]) case 'R': /* Reopen (start running on an existing database) */ g.reopen = true; break; + case 'S': /* Configuration syntax check */ + syntax_check = true; + break; case 'T': /* Trace specifics. */ - if (trace_config(__wt_optarg) != 0) { - fprintf(stderr, "unexpected trace configuration \"%s\"\n", __wt_optarg); - usage(); - } - /* FALLTHROUGH */ + trace_config(__wt_optarg); + /* FALLTHROUGH */ case 't': /* Trace */ g.trace = true; break; @@ -207,14 +212,35 @@ main(int argc, char *argv[]) } argv += __wt_optind; + /* format.sh looks for this line in the log file, push it out quickly. */ + if (!syntax_check) { + printf("%s: process %" PRIdMAX " running\n", progname, (intmax_t)getpid()); + fflush(stdout); + } + + __wt_random_init_seed(NULL, &g.rnd); /* Initialize the RNG. */ + + /* Printable thread ID. */ + testutil_check(__wt_thread_str(g.tidbuf, sizeof(g.tidbuf))); + + /* Initialize lock to ensure single threading during failure handling */ + testutil_check(pthread_rwlock_init(&g.death_lock, NULL)); + + /* + * Initialize the tables array and default to multi-table testing if not in backward-compatible + * mode. + */ + tables[0] = dcalloc(1, sizeof(TABLE)); + tables[0]->id = 1; + g.multi_table_config = !g.backward_compatible; + /* Set up paths. */ path_setup(home); /* - * If it's a reopen, use the already existing home directory's CONFIG file. - * - * If we weren't given a configuration file, set values from "CONFIG", if it exists. Small hack - * to ignore any CONFIG file named ".", that just makes it possible to ignore any local CONFIG + * If it's a reopen, use the already existing home directory's CONFIG file. Otherwise, if we + * weren't given a configuration file, set values from "CONFIG", if it exists. Small hack to + * ignore any CONFIG file named ".", that just makes it possible to ignore any local CONFIG * file, used when running checks. */ if (g.reopen) { @@ -233,16 +259,62 @@ main(int argc, char *argv[]) * which can lead to a lot of hurt if you're not careful. */ for (; *argv != NULL; ++argv) - config_single(*argv, true); + config_single(NULL, *argv, true); /* - * Let the command line -1 and -q flags override values configured from other sources. - * Regardless, don't go all verbose if we're not talking to a terminal. + * Let the command line -q flag override values configured from other sources. Regardless, don't + * go all verbose if we're not talking to a terminal. */ - if (one_flag) - g.c_runs = 1; if (quiet_flag || !isatty(1)) - g.c_quiet = 1; + GV(QUIET) = 1; + + /* Configure the run. */ + config_run(); + g.configured = true; + + /* If checking a CONFIG file syntax, we're done. */ + if (syntax_check) + exit(0); + + /* Initialize locks to single-thread backups and timestamps. */ + lock_init(g.wts_session, &g.backup_lock); + lock_init(g.wts_session, &g.ts_lock); + lock_init(g.wts_session, &g.prepare_commit_lock); + + __wt_seconds(NULL, &start); + track("starting up", 0ULL); + + /* Create and open, or reopen the database. */ + if (g.reopen) { + if (GV(RUNS_IN_MEMORY)) + testutil_die(0, "reopen impossible after in-memory run"); + wts_open(g.home, &g.wts_conn, &g.wts_session, true); + timestamp_init(); + set_oldest_timestamp(); + } else { + wts_create_home(); + config_print(false); + wts_create_database(); + wts_open(g.home, &g.wts_conn, &g.wts_session, true); + timestamp_init(); + } + + trace_init(); /* Initialize operation tracing. */ + + /* + * Initialize key/value information. Load and verify initial records (at least a brief scan if + * not doing a full verify). + */ + tables_apply(key_init, NULL); + tables_apply(val_init, NULL); + if (!g.reopen) + TIMED_MAJOR_OP(tables_apply(wts_load, NULL)); + TIMED_MAJOR_OP(tables_apply(wts_verify, g.wts_conn)); + if (GV(OPS_VERIFY) == 0) + TIMED_MAJOR_OP(tables_apply(wts_read_scan, g.wts_conn)); + + /* Optionally start checkpoints. */ + wts_checkpoints(); /* * Calculate how long each operations loop should run. Take any timer value and convert it to @@ -253,101 +325,62 @@ main(int argc, char *argv[]) * even if we run out of time, otherwise it won't get done. So, in summary pick a reasonable * time and then don't check for timer expiration once the main operations loop completes. */ - ops_seconds = g.c_timer == 0 ? 0 : ((g.c_timer * 60) - 15) / FORMAT_OPERATION_REPS; - - testutil_check(__wt_thread_str(g.tidbuf, sizeof(g.tidbuf))); - - while (++g.run_cnt <= g.c_runs || g.c_runs == 0) { - __wt_seconds(NULL, &start); - track("starting up", 0ULL, NULL); - - config_run(); - - if (g.reopen) { - config_final(); - wts_open(g.home, &g.wts_conn, &g.wts_session, true); - timestamp_init(); - set_oldest_timestamp(); - } else { - wts_create(g.home); - config_final(); - wts_open(g.home, &g.wts_conn, &g.wts_session, true); - timestamp_init(); - - trace_init(); - } - - /* Initialize locks to single-thread backups, failures, and timestamp updates. */ - lock_init(g.wts_session, &g.backup_lock); - lock_init(g.wts_session, &g.ts_lock); - lock_init(g.wts_session, &g.prepare_commit_lock); - - if (!g.reopen) - TIMED_MAJOR_OP(wts_load()); /* Load and verify initial records */ - - TIMED_MAJOR_OP(wts_verify(g.wts_conn, "verify")); - TIMED_MAJOR_OP(wts_read_scan()); - - wts_checkpoints(); + ops_seconds = GV(RUNS_TIMER) == 0 ? 0 : ((GV(RUNS_TIMER) * 60) - 15) / FORMAT_OPERATION_REPS; + for (reps = 1; reps <= FORMAT_OPERATION_REPS; ++reps) + operations(ops_seconds, reps == FORMAT_OPERATION_REPS); - /* Operations. */ - for (reps = 1; reps <= FORMAT_OPERATION_REPS; ++reps) - operations(ops_seconds, reps == FORMAT_OPERATION_REPS); + /* Copy out the run's statistics. */ + TIMED_MAJOR_OP(wts_stats()); - /* Copy out the run's statistics. */ - TIMED_MAJOR_OP(wts_stats()); - - /* - * Verify the objects. Verify closes the underlying handle and discards the statistics, read - * them first. - */ - TIMED_MAJOR_OP(wts_verify(g.wts_conn, "post-ops verify")); - - lock_destroy(g.wts_session, &g.backup_lock); - lock_destroy(g.wts_session, &g.ts_lock); - lock_destroy(g.wts_session, &g.prepare_commit_lock); - - track("shutting down", 0ULL, NULL); - wts_close(&g.wts_conn, &g.wts_session); + /* + * Verify the objects. Verify closes the underlying handle and discards the statistics, read + * them first. + */ + TIMED_MAJOR_OP(tables_apply(wts_verify, g.wts_conn)); - /* - * Salvage testing. - */ - TIMED_MAJOR_OP(wts_salvage()); + track("shutting down", 0ULL); + wts_close(&g.wts_conn, &g.wts_session); - trace_teardown(); + /* Salvage testing. */ + TIMED_MAJOR_OP(tables_apply(wts_salvage, NULL)); - /* Overwrite the progress line with a completion line. */ - if (!g.c_quiet) - printf("\r%78s\r", " "); - __wt_seconds(NULL, &now); - printf("%4" PRIu32 ": %s, %s (%" PRIu64 " seconds)\n", g.run_cnt, g.c_data_source, - g.c_file_type, now - start); - fflush(stdout); - } + trace_teardown(); - config_print(false); + /* Overwrite the progress line with a completion line. */ + if (!GV(QUIET)) + printf("\r%78s\r", " "); + __wt_seconds(NULL, &now); + printf("%s: successful run completed (%" PRIu64 " seconds)\n ", progname, now - start); + fflush(stdout); config_clear(); - printf("%s: successful run completed\n", progname); + lock_destroy(g.wts_session, &g.backup_lock); + lock_destroy(g.wts_session, &g.ts_lock); + lock_destroy(g.wts_session, &g.prepare_commit_lock); return (EXIT_SUCCESS); } /* - * die -- + * format_die -- * Report an error, dumping the configuration. */ static void format_die(void) { + /* If only checking configuration syntax, no need to message or drop core. */ + if (syntax_check) + exit(1); + /* - * Turn off progress reports so we don't obscure the error message. The lock we're about to - * acquire will act as a barrier to schedule the write. This is really a "best effort" more than - * a guarantee, there's too much stuff in flight to be sure. + * Turn off progress reports and tracing so we don't obscure the error message or drop core when + * using a session that's being closed. The lock we're about to acquire will act as a barrier to + * schedule the write. This is really a "best effort" more than a guarantee, there's too much + * stuff in flight to be sure. */ - g.c_quiet = 1; + GV(QUIET) = 1; + g.trace = 0; /* * Single-thread error handling, our caller exits after calling us (we never release the lock). @@ -359,14 +392,17 @@ format_die(void) fflush(stderr); fflush(stdout); + /* Display the configuration that failed. */ + if (g.configured) + config_print(true); + /* Flush the logs, they may contain debugging information. */ - trace_teardown(); - if (g.c_logging && g.wts_session != NULL) + if (GV(LOGGING) && g.wts_session != NULL) testutil_check(g.wts_session->log_flush(g.wts_session, "sync=off")); - /* Display the configuration that failed. */ - if (g.run_cnt) - config_print(true); + /* Now about to close shared resources, give them a chance to empty. */ + __wt_sleep(2, 0); + trace_teardown(); #ifdef HAVE_DIAGNOSTIC /* @@ -379,7 +415,7 @@ format_die(void) * which can potentially be very large. If it becomes a problem, this can be modified to just * dump out the page this key is on. */ - if (g.c_verify_failure_dump && g.page_dump_cursor != NULL) { + if (GV(RUNS_VERIFY_FAILURE_DUMP) && g.page_dump_cursor != NULL) { set_core_off(); fprintf(stderr, "snapshot-isolation error: Dumping page to %s\n", g.home_pagedump); diff --git a/src/third_party/wiredtiger/test/format/trace.c b/src/third_party/wiredtiger/test/format/trace.c index 24971032279..b365505a05c 100644 --- a/src/third_party/wiredtiger/test/format/trace.c +++ b/src/third_party/wiredtiger/test/format/trace.c @@ -31,10 +31,13 @@ #define TRACE_DIR "OPS.TRACE" #define TRACE_INIT_CMD "rm -rf %s/" TRACE_DIR " && mkdir %s/" TRACE_DIR -int +/* + * trace_config -- + * Configure operation tracing. + */ +void trace_config(const char *config) { - WT_DECL_RET; char *copy, *p; copy = dstrdup(config); @@ -53,15 +56,16 @@ trace_config(const char *config) } for (p = copy; *p != '\0'; ++p) - if (*p != ',' && !__wt_isspace((u_char)*p)) { - ret = EINVAL; - break; - } + if (*p != ',' && !__wt_isspace((u_char)*p)) + testutil_assertfmt(0, "unexpected trace configuration \"%s\"\n", config); free(copy); - return (ret); } +/* + * trace_init -- + * Initialize operation tracing. + */ void trace_init(void) { @@ -76,7 +80,7 @@ trace_init(void) /* Write traces to a separate database by default, optionally write traces to the primary. */ if (g.trace_local) { - if (!g.c_logging) + if (!GV(LOGGING)) testutil_die(EINVAL, "operation logging to the primary database requires logging be configured for that " "database"); @@ -107,6 +111,10 @@ trace_init(void) g.trace_session = session; } +/* + * trace_teardown -- + * Close operation tracing. + */ void trace_teardown(void) { @@ -115,12 +123,14 @@ trace_teardown(void) conn = g.trace_conn; g.trace_conn = NULL; - if (!g.trace || g.trace_local || conn == NULL) - return; - - testutil_check(conn->close(conn, NULL)); + if (conn != NULL) + testutil_check(conn->close(conn, NULL)); } +/* + * trace_ops_init -- + * Per thread operation tracing setup. + */ void trace_ops_init(TINFO *tinfo) { diff --git a/src/third_party/wiredtiger/test/format/util.c b/src/third_party/wiredtiger/test/format/util.c index 42e6bb8c2ef..45f4f809815 100644 --- a/src/third_party/wiredtiger/test/format/util.c +++ b/src/third_party/wiredtiger/test/format/util.c @@ -56,85 +56,109 @@ track_ts_dots(u_int dot_count) } /* - * track -- + * track_write -- + * Write out a tracking message. + */ +static void +track_write(char *msg, size_t len) +{ + static size_t last_len; /* callers must be single-threaded */ + + if (last_len > len) { + memset(msg + len, ' ', (size_t)(last_len - len)); + msg[last_len] = '\0'; + } + last_len = len; + + if (printf("%s\r", msg) < 0) + testutil_die(EIO, "printf"); + if (fflush(stdout) == EOF) + testutil_die(errno, "fflush"); +} + +/* + * track_ops -- * Show a status line of operations and time stamp progress. */ void -track(const char *tag, uint64_t cnt, TINFO *tinfo) +track_ops(TINFO *tinfo) { - static size_t last_len; static uint64_t last_cur, last_old, last_stable; static u_int cur_dot_cnt, old_dot_cnt, stable_dot_cnt; size_t len; uint64_t cur_ts, old_ts, stable_ts; char msg[128], ts_msg[64]; - if (g.c_quiet || tag == NULL) + if (GV(QUIET)) return; - if (tinfo == NULL && cnt == 0) - testutil_check( - __wt_snprintf_len_set(msg, sizeof(msg), &len, "%4" PRIu32 ": %s", g.run_cnt, tag)); - else if (tinfo == NULL) - testutil_check(__wt_snprintf_len_set( - msg, sizeof(msg), &len, "%4" PRIu32 ": %s: %" PRIu64, g.run_cnt, tag, cnt)); - else { - ts_msg[0] = '\0'; - if (g.c_txn_timestamps) { - /* - * Don't worry about having a completely consistent set of timestamps. - */ - old_ts = g.oldest_timestamp; - stable_ts = g.stable_timestamp; - cur_ts = g.timestamp; - - if (old_ts != last_old) { - ++old_dot_cnt; - last_old = old_ts; - } - if (stable_ts != last_stable) { - ++stable_dot_cnt; - last_stable = stable_ts; - } - if (cur_ts != last_cur) { - ++cur_dot_cnt; - last_cur = cur_ts; - } - - testutil_check(__wt_snprintf(ts_msg, sizeof(ts_msg), - " old%s" - "stb%s%s" - "ts%s%s", - track_ts_dots(old_dot_cnt), track_ts_diff(old_ts, stable_ts), - track_ts_dots(stable_dot_cnt), track_ts_diff(stable_ts, cur_ts), - track_ts_dots(cur_dot_cnt))); + ts_msg[0] = '\0'; + if (g.transaction_timestamps_config) { + /* + * Don't worry about having a completely consistent set of timestamps. + */ + old_ts = g.oldest_timestamp; + stable_ts = g.stable_timestamp; + cur_ts = g.timestamp; + + if (old_ts != last_old) { + ++old_dot_cnt; + last_old = old_ts; } - testutil_check( - __wt_snprintf_len_set(msg, sizeof(msg), &len, - "%4" PRIu32 ": %s: " - "S %" PRIu64 "%s, " - "I %" PRIu64 "%s, " - "U %" PRIu64 "%s, " - "R %" PRIu64 "%s%s", - g.run_cnt, tag, tinfo->search > M(9) ? tinfo->search / M(1) : tinfo->search, - tinfo->search > M(9) ? "M" : "", - tinfo->insert > M(9) ? tinfo->insert / M(1) : tinfo->insert, - tinfo->insert > M(9) ? "M" : "", - tinfo->update > M(9) ? tinfo->update / M(1) : tinfo->update, - tinfo->update > M(9) ? "M" : "", - tinfo->remove > M(9) ? tinfo->remove / M(1) : tinfo->remove, - tinfo->remove > M(9) ? "M" : "", ts_msg)); - } - if (last_len > len) { - memset(msg + len, ' ', (size_t)(last_len - len)); - msg[last_len] = '\0'; + if (stable_ts != last_stable) { + ++stable_dot_cnt; + last_stable = stable_ts; + } + if (cur_ts != last_cur) { + ++cur_dot_cnt; + last_cur = cur_ts; + } + + testutil_check(__wt_snprintf(ts_msg, sizeof(ts_msg), + " old%s" + "stb%s%s" + "ts%s%s", + track_ts_dots(old_dot_cnt), track_ts_diff(old_ts, stable_ts), + track_ts_dots(stable_dot_cnt), track_ts_diff(stable_ts, cur_ts), + track_ts_dots(cur_dot_cnt))); } - last_len = len; + testutil_check(__wt_snprintf_len_set(msg, sizeof(msg), &len, + "ops: " + "S %" PRIu64 + "%s, " + "I %" PRIu64 + "%s, " + "U %" PRIu64 + "%s, " + "R %" PRIu64 "%s%s", + tinfo->search > M(9) ? tinfo->search / M(1) : tinfo->search, tinfo->search > M(9) ? "M" : "", + tinfo->insert > M(9) ? tinfo->insert / M(1) : tinfo->insert, tinfo->insert > M(9) ? "M" : "", + tinfo->update > M(9) ? tinfo->update / M(1) : tinfo->update, tinfo->update > M(9) ? "M" : "", + tinfo->remove > M(9) ? tinfo->remove / M(1) : tinfo->remove, tinfo->remove > M(9) ? "M" : "", + ts_msg)); + + track_write(msg, len); +} - if (printf("%s\r", msg) < 0) - testutil_die(EIO, "printf"); - if (fflush(stdout) == EOF) - testutil_die(errno, "fflush"); +/* + * track -- + * Show general operation progress. + */ +void +track(const char *tag, uint64_t cnt) +{ + size_t len; + char msg[128]; + + if (GV(QUIET)) + return; + + if (cnt == 0) + testutil_check(__wt_snprintf_len_set(msg, sizeof(msg), &len, "%s", tag)); + else + testutil_check(__wt_snprintf_len_set(msg, sizeof(msg), &len, "%s: %" PRIu64, tag, cnt)); + + track_write(msg, len); } /* @@ -162,12 +186,6 @@ path_setup(const char *home) g.home_key = dmalloc(len); testutil_check(__wt_snprintf(g.home_key, len, "%s/%s", g.home, name)); - /* RNG log file. */ - name = "CONFIG.rand"; - len = strlen(g.home) + strlen(name) + 2; - g.home_rand = dmalloc(len); - testutil_check(__wt_snprintf(g.home_rand, len, "%s/%s", g.home, name)); - /* History store dump file. */ name = "FAIL.HSdump"; len = strlen(g.home) + strlen(name) + 2; @@ -195,7 +213,7 @@ bool fp_readv(FILE *fp, char *name, uint32_t *vp) { u_long ulv; - char *endptr, buf[100]; + char *endptr, buf[64]; if (fgets(buf, sizeof(buf), fp) == NULL) testutil_die(errno, "%s: read-value error", name); @@ -239,7 +257,7 @@ timestamp_parse(const char *p, uint64_t *tsp) } /* - * timestamp_stable -- + * timestamp_init -- * Set the stable timestamp on open. */ void @@ -317,7 +335,8 @@ timestamp(void *arg) WT_CONNECTION *conn; WT_SESSION *session; - (void)(arg); + (void)arg; /* Unused argument */ + conn = g.wts_conn; /* Locks need session */ @@ -390,7 +409,7 @@ lock_init(WT_SESSION *session, RWLOCK *lock) { testutil_assert(lock->lock_type == LOCK_NONE); - if (g.c_wt_mutex) { + if (GV(WIREDTIGER_RWLOCK)) { testutil_check(__wt_rwlock_init((WT_SESSION_IMPL *)session, &lock->l.wt)); lock->lock_type = LOCK_WT; } else { @@ -408,10 +427,9 @@ lock_destroy(WT_SESSION *session, RWLOCK *lock) { testutil_assert(LOCK_INITIALIZED(lock)); - if (lock->lock_type == LOCK_WT) { + if (lock->lock_type == LOCK_WT) __wt_rwlock_destroy((WT_SESSION_IMPL *)session, &lock->l.wt); - } else { + else testutil_check(pthread_rwlock_destroy(&lock->l.pthread)); - } lock->lock_type = LOCK_NONE; } diff --git a/src/third_party/wiredtiger/test/format/wts.c b/src/third_party/wiredtiger/test/format/wts.c index 74027748246..b059acecbbf 100644 --- a/src/third_party/wiredtiger/test/format/wts.c +++ b/src/third_party/wiredtiger/test/format/wts.c @@ -28,74 +28,28 @@ #include "format.h" -/* - * Home directory initialize command: create the directory if it doesn't exist, else remove - * everything except the RNG log file. - * - * Redirect the "cd" command to /dev/null so chatty cd implementations don't add the new working - * directory to our output. - */ -#define FORMAT_HOME_INIT_CMD \ - "test -e %s || mkdir %s; " \ - "cd %s > /dev/null && rm -rf `ls | sed /CONFIG.rand/d`" - -/* - * compressor -- - * Configure compression. - */ -static const char * -compressor(uint32_t compress_flag) -{ - const char *p; - - p = "unrecognized compressor flag"; - switch (compress_flag) { - case COMPRESS_NONE: - p = "none"; - break; - case COMPRESS_LZ4: - p = "lz4"; - break; - case COMPRESS_SNAPPY: - p = "snappy"; - break; - case COMPRESS_ZLIB: - p = "zlib"; - break; - case COMPRESS_ZSTD: - p = "zstd"; - break; - default: - testutil_die(EINVAL, "illegal compression flag: %#" PRIx32, compress_flag); - /* NOTREACHED */ - } - return (p); -} +static void create_object(TABLE *, void *); /* * encryptor -- * Configure encryption. */ static const char * -encryptor(uint32_t encrypt_flag) +encryptor(void) { - const char *p; + const char *p, *s; - p = "unrecognized encryptor flag"; - switch (encrypt_flag) { - case ENCRYPT_NONE: + s = GVS(DISK_ENCRYPTION); + if (strcmp(s, "off") == 0) p = "none"; - break; - case ENCRYPT_ROTN_7: + else if (strcmp(s, "rotn-7") == 0) p = "rotn,keyid=7"; - break; - case ENCRYPT_SODIUM: + else if (strcmp(s, "sodium") == 0) p = "sodium,secretkey=" SODIUM_TESTKEY; - break; - default: - testutil_die(EINVAL, "illegal encryption flag: %#" PRIx32, encrypt_flag); - /* NOTREACHED */ - } + else + testutil_die(EINVAL, "illegal encryption configuration: %s", s); + + /* Returns "none" or the name of an encryptor. */ return (p); } @@ -106,34 +60,36 @@ encryptor(uint32_t encrypt_flag) * This must set any secretkey. When keyids are in use it can return NULL. */ static const char * -encryptor_at_open(uint32_t encrypt_flag) +encryptor_at_open(void) { - const char *p; - - p = NULL; - switch (encrypt_flag) { - case ENCRYPT_NONE: - break; - case ENCRYPT_ROTN_7: - break; - case ENCRYPT_SODIUM: + const char *p, *s; + + s = GVS(DISK_ENCRYPTION); + if (strcmp(s, "off") == 0) + p = NULL; + else if (strcmp(s, "rotn-7") == 0) + p = NULL; + else if (strcmp(s, "sodium") == 0) p = "sodium,secretkey=" SODIUM_TESTKEY; - break; - default: - testutil_die(EINVAL, "illegal encryption flag: %#" PRIx32, encrypt_flag); - /* NOTREACHED */ - } + else + testutil_die(EINVAL, "illegal encryption configuration: %s", s); + + /* Returns NULL or the name of an encryptor. */ return (p); } +/* + * handle_message -- + * Event handler for verbose and error messages. + */ static int handle_message(WT_EVENT_HANDLER *handler, WT_SESSION *session, const char *message) { WT_DECL_RET; int nw; - (void)(handler); - (void)(session); + (void)handler; + (void)session; /* * WiredTiger logs a verbose message when the read timestamp is set to a value older than the @@ -149,14 +105,26 @@ handle_message(WT_EVENT_HANDLER *handler, WT_SESSION *session, const char *messa return (nw < 0 ? EIO : (ret == EOF ? errno : 0)); } +/* + * handle_progress -- + * Event handler for progress messages. + */ static int handle_progress( WT_EVENT_HANDLER *handler, WT_SESSION *session, const char *operation, uint64_t progress) { - (void)(handler); - (void)(session); + char buf[256]; + + (void)handler; + (void)session; - track(operation, progress, NULL); + if (session->app_private == NULL) + track(operation, progress); + else { + testutil_check( + __wt_snprintf(buf, sizeof(buf), "%s %s", (char *)session->app_private, operation)); + track(buf, progress); + } return (0); } @@ -175,6 +143,51 @@ static WT_EVENT_HANDLER event_handler = { } while (0) /* + * configure_timing_stress -- + * Configure stressing settings. + */ +static void +configure_timing_stress(char *p, size_t max) +{ + CONFIG_APPEND(p, ",timing_stress_for_test=["); + if (GV(STRESS_AGGRESSIVE_SWEEP)) + CONFIG_APPEND(p, ",aggressive_sweep"); + if (GV(STRESS_CHECKPOINT)) + CONFIG_APPEND(p, ",checkpoint_slow"); + if (GV(STRESS_CHECKPOINT_PREPARE)) + CONFIG_APPEND(p, ",prepare_checkpoint_delay"); + if (GV(STRESS_CHECKPOINT_RESERVED_TXNID_DELAY)) + CONFIG_APPEND(p, ",checkpoint_reserved_txnid_delay"); + if (GV(STRESS_FAILPOINT_HS_DELETE_KEY_FROM_TS)) + CONFIG_APPEND(p, ",failpoint_history_store_delete_key_from_ts"); + if (GV(STRESS_FAILPOINT_HS_INSERT_1)) + CONFIG_APPEND(p, ",failpoint_history_store_insert_1"); + if (GV(STRESS_FAILPOINT_HS_INSERT_2)) + CONFIG_APPEND(p, ",failpoint_history_store_insert_2"); + if (GV(STRESS_HS_CHECKPOINT_DELAY)) + CONFIG_APPEND(p, ",history_store_checkpoint_delay"); + if (GV(STRESS_HS_SEARCH)) + CONFIG_APPEND(p, ",history_store_search"); + if (GV(STRESS_HS_SWEEP)) + CONFIG_APPEND(p, ",history_store_sweep_race"); + if (GV(STRESS_SPLIT_1)) + CONFIG_APPEND(p, ",split_1"); + if (GV(STRESS_SPLIT_2)) + CONFIG_APPEND(p, ",split_2"); + if (GV(STRESS_SPLIT_3)) + CONFIG_APPEND(p, ",split_3"); + if (GV(STRESS_SPLIT_4)) + CONFIG_APPEND(p, ",split_4"); + if (GV(STRESS_SPLIT_5)) + CONFIG_APPEND(p, ",split_5"); + if (GV(STRESS_SPLIT_6)) + CONFIG_APPEND(p, ",split_6"); + if (GV(STRESS_SPLIT_7)) + CONFIG_APPEND(p, ",split_7"); + CONFIG_APPEND(p, "]"); +} + +/* * create_database -- * Create a WiredTiger database. */ @@ -184,7 +197,7 @@ create_database(const char *home, WT_CONNECTION **connp) WT_CONNECTION *conn; size_t max; char config[8 * 1024], *p; - const char *enc; + const char *s; p = config; max = sizeof(config); @@ -196,107 +209,83 @@ create_database(const char *home, WT_CONNECTION **connp) ",checkpoint_sync=false" ",error_prefix=\"%s\"" ",operation_timeout_ms=2000", - g.c_cache, progname); + GV(CACHE), progname); /* In-memory configuration. */ - if (g.c_in_memory != 0) + if (GV(RUNS_IN_MEMORY) != 0) CONFIG_APPEND(p, ",in_memory=1"); + /* FIXME WT-8314: configuring a block cache corrupts tables. */ +#if 0 + /* Block cache configuration. */ + if (GV(BLOCK_CACHE) != 0) + CONFIG_APPEND(p, + ",block_cache=(enabled=true,type=\"dram\"" + ",cache_on_checkpoint=%s" + ",cache_on_writes=%s" + ",size=%" PRIu32 "MB)", + GV(BLOCK_CACHE_CACHE_ON_CHECKPOINT) == 0 ? "false" : "true", + GV(BLOCK_CACHE_CACHE_ON_WRITES) == 0 ? "false" : "true", GV(BLOCK_CACHE_SIZE)); +#endif + /* LSM configuration. */ - if (DATASOURCE("lsm")) - CONFIG_APPEND(p, ",lsm_manager=(worker_thread_max=%" PRIu32 "),", g.c_lsm_worker_threads); + if (g.lsm_config) + CONFIG_APPEND(p, ",lsm_manager=(worker_thread_max=%" PRIu32 "),", GV(LSM_WORKER_THREADS)); - if (DATASOURCE("lsm") || g.c_cache < 20) + if (g.lsm_config || GV(CACHE) < 20) CONFIG_APPEND(p, ",eviction_dirty_trigger=95"); - /* Eviction worker configuration. */ - if (g.c_evict_max != 0) - CONFIG_APPEND(p, ",eviction=(threads_max=%" PRIu32 ")", g.c_evict_max); + /* Eviction configuration. */ + if (GV(CACHE_EVICT_MAX) != 0) + CONFIG_APPEND(p, ",eviction=(threads_max=%" PRIu32 ")", GV(CACHE_EVICT_MAX)); /* Logging configuration. */ - if (g.c_logging) + if (GV(LOGGING)) { + s = GVS(LOGGING_COMPRESSION); CONFIG_APPEND(p, ",log=(enabled=true,archive=%d,prealloc=%d,file_max=%" PRIu32 ",compressor=\"%s\")", - g.c_logging_archive ? 1 : 0, g.c_logging_prealloc ? 1 : 0, KILOBYTE(g.c_logging_file_max), - compressor(g.c_logging_compression_flag)); + GV(LOGGING_ARCHIVE) ? 1 : 0, GV(LOGGING_PREALLOC) ? 1 : 0, KILOBYTE(GV(LOGGING_FILE_MAX)), + strcmp(s, "off") == 0 ? "none" : s); + } /* Encryption. */ - if (g.c_encryption) { - enc = encryptor(g.c_encryption_flag); - if (enc != NULL) - CONFIG_APPEND(p, ",encryption=(name=%s)", enc); - } + CONFIG_APPEND(p, ",encryption=(name=%s)", encryptor()); /* Miscellaneous. */ #ifdef HAVE_POSIX_MEMALIGN CONFIG_APPEND(p, ",buffer_alignment=512"); #endif - if (g.c_mmap) + if (GV(DISK_MMAP)) CONFIG_APPEND(p, ",mmap=1"); - if (g.c_mmap_all) + if (GV(DISK_MMAP_ALL)) CONFIG_APPEND(p, ",mmap_all=1"); - if (g.c_direct_io) + if (GV(DISK_DIRECT_IO)) CONFIG_APPEND(p, ",direct_io=(data)"); - if (g.c_data_extend) + if (GV(DISK_DATA_EXTEND)) CONFIG_APPEND(p, ",file_extend=(data=8MB)"); /* * Run the statistics server and/or maintain statistics in the engine. Sometimes specify a set * of sources just to exercise that code. */ - if (g.c_statistics_server) { - if (mmrand(NULL, 0, 5) == 1 && memcmp(g.uri, "file:", strlen("file:")) == 0) + if (GV(STATISTICS_SERVER)) { + if (mmrand(NULL, 0, 20) == 1) CONFIG_APPEND( p, ",statistics=(fast),statistics_log=(json,on_close,wait=5,sources=(\"file:\"))"); else CONFIG_APPEND(p, ",statistics=(fast),statistics_log=(json,on_close,wait=5)"); } else - CONFIG_APPEND(p, ",statistics=(%s)", g.c_statistics ? "fast" : "none"); + CONFIG_APPEND(p, ",statistics=(%s)", GV(STATISTICS) ? "fast" : "none"); - /* Optionally stress operations. */ - CONFIG_APPEND(p, ",timing_stress_for_test=["); - if (g.c_timing_stress_aggressive_sweep) - CONFIG_APPEND(p, ",aggressive_sweep"); - if (g.c_timing_stress_checkpoint) - CONFIG_APPEND(p, ",checkpoint_slow"); - if (g.c_timing_stress_checkpoint_prepare) - CONFIG_APPEND(p, ",prepare_checkpoint_delay"); - if (g.c_timing_stress_checkpoint_reserved_txnid_delay) - CONFIG_APPEND(p, ",checkpoint_reserved_txnid_delay"); - if (g.c_timing_stress_failpoint_hs_delete_key_from_ts) - CONFIG_APPEND(p, ",failpoint_history_store_delete_key_from_ts"); - if (g.c_timing_stress_failpoint_hs_insert_1) - CONFIG_APPEND(p, ",failpoint_history_store_insert_1"); - if (g.c_timing_stress_failpoint_hs_insert_2) - CONFIG_APPEND(p, ",failpoint_history_store_insert_2"); - if (g.c_timing_stress_hs_checkpoint_delay) - CONFIG_APPEND(p, ",history_store_checkpoint_delay"); - if (g.c_timing_stress_hs_search) - CONFIG_APPEND(p, ",history_store_search"); - if (g.c_timing_stress_hs_sweep) - CONFIG_APPEND(p, ",history_store_sweep_race"); - if (g.c_timing_stress_split_1) - CONFIG_APPEND(p, ",split_1"); - if (g.c_timing_stress_split_2) - CONFIG_APPEND(p, ",split_2"); - if (g.c_timing_stress_split_3) - CONFIG_APPEND(p, ",split_3"); - if (g.c_timing_stress_split_4) - CONFIG_APPEND(p, ",split_4"); - if (g.c_timing_stress_split_5) - CONFIG_APPEND(p, ",split_5"); - if (g.c_timing_stress_split_6) - CONFIG_APPEND(p, ",split_6"); - if (g.c_timing_stress_split_7) - CONFIG_APPEND(p, ",split_7"); - CONFIG_APPEND(p, "]"); + /* Optional timing stress. */ + configure_timing_stress(p, max); /* Extensions. */ CONFIG_APPEND(p, ",extensions=[\"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\"],", - g.c_reverse ? REVERSE_PATH : "", access(LZ4_PATH, R_OK) == 0 ? LZ4_PATH : "", + REVERSE_PATH, access(LZ4_PATH, R_OK) == 0 ? LZ4_PATH : "", access(ROTN_PATH, R_OK) == 0 ? ROTN_PATH : "", access(SNAPPY_PATH, R_OK) == 0 ? SNAPPY_PATH : "", access(ZLIB_PATH, R_OK) == 0 ? ZLIB_PATH : "", access(ZSTD_PATH, R_OK) == 0 ? ZSTD_PATH : "", @@ -306,8 +295,9 @@ create_database(const char *home, WT_CONNECTION **connp) * Put configuration file configuration options second to last. Put command line configuration * options at the end. Do this so they override the standard configuration. */ - if (g.c_config_open != NULL) - CONFIG_APPEND(p, ",%s", g.c_config_open); + s = GVS(WIREDTIGER_CONFIG); + if (strcmp(s, "off") != 0) + CONFIG_APPEND(p, ",%s", s); if (g.config_open != NULL) CONFIG_APPEND(p, ",%s", g.config_open); @@ -324,102 +314,93 @@ create_database(const char *home, WT_CONNECTION **connp) * Create the database object. */ static void -create_object(WT_CONNECTION *conn) +create_object(TABLE *table, void *arg) { + WT_CONNECTION *conn; WT_SESSION *session; size_t max; - uint32_t maxintlkey, maxleafkey, maxleafvalue; + uint32_t maxleafkey, maxleafvalue; char config[4096], *p; + const char *s; + conn = (WT_CONNECTION *)arg; p = config; max = sizeof(config); +/* The page must be a multiple of the allocation size, and 512 always works. */ +#define BLOCK_ALLOCATION_SIZE 512 CONFIG_APPEND(p, "key_format=%s,allocation_size=%d,%s,internal_page_max=%" PRIu32 ",leaf_page_max=%" PRIu32 ",memory_page_max=%" PRIu32, - (g.type == ROW) ? "u" : "r", BLOCK_ALLOCATION_SIZE, - g.c_firstfit ? "block_allocation=first" : "", g.intl_page_max, g.leaf_page_max, - MEGABYTE(g.c_memory_page_max)); + (table->type == ROW) ? "u" : "r", BLOCK_ALLOCATION_SIZE, + TV(DISK_FIRSTFIT) ? "block_allocation=first" : "", table->max_intl_page, table->max_leaf_page, + table->max_mem_page); /* * Configure the maximum key/value sizes, but leave it as the default if we come up with * something crazy. */ - maxintlkey = mmrand(NULL, g.intl_page_max / 50, g.intl_page_max / 40); - if (maxintlkey > 20) - CONFIG_APPEND(p, ",internal_key_max=%" PRIu32, maxintlkey); - maxleafkey = mmrand(NULL, g.leaf_page_max / 50, g.leaf_page_max / 40); + maxleafkey = mmrand(NULL, table->max_leaf_page / 50, table->max_leaf_page / 40); if (maxleafkey > 20) CONFIG_APPEND(p, ",leaf_key_max=%" PRIu32, maxleafkey); - maxleafvalue = mmrand(NULL, g.leaf_page_max * 10, g.leaf_page_max / 40); + maxleafvalue = mmrand(NULL, table->max_leaf_page * 10, table->max_leaf_page / 40); if (maxleafvalue > 40 && maxleafvalue < 100 * 1024) CONFIG_APPEND(p, ",leaf_value_max=%" PRIu32, maxleafvalue); - switch (g.type) { + switch (table->type) { case FIX: - CONFIG_APPEND(p, ",value_format=%" PRIu32 "t", g.c_bitcnt); + CONFIG_APPEND(p, ",value_format=%" PRIu32 "t", TV(BTREE_BITCNT)); break; case ROW: CONFIG_APPEND(p, ",prefix_compression=%s,prefix_compression_min=%" PRIu32, - g.c_prefix_compression == 0 ? "false" : "true", g.c_prefix_compression_min); - if (g.c_reverse) + TV(BTREE_PREFIX_COMPRESSION) == 0 ? "false" : "true", TV(BTREE_PREFIX_COMPRESSION_MIN)); + if (TV(BTREE_REVERSE)) CONFIG_APPEND(p, ",collator=reverse"); /* FALLTHROUGH */ case VAR: - if (g.c_huffman_value) + if (TV(BTREE_HUFFMAN_VALUE)) CONFIG_APPEND(p, ",huffman_value=english"); - if (g.c_dictionary) + if (TV(BTREE_DICTIONARY)) CONFIG_APPEND(p, ",dictionary=%" PRIu32, mmrand(NULL, 123, 517)); break; } /* Configure checksums. */ - switch (g.c_checksum_flag) { - case CHECKSUM_OFF: - CONFIG_APPEND(p, ",checksum=\"off\""); - break; - case CHECKSUM_ON: - CONFIG_APPEND(p, ",checksum=\"on\""); - break; - case CHECKSUM_UNCOMPRESSED: - CONFIG_APPEND(p, ",checksum=\"uncompressed\""); - break; - case CHECKSUM_UNENCRYPTED: - CONFIG_APPEND(p, ",checksum=\"unencrypted\""); - break; - } + CONFIG_APPEND(p, ",checksum=\"%s\"", TVS(DISK_CHECKSUM)); /* Configure compression. */ - if (g.c_compression_flag != COMPRESS_NONE) - CONFIG_APPEND(p, ",block_compressor=\"%s\"", compressor(g.c_compression_flag)); + s = TVS(BTREE_COMPRESSION); + CONFIG_APPEND(p, ",block_compressor=\"%s\"", strcmp(s, "off") == 0 ? "none" : s); /* Configure Btree. */ - CONFIG_APPEND(p, ",internal_key_truncate=%s", g.c_internal_key_truncation ? "true" : "false"); - CONFIG_APPEND(p, ",split_pct=%" PRIu32, g.c_split_pct); + CONFIG_APPEND( + p, ",internal_key_truncate=%s", TV(BTREE_INTERNAL_KEY_TRUNCATION) ? "true" : "false"); + CONFIG_APPEND(p, ",split_pct=%" PRIu32, TV(BTREE_SPLIT_PCT)); /* Assertions: assertions slow down the code for additional diagnostic checking. */ - if (g.c_assert_read_timestamp) - CONFIG_APPEND(p, ",assert=(read_timestamp=%s)", g.c_txn_timestamps ? "always" : "never"); - if (g.c_assert_write_timestamp) + if (GV(ASSERT_READ_TIMESTAMP)) + CONFIG_APPEND( + p, ",assert=(read_timestamp=%s)", g.transaction_timestamps_config ? "always" : "never"); + if (GV(ASSERT_WRITE_TIMESTAMP)) CONFIG_APPEND(p, ",assert=(write_timestamp=on),write_timestamp_usage=%s", - g.c_txn_timestamps ? "key_consistent" : "never"); + g.transaction_timestamps_config ? "key_consistent" : "never"); /* Configure LSM. */ - if (DATASOURCE("lsm")) { + if (DATASOURCE(table, "lsm")) { CONFIG_APPEND(p, ",type=lsm,lsm=("); - CONFIG_APPEND(p, "auto_throttle=%s,", g.c_auto_throttle ? "true" : "false"); - CONFIG_APPEND(p, "chunk_size=%" PRIu32 "MB,", g.c_chunk_size); + CONFIG_APPEND(p, "auto_throttle=%s,", TV(LSM_AUTO_THROTTLE) ? "true" : "false"); + CONFIG_APPEND(p, "chunk_size=%" PRIu32 "MB,", TV(LSM_CHUNK_SIZE)); /* * We can't set bloom_oldest without bloom, and we want to test with Bloom filters on most * of the time anyway. */ - if (g.c_bloom_oldest) - g.c_bloom = 1; - CONFIG_APPEND(p, "bloom=%s,", g.c_bloom ? "true" : "false"); - CONFIG_APPEND(p, "bloom_bit_count=%" PRIu32 ",", g.c_bloom_bit_count); - CONFIG_APPEND(p, "bloom_hash_count=%" PRIu32 ",", g.c_bloom_hash_count); - CONFIG_APPEND(p, "bloom_oldest=%s,", g.c_bloom_oldest ? "true" : "false"); - CONFIG_APPEND(p, "merge_max=%" PRIu32 ",", g.c_merge_max); + if (TV(LSM_BLOOM_OLDEST)) + TV(LSM_BLOOM) = 1; + CONFIG_APPEND(p, "bloom=%s,", TV(LSM_BLOOM) ? "true" : "false"); + CONFIG_APPEND(p, "bloom_bit_count=%" PRIu32 ",", TV(LSM_BLOOM_BIT_COUNT)); + CONFIG_APPEND(p, "bloom_hash_count=%" PRIu32 ",", TV(LSM_BLOOM_HASH_COUNT)); + CONFIG_APPEND(p, "bloom_oldest=%s,", TV(LSM_BLOOM_OLDEST) ? "true" : "false"); + CONFIG_APPEND(p, "merge_max=%" PRIu32 ",", TV(LSM_MERGE_MAX)); CONFIG_APPEND(p, ",)"); } @@ -430,35 +411,41 @@ create_object(WT_CONNECTION *conn) * Create the underlying store. */ testutil_check(conn->open_session(conn, NULL, NULL, &session)); - testutil_checkfmt(session->create(session, g.uri, config), "%s", g.uri); + testutil_checkfmt(session->create(session, table->uri, config), "%s", table->uri); testutil_check(session->close(session, NULL)); } /* - * wts_create -- - * Create the database home and objects. + * wts_create_home -- + * Remove and re-create the directory. + */ +void +wts_create_home(void) +{ + char buf[MAX_FORMAT_PATH * 2]; + + testutil_check(__wt_snprintf(buf, sizeof(buf), "rm -rf %s && mkdir %s", g.home, g.home)); + testutil_checkfmt(system(buf), "database home creation (\"%s\") failed", buf); +} + +/* + * wts_create_database -- + * Create the database. */ void -wts_create(const char *home) +wts_create_database(void) { WT_CONNECTION *conn; - WT_DECL_RET; - size_t len; - char *cmd; - - len = strlen(g.home) * 3 + strlen(FORMAT_HOME_INIT_CMD) + 1; - cmd = dmalloc(len); - testutil_check(__wt_snprintf(cmd, len, FORMAT_HOME_INIT_CMD, g.home, g.home, g.home)); - if ((ret = system(cmd)) != 0) - testutil_die(ret, "home initialization (\"%s\") failed", cmd); - free(cmd); - - create_database(home, &conn); - create_object(conn); - if (g.c_in_memory != 0) - g.wts_conn_inmemory = conn; + + create_database(g.home, &conn); + + g.wts_conn = conn; + tables_apply(create_object, g.wts_conn); + if (GV(RUNS_IN_MEMORY) != 0) + g.wts_conn_inmemory = g.wts_conn; else testutil_check(conn->close(conn, NULL)); + g.wts_conn = NULL; } /* @@ -480,55 +467,22 @@ wts_open(const char *home, WT_CONNECTION **connp, WT_SESSION **sessionp, bool al max = sizeof(config); config[0] = '\0'; - enc = encryptor_at_open(g.c_encryption_flag); + /* Configuration settings that are not persistent between open calls. */ + enc = encryptor_at_open(); if (enc != NULL) CONFIG_APPEND(p, ",encryption=(name=%s)", enc); - /* - * Timing stress options aren't persisted in the base config and need to be added to the - * configuration for re-open. - */ - CONFIG_APPEND(p, ",timing_stress_for_test=["); - if (g.c_timing_stress_aggressive_sweep) - CONFIG_APPEND(p, ",aggressive_sweep"); - if (g.c_timing_stress_checkpoint) - CONFIG_APPEND(p, ",checkpoint_slow"); - if (g.c_timing_stress_checkpoint_prepare) - CONFIG_APPEND(p, ",prepare_checkpoint_delay"); - if (g.c_timing_stress_failpoint_hs_delete_key_from_ts) - CONFIG_APPEND(p, ",failpoint_history_store_delete_key_from_ts"); - if (g.c_timing_stress_failpoint_hs_insert_1) - CONFIG_APPEND(p, ",failpoint_history_store_insert_1"); - if (g.c_timing_stress_failpoint_hs_insert_2) - CONFIG_APPEND(p, ",failpoint_history_store_insert_2"); - if (g.c_timing_stress_hs_checkpoint_delay) - CONFIG_APPEND(p, ",history_store_checkpoint_delay"); - if (g.c_timing_stress_hs_search) - CONFIG_APPEND(p, ",history_store_search"); - if (g.c_timing_stress_hs_sweep) - CONFIG_APPEND(p, ",history_store_sweep_race"); - if (g.c_timing_stress_split_1) - CONFIG_APPEND(p, ",split_1"); - if (g.c_timing_stress_split_2) - CONFIG_APPEND(p, ",split_2"); - if (g.c_timing_stress_split_3) - CONFIG_APPEND(p, ",split_3"); - if (g.c_timing_stress_split_4) - CONFIG_APPEND(p, ",split_4"); - if (g.c_timing_stress_split_5) - CONFIG_APPEND(p, ",split_5"); - if (g.c_timing_stress_split_6) - CONFIG_APPEND(p, ",split_6"); - if (g.c_timing_stress_split_7) - CONFIG_APPEND(p, ",split_7"); - CONFIG_APPEND(p, "]"); + CONFIG_APPEND(p, ",error_prefix=\"%s\"", progname); + + /* Optional timing stress. */ + configure_timing_stress(p, max); /* If in-memory, there's only a single, shared WT_CONNECTION handle. */ - if (g.c_in_memory != 0) + if (GV(RUNS_IN_MEMORY) != 0) conn = g.wts_conn_inmemory; else { #if WIREDTIGER_VERSION_MAJOR >= 10 - if (g.c_verify && allow_verify) + if (GV(OPS_VERIFY) && allow_verify) CONFIG_APPEND(p, ",verify_metadata=true"); #else WT_UNUSED(allow_verify); @@ -540,6 +494,10 @@ wts_open(const char *home, WT_CONNECTION **connp, WT_SESSION **sessionp, bool al *connp = conn; } +/* + * wts_close -- + * Close the open database. + */ void wts_close(WT_CONNECTION **connp, WT_SESSION **sessionp) { @@ -561,34 +519,82 @@ wts_close(WT_CONNECTION **connp, WT_SESSION **sessionp) if (g.backward_compatible) testutil_check(conn->reconfigure(conn, "compatibility=(release=3.3)")); - testutil_check(conn->close(conn, g.c_leak_memory ? "leak_memory" : NULL)); + testutil_check(conn->close(conn, GV(WIREDTIGER_LEAK_MEMORY) ? "leak_memory" : NULL)); } +/* + * wts_verify -- + * Verify a table. + */ void -wts_verify(WT_CONNECTION *conn, const char *tag) +wts_verify(TABLE *table, void *arg) { + WT_CONNECTION *conn; WT_DECL_RET; WT_SESSION *session; - if (g.c_verify == 0) - return; - - track("verify", 0ULL, NULL); + conn = (WT_CONNECTION *)arg; - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - trace_msg("%s", "=============== verify start"); + if (GV(OPS_VERIFY) == 0) + return; /* * Verify can return EBUSY if the handle isn't available. Don't yield and retry, in the case of * LSM, the handle may not be available for a long time. */ - ret = session->verify(session, g.uri, "strict"); - testutil_assertfmt(ret == 0 || ret == EBUSY, "session.verify: %s: %s", g.uri, tag); - - trace_msg("%s", "=============== verify stop"); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + session->app_private = table->track_prefix; + ret = session->verify(session, table->uri, "strict"); + testutil_assert(ret == 0 || ret == EBUSY); testutil_check(session->close(session, NULL)); } +struct stats_args { + FILE *fp; + WT_SESSION *session; +}; + +/* + * stats_data_print -- + * Print out the statistics. + */ +static void +stats_data_print(WT_SESSION *session, const char *uri, FILE *fp) +{ + WT_CURSOR *cursor; + WT_DECL_RET; + uint64_t v; + const char *desc, *pval; + + wiredtiger_open_cursor(session, uri, NULL, &cursor); + while ( + (ret = cursor->next(cursor)) == 0 && (ret = cursor->get_value(cursor, &desc, &pval, &v)) == 0) + testutil_assert(fprintf(fp, "%s=%s\n", desc, pval) >= 0); + testutil_assert(ret == WT_NOTFOUND); + testutil_check(cursor->close(cursor)); +} + +/* + * stats_data_source -- + * Dump each data source's statistics. + */ +static void +stats_data_source(TABLE *table, void *arg) +{ + struct stats_args *args; + FILE *fp; + WT_SESSION *session; + char buf[1024]; + + args = arg; + fp = args->fp; + session = args->session; + + testutil_assert(fprintf(fp, "\n\n====== Data source statistics: %s\n", table->uri) >= 0); + testutil_check(__wt_snprintf(buf, sizeof(buf), "statistics:%s", table->uri)); + stats_data_print(session, buf, fp); +} + /* * wts_stats -- * Dump the run's statistics. @@ -596,59 +602,29 @@ wts_verify(WT_CONNECTION *conn, const char *tag) void wts_stats(void) { + struct stats_args args; FILE *fp; WT_CONNECTION *conn; - WT_CURSOR *cursor; - WT_DECL_RET; WT_SESSION *session; - size_t len; - uint64_t v; - char *stat_name; - const char *desc, *pval; /* Ignore statistics if they're not configured. */ - if (g.c_statistics == 0) + if (GV(STATISTICS) == 0) return; conn = g.wts_conn; - track("stat", 0ULL, NULL); - - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - - if ((fp = fopen(g.home_stats, "w")) == NULL) - testutil_die(errno, "fopen: %s", g.home_stats); + track("stat", 0ULL); /* Connection statistics. */ - fprintf(fp, "====== Connection statistics:\n"); - testutil_check(session->open_cursor(session, "statistics:", NULL, NULL, &cursor)); - - while ( - (ret = cursor->next(cursor)) == 0 && (ret = cursor->get_value(cursor, &desc, &pval, &v)) == 0) - if (fprintf(fp, "%s=%s\n", desc, pval) < 0) - testutil_die(errno, "fprintf"); - - if (ret != WT_NOTFOUND) - testutil_die(ret, "cursor.next"); - testutil_check(cursor->close(cursor)); + testutil_assert((fp = fopen(g.home_stats, "w")) != NULL); + testutil_assert(fprintf(fp, "====== Connection statistics:\n") >= 0); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + stats_data_print(session, "statistics:", fp); /* Data source statistics. */ - fprintf(fp, "\n\n====== Data source statistics:\n"); - len = strlen("statistics:") + strlen(g.uri) + 1; - stat_name = dmalloc(len); - testutil_check(__wt_snprintf(stat_name, len, "statistics:%s", g.uri)); - testutil_check(session->open_cursor(session, stat_name, NULL, NULL, &cursor)); - free(stat_name); - - while ( - (ret = cursor->next(cursor)) == 0 && (ret = cursor->get_value(cursor, &desc, &pval, &v)) == 0) - if (fprintf(fp, "%s=%s\n", desc, pval) < 0) - testutil_die(errno, "fprintf"); - - if (ret != WT_NOTFOUND) - testutil_die(ret, "cursor.next"); - testutil_check(cursor->close(cursor)); - - fclose_and_clear(&fp); + args.fp = fp; + args.session = session; + tables_apply(stats_data_source, &args); testutil_check(session->close(session, NULL)); + fclose_and_clear(&fp); } diff --git a/src/third_party/wiredtiger/test/suite/test_compact01.py b/src/third_party/wiredtiger/test/suite/test_compact01.py index b5a21ce96a6..ca1cbc9f22b 100755 --- a/src/third_party/wiredtiger/test/suite/test_compact01.py +++ b/src/third_party/wiredtiger/test/suite/test_compact01.py @@ -60,6 +60,17 @@ class test_compact(wttest.WiredTigerTestCase, suite_subprocess): conn_config = 'cache_size=1GB,eviction_checkpoint_target=80,' +\ 'eviction_dirty_target=80,eviction_dirty_trigger=95,statistics=(all)' + # Return stats that track the progress of compaction. + def getCompactProgressStats(self, uri): + cstat = self.session.open_cursor( + 'statistics:' + uri, None, 'statistics=(all)') + statDict = {} + statDict["pages_reviewed"] = cstat[stat.dsrc.btree_compact_pages_reviewed][2] + statDict["pages_skipped"] = cstat[stat.dsrc.btree_compact_pages_skipped][2] + statDict["pages_rewritten"] = cstat[stat.dsrc.btree_compact_pages_rewritten][2] + cstat.close() + return statDict + # Test compaction. def test_compact(self): # FIXME-WT-7187 @@ -103,6 +114,15 @@ class test_compact(wttest.WiredTigerTestCase, suite_subprocess): self.session.compact(uri, None) + # Verify compact progress stats. We can't do this with utility method as reopening the + # connection would reset the stats. + if self.utility == 0 and self.reopen == 0: + statDict = self.getCompactProgressStats(uri) + self.assertGreater(statDict["pages_reviewed"],0) + self.assertGreater(statDict["pages_rewritten"],0) + self.assertEqual(statDict["pages_rewritten"] + statDict["pages_skipped"], + statDict["pages_reviewed"]) + # Confirm compaction worked: check the number of on-disk pages self.reopen_conn() stat_cursor = self.session.open_cursor('statistics:' + uri, None, None) diff --git a/src/third_party/wiredtiger/test/suite/test_compact02.py b/src/third_party/wiredtiger/test/suite/test_compact02.py index 81d636eee6b..dc74ca4be71 100755 --- a/src/third_party/wiredtiger/test/suite/test_compact02.py +++ b/src/third_party/wiredtiger/test/suite/test_compact02.py @@ -79,6 +79,17 @@ class test_compact02(wttest.WiredTigerTestCase): fullsize = nrecords // 2 * len(bigvalue) + nrecords // 2 * len(smallvalue) + # Return stats that track the progress of compaction. + def getCompactProgressStats(self): + cstat = self.session.open_cursor( + 'statistics:' + self.uri, None, 'statistics=(all)') + statDict = {} + statDict["pages_reviewed"] = cstat[stat.dsrc.btree_compact_pages_reviewed][2] + statDict["pages_skipped"] = cstat[stat.dsrc.btree_compact_pages_skipped][2] + statDict["pages_rewritten"] = cstat[stat.dsrc.btree_compact_pages_rewritten][2] + cstat.close() + return statDict + # Return the size of the file def getSize(self): # To allow this to work on systems without ftruncate, @@ -101,7 +112,7 @@ class test_compact02(wttest.WiredTigerTestCase): conn_params = 'create,' + \ cacheSize + ',error_prefix="%s",' % self.shortid() + \ 'statistics=(all),' + \ - 'eviction_dirty_target=99,eviction_dirty_trigger=99' + 'eviction_dirty_target=80,eviction_dirty_trigger=99' try: self.conn = wiredtiger.wiredtiger_open(self.home, conn_params) except wiredtiger.WiredTigerError as e: @@ -110,6 +121,8 @@ class test_compact02(wttest.WiredTigerTestCase): # Create a table, add keys with both big and small values. def test_compact02(self): + mb = 1024 * 1024 + # FIXME-WT-7187 # This test is temporarily disabled for OS/X, it fails, but not consistently. import platform @@ -118,8 +131,9 @@ class test_compact02(wttest.WiredTigerTestCase): self.ConnectionOpen(self.cacheSize) - mb = 1024 * 1024 - params = 'key_format=i,value_format=S,' + self.fileConfig + # Set the leaf_value_max to ensure we never create overflow items. + # FIXME: WT-2298 + params = 'key_format=i,value_format=S,leaf_value_max=10MB,' + self.fileConfig # 1. Create a table with the data, alternating record size. self.session.create(self.uri, params) @@ -168,5 +182,12 @@ class test_compact02(wttest.WiredTigerTestCase): # After compact, the file size should be less than half the full size. self.assertLess(sz, self.fullsize // 2) + # Verify compact progress stats. + statDict = self.getCompactProgressStats() + self.assertGreater(statDict["pages_reviewed"],0) + self.assertGreater(statDict["pages_rewritten"],0) + self.assertEqual(statDict["pages_rewritten"] + statDict["pages_skipped"], + statDict["pages_reviewed"]) + if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_compact03.py b/src/third_party/wiredtiger/test/suite/test_compact03.py index 732bd39a597..19a1b4a22d5 100644 --- a/src/third_party/wiredtiger/test/suite/test_compact03.py +++ b/src/third_party/wiredtiger/test/suite/test_compact03.py @@ -79,6 +79,17 @@ class test_compact03(wttest.WiredTigerTestCase): expectedTableSize = 20 # Mbytes nOverflowRecords = 5000 + # Return stats that track the progress of compaction. + def getCompactProgressStats(self): + cstat = self.session.open_cursor( + 'statistics:' + self.uri, None, 'statistics=(all)') + statDict = {} + statDict["pages_reviewed"] = cstat[stat.dsrc.btree_compact_pages_reviewed][2] + statDict["pages_skipped"] = cstat[stat.dsrc.btree_compact_pages_skipped][2] + statDict["pages_rewritten"] = cstat[stat.dsrc.btree_compact_pages_rewritten][2] + cstat.close() + return statDict + # Return the size of the file def getSize(self): # To allow this to work on systems without ftruncate, @@ -147,6 +158,13 @@ class test_compact03(wttest.WiredTigerTestCase): self.pr('After deleting values and compactions ' + str(sizeAfterCompact // mb) + 'MB') self.assertGreater(sizeAfterCompact, (sizeWithOverflow // 10) * 9) + # Verify that we did indeed rewrote some pages but that didn't help with the file size. + statDict = self.getCompactProgressStats() + self.assertGreater(statDict["pages_reviewed"],0) + self.assertGreater(statDict["pages_rewritten"],0) + self.assertEqual(statDict["pages_rewritten"] + statDict["pages_skipped"], + statDict["pages_reviewed"]) + # 9. Insert some normal values and expect that file size won't increase as free extents # in the middle of the file will be used to write new data. diff --git a/src/third_party/wiredtiger/test/suite/test_config04.py b/src/third_party/wiredtiger/test/suite/test_config04.py index ab72ee139e5..51e5a4d4b7a 100755 --- a/src/third_party/wiredtiger/test/suite/test_config04.py +++ b/src/third_party/wiredtiger/test/suite/test_config04.py @@ -155,12 +155,12 @@ class test_config04(wttest.WiredTigerTestCase): def test_eviction_absolute(self): self.common_test('eviction_target=50MB,eviction_trigger=60MB,' - 'eviction_dirty_target=20MB,eviction_dirty_trigger=15MB,' + 'eviction_dirty_target=20MB,eviction_dirty_trigger=25MB,' 'eviction_checkpoint_target=13MB') def test_eviction_abs_and_pct(self): self.common_test('eviction_target=50,eviction_trigger=60MB,' - 'eviction_dirty_target=20,eviction_dirty_trigger=15MB') + 'eviction_dirty_target=20,eviction_dirty_trigger=25MB') def test_eviction_abs_less_than_one_pct(self): self.wiredtiger_open('.','create,cache_size=8GB,eviction_target=70MB,' @@ -203,18 +203,48 @@ class test_config04(wttest.WiredTigerTestCase): 'eviction_dirty_target=1G'), '/eviction dirty target should not exceed cache size/') - def test_eviction_dirty_trigggr_abs_too_large(self): + def test_eviction_dirty_trigger_abs_too_large(self): self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.wiredtiger_open('.','create,cache_size=500MB,' 'eviction_dirty_trigger=1G'), '/eviction dirty trigger should not exceed cache size/') + def test_eviction_dirty_trigger_abs_equal_to_dirty_target(self): + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: + self.wiredtiger_open('.','create,eviction_dirty_trigger=10MB,' + 'eviction_dirty_target=10MB'), + '/eviction dirty target must be lower than the eviction dirty trigger/') + + def test_eviction_dirty_trigger_abs_too_low(self): + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: + self.wiredtiger_open('.','create,eviction_dirty_trigger=9MB,' + 'eviction_dirty_target=10MB'), + '/eviction dirty target must be lower than the eviction dirty trigger/') + def test_eviction_checkpoint_tgt_abs_too_large(self): self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.wiredtiger_open('.','create,cache_size=500MB,' 'eviction_checkpoint_target=1G'), '/eviction checkpoint target should not exceed cache size/') + def test_eviction_updates_tgt_abs_too_large(self): + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: + self.wiredtiger_open('.','create,cache_size=500MB,' + 'eviction_updates_target=1G'), + '/eviction updates target should not exceed cache size/') + + def test_eviction_updates_trigger_abs_equal_to_updates_target(self): + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: + self.wiredtiger_open('.','create,eviction_updates_target=10MB,' + 'eviction_updates_trigger=10MB'), + '/eviction updates target must be lower than the eviction updates trigger/') + + def test_eviction_updates_trigger_abs_too_low(self): + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: + self.wiredtiger_open('.','create,eviction_updates_trigger=9MB,' + 'eviction_updates_target=10MB'), + '/eviction updates target must be lower than the eviction updates trigger/') + def test_invalid_config(self): msg = '/Unbalanced brackets/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, diff --git a/src/third_party/wiredtiger/test/suite/test_cursor17.py b/src/third_party/wiredtiger/test/suite/test_cursor17.py index becb6fcd5d3..cf186131091 100644 --- a/src/third_party/wiredtiger/test/suite/test_cursor17.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor17.py @@ -129,6 +129,20 @@ class test_cursor17(wttest.WiredTigerTestCase): self.session.rollback_transaction() session2.rollback_transaction() + + def test_aborted_insert(self): + self.populate(100) + + cursor = self.session.open_cursor(self.type + self.tablename, None) + self.session.begin_transaction() + cursor[200] = self.ds.value(200) + self.session.rollback_transaction() + + # Verify the largest key. + self.session.begin_transaction() + self.assertEqual(cursor.largest_key(), 0) + self.assertEquals(cursor.get_key(), 200) + self.session.rollback_transaction() def test_invisible_timestamp(self): self.populate(100) diff --git a/src/third_party/wiredtiger/test/suite/test_empty_value.py b/src/third_party/wiredtiger/test/suite/test_empty_value.py index be0f8a4e3be..2d7bce63309 100644 --- a/src/third_party/wiredtiger/test/suite/test_empty_value.py +++ b/src/third_party/wiredtiger/test/suite/test_empty_value.py @@ -30,13 +30,14 @@ # Smoke test empty row-store values. from wiredtiger import stat -import wiredtiger, wttest +import wiredtiger, wttest, unittest # Smoke test empty row-store values. class test_row_store_empty_values(wttest.WiredTigerTestCase): conn_config = 'statistics=(all)' # Smoke test empty row-store values. + @unittest.skip("Skipping, to be removed or updated in future ticket") def test_row_store_empty_values(self): nentries = 25000 uri = 'file:test_empty_values' # This is a btree layer test. diff --git a/src/third_party/wiredtiger/test/suite/test_hs05.py b/src/third_party/wiredtiger/test/suite/test_hs05.py index 6f869e83aaa..58cbfefccd7 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs05.py +++ b/src/third_party/wiredtiger/test/suite/test_hs05.py @@ -26,8 +26,7 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -from helper import copy_wiredtiger_home -import wiredtiger, wttest +import wttest from wiredtiger import stat from wtdataset import SimpleDataSet from wtscenario import make_scenarios @@ -38,8 +37,9 @@ from wtscenario import make_scenarios class test_hs05(wttest.WiredTigerTestCase): # Force a small cache, but disable eviction of dirty pages until the cache is full. conn_config = 'cache_size=50MB,statistics=(fast),' - conn_config += 'eviction_dirty_target=100,eviction_dirty_trigger=100,' - conn_config += 'eviction_updates_target=100,eviction_updates_trigger=100' + conn_config += 'eviction_trigger=100,' + conn_config += 'eviction_dirty_target=95,eviction_dirty_trigger=100,' + conn_config += 'eviction_updates_target=95,eviction_updates_trigger=100' session_config = 'isolation=snapshot' stable = 1 key_format_values = [ diff --git a/src/third_party/wiredtiger/test/suite/test_hs18.py b/src/third_party/wiredtiger/test/suite/test_hs18.py index 5ed21e3c90a..00fa6a96f5a 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs18.py +++ b/src/third_party/wiredtiger/test/suite/test_hs18.py @@ -56,6 +56,14 @@ class test_hs18(wttest.WiredTigerTestCase): sessions[i].begin_transaction() self.check_value(cursors[i], values[i]) + def evict_key(self, uri): + # Evict the update using a debug cursor + evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") + evict_cursor.set_key(self.create_key(1)) + self.assertEqual(evict_cursor.search(), 0) + evict_cursor.reset() + evict_cursor.close() + def test_base_scenario(self): uri = 'table:test_base_scenario' self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) @@ -89,11 +97,9 @@ class test_hs18(wttest.WiredTigerTestCase): cursor[self.create_key(1)] = value2 self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) - # Insert a bunch of contents to fill the cache - for i in range(2000, 10000): - self.session.begin_transaction() - cursor[self.create_key(i)] = value3 - self.session.commit_transaction() + # Evict the update using a debug cursor + cursor.reset() + self.evict_key(uri) # Commit an update without a timestamp on our original key self.session.begin_transaction() @@ -108,11 +114,9 @@ class test_hs18(wttest.WiredTigerTestCase): # Check our value is still correct. self.check_value(cursor2, value0) - # Insert a bunch of other contents to trigger eviction - for i in range(10001, 11000): - self.session.begin_transaction() - cursor[self.create_key(i)] = value3 - self.session.commit_transaction() + # Evict the update using a debug cursor + cursor.reset() + self.evict_key(uri) # Check our value is still correct. self.check_value(cursor2, value0) @@ -153,11 +157,9 @@ class test_hs18(wttest.WiredTigerTestCase): # Check our value is still correct. self.check_value(cursor3, value1) - # Insert a bunch of contents to fill the cache - for i in range(1000, 10000): - self.session.begin_transaction() - cursor[self.create_key(i)] = value3 - self.session.commit_transaction() + # Evict the update using a debug cursor + cursor.reset() + self.evict_key(uri) # Commit an update without a timestamp on our original key self.session.begin_transaction() @@ -173,11 +175,9 @@ class test_hs18(wttest.WiredTigerTestCase): self.check_value(cursor2, value1) self.check_value(cursor3, value1) - # Insert a bunch of other contents to trigger eviction - for i in range(10001, 20000): - self.session.begin_transaction() - cursor[self.create_key(i)] = value3 - self.session.commit_transaction() + # Evict the update using a debug cursor + cursor.reset() + self.evict_key(uri) # Check our value is still correct. self.check_value(cursor2, value1) @@ -218,11 +218,9 @@ class test_hs18(wttest.WiredTigerTestCase): cursor[self.create_key(1)] = value2 self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) - # Insert a bunch of other contents to trigger eviction - for i in range(2, 10000): - self.session.begin_transaction() - cursor[self.create_key(i)] = value3 - self.session.commit_transaction() + # Evict the update using a debug cursor + cursor.reset() + self.evict_key(uri) # Check our value is still correct. self.check_value(cursor2, value0) @@ -232,11 +230,9 @@ class test_hs18(wttest.WiredTigerTestCase): cursor[self.create_key(1)] = value4 self.session.commit_transaction() - # Insert a bunch of other contents to trigger eviction - for i in range(10000, 11000): - self.session.begin_transaction() - cursor[self.create_key(i)] = value3 - self.session.commit_transaction() + # Evict the update using a debug cursor + cursor.reset() + self.evict_key(uri) # Check our value is still correct. self.check_value(cursor2, value0) @@ -256,8 +252,6 @@ class test_hs18(wttest.WiredTigerTestCase): cursors.append(sessions[i].open_cursor(uri)) values.append(str(i) * 10) - value_junk = 'aaaaa' * 100 - # Insert an update at timestamp 3 self.session.begin_transaction() cursor[self.create_key(1)] = values[0] @@ -282,11 +276,9 @@ class test_hs18(wttest.WiredTigerTestCase): # Start a transaction that will see update 2. self.start_txn(sessions, cursors, values, 2) - # Insert a bunch of other contents to trigger eviction - for i in range(1000, 10000): - self.session.begin_transaction() - cursor[self.create_key(i)] = value_junk - self.session.commit_transaction() + # Evict the update using a debug cursor + cursor.reset() + self.evict_key(uri) # Commit an update without a timestamp on our original key self.session.begin_transaction() @@ -301,11 +293,9 @@ class test_hs18(wttest.WiredTigerTestCase): cursor[self.create_key(1)] = values[4] self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(15)) - # Insert a bunch of other contents to trigger eviction - for i in range(10001, 20000): - self.session.begin_transaction() - cursor[self.create_key(i)] = value_junk - self.session.commit_transaction() + # Evict the update using a debug cursor + cursor.reset() + self.evict_key(uri) # Validate all values are visible and correct. for i in range(0, 3): @@ -328,8 +318,6 @@ class test_hs18(wttest.WiredTigerTestCase): cursors.append(sessions[i].open_cursor(uri)) values.append(str(i) * 10) - value_junk = 'aaaaa' * 100 - # Insert an update at timestamp 3 self.session.begin_transaction() cursor[self.create_key(1)] = values[0] @@ -354,11 +342,9 @@ class test_hs18(wttest.WiredTigerTestCase): # Start a transaction that will see update 2. self.start_txn(sessions, cursors, values, 2) - # Insert a bunch of other contents to trigger eviction - for i in range(1000, 10000): - self.session.begin_transaction() - cursor[self.create_key(i)] = value_junk - self.session.commit_transaction() + # Evict the update using a debug cursor + cursor.reset() + self.evict_key(uri) # Commit an update without a timestamp on our original key self.session.begin_transaction() @@ -392,11 +378,9 @@ class test_hs18(wttest.WiredTigerTestCase): # Start a transaction that will see update 6. self.start_txn(sessions, cursors, values, 6) - # Insert a bunch of other contents to trigger eviction - for i in range(10001, 20000): - self.session.begin_transaction() - cursor[self.create_key(i)] = value_junk - self.session.commit_transaction() + # Evict the update using a debug cursor + cursor.reset() + self.evict_key(uri) # Validate all values are visible and correct. for i in range(0, 6): @@ -418,11 +402,9 @@ class test_hs18(wttest.WiredTigerTestCase): cursor[self.create_key(1)] = values[8] self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) - # Insert a bunch of other contents to trigger eviction - for i in range(10001, 20000): - self.session.begin_transaction() - cursor[self.create_key(i)] = values[3] - self.session.commit_transaction() + # Evict the update using a debug cursor + cursor.reset() + self.evict_key(uri) # Validate all values are visible and correct. for i in range(0, 7): @@ -483,11 +465,7 @@ class test_hs18(wttest.WiredTigerTestCase): # Evict the update using a debug cursor cursor.reset() - evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") - evict_cursor.set_key(self.create_key(1)) - self.assertEqual(evict_cursor.search(), 0) - evict_cursor.reset() - evict_cursor.close() + self.evict_key(uri) # Commit a modify without a timestamp on our original key self.session.begin_transaction() @@ -511,11 +489,7 @@ class test_hs18(wttest.WiredTigerTestCase): # Evict the update using a debug cursor cursor.reset() - evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") - evict_cursor.set_key(self.create_key(1)) - self.assertEqual(evict_cursor.search(), 0) - evict_cursor.reset() - evict_cursor.close() + self.evict_key(uri) # Check our values are still correct. for i in range(0, 5): diff --git a/src/third_party/wiredtiger/test/suite/test_prepare06.py b/src/third_party/wiredtiger/test/suite/test_prepare06.py index de9a7475817..b4e65f183b0 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare06.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare06.py @@ -90,18 +90,24 @@ class test_prepare06(wttest.WiredTigerTestCase, suite_subprocess): s_reader = self.conn.open_session() s_reader.begin_transaction('read_timestamp=' + self.timestamp_str(40)) - # It is legal to set the prepare timestamp earlier than an active - # read timestamp with roundup_timestamps settings. This is only + # It is illegal to set the prepare timestamp as earlier than an active + # read timestamp even with roundup_timestamps settings. This is only # checked in diagnostic builds. if wiredtiger.diagnostic_build(): self.session.begin_transaction('roundup_timestamps=(prepared=true)') - self.assertEqual(self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(10)), 0) + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.prepare_transaction( + 'prepare_timestamp=' + self.timestamp_str(10)), + "/must be greater than the latest active read timestamp/") self.session.rollback_transaction() - # It is legal to set the prepare timestamp the same as an active read - # timestamp with roundup_timestamps settings. + # It is illegal to set the prepare timestamp the same as an active read + # timestamp even with roundup_timestamps settings. self.session.begin_transaction('roundup_timestamps=(prepared=true)') - self.assertEqual(self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(40)), 0) + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.prepare_transaction( + 'prepare_timestamp=' + self.timestamp_str(40)), + "/must be greater than the latest active read timestamp/") self.session.rollback_transaction() ''' @@ -138,7 +144,5 @@ class test_prepare06(wttest.WiredTigerTestCase, suite_subprocess): s_reader.commit_transaction() - self.ignoreStdoutPatternIfExists("Skip checking prepare timestamp") - if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_prepare17.py b/src/third_party/wiredtiger/test/suite/test_prepare17.py new file mode 100644 index 00000000000..ae514b3865f --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_prepare17.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +from wtscenario import make_scenarios +from helper import simulate_crash_restart + +# test_prepare17.py +# The following test is to verify that if the out of order commit timestamp(for a transaction, say T2) lies between +# previous commit and durable timestamps(for a transaction, say T1), the durable timestamp of T1 changes to +# the commit timestamp of T2. +class test_prepare17(wttest.WiredTigerTestCase): + session_config = 'isolation=snapshot' + uri = 'table:test_prepare17' + nrows = 1000 + value1 = 'aaaaa' + value2 = 'bbbbb' + + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] + update = [ + ('prepare', dict(prepare=True)), + ('non-prepare', dict(prepare=False)), + ] + scenarios = make_scenarios(key_format_values, update) + + def test_prepare(self): + create_params = 'key_format={},value_format=S'.format(self.key_format) + self.session.create(self.uri, create_params) + cursor = self.session.open_cursor(self.uri) + + # Transaction one + self.session.begin_transaction() + for i in range(1, self.nrows + 1): + cursor[i] = self.value1 + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)+ ',durable_timestamp=' + self.timestamp_str(6)) + + # In the case below, the commit timestamp lies between the previous commit and durable timestamps. + # Internally, WiredTiger changes the durable timestamp of Transaction one, i.e. 6 to the commit timestamp + # of the transaction below, i.e, 4. + # As per the time window validation the commit timestamp cannot be in between any previous commit and + # durable timestamps. + # + # Note: The scenario where commit timestamp lies between the previous commit and durable timestamps + # is not expected from MongoDB, but WiredTiger API can allow it. + if self.prepare: + self.session.begin_transaction() + for i in range(1, self.nrows + 1): + cursor[i] = self.value2 + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4) + ',durable_timestamp=' + self.timestamp_str(7)) + else: + self.session.begin_transaction() + for i in range(1, self.nrows + 1): + cursor[i] = self.value2 + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) + + # Time window validation occurs as part of checkpoint. + self.session.checkpoint() + + def test_prepare_insert_remove(self): + if not self.prepare: + return + + create_params = 'key_format={},value_format=S'.format(self.key_format) + self.session.create(self.uri, create_params) + cursor = self.session.open_cursor(self.uri) + + self.session.begin_transaction() + for i in range(1, self.nrows + 1): + cursor[i] = self.value1 + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) + + # Transaction 2 + self.session.begin_transaction() + for i in range(1, self.nrows + 1): + cursor[i] = self.value2 + cursor.set_key(i) + cursor.remove() + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4) + ',durable_timestamp=' + self.timestamp_str(7)) + + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(6)) + self.session.checkpoint() + simulate_crash_restart(self, ".", "RESTART") + + # Update in Transaction2 should be removed as the stable timestamp (6) is less than the durable timestamp (7). + cursor = self.session.open_cursor(self.uri) + self.session.begin_transaction() + for i in range(1, self.nrows + 1): + cursor.set_key(i) + cursor.search() + self.assertEqual(cursor.get_value(), self.value1) + self.session.rollback_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable18.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable18.py index d34e4ffb08b..01e0898a973 100644 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable18.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable18.py @@ -31,8 +31,6 @@ # aggregated_time_windows # [END_TAGS] -import fnmatch, os, shutil, time -from helper import simulate_crash_restart from test_rollback_to_stable01 import test_rollback_to_stable_base from wiredtiger import stat from wtdataset import SimpleDataSet @@ -56,7 +54,8 @@ class test_rollback_to_stable18(test_rollback_to_stable_base): scenarios = make_scenarios(key_format_values, prepare_values) def conn_config(self): - config = 'cache_size=50MB,in_memory=true,statistics=(all),log=(enabled=false),eviction_dirty_trigger=5,eviction_updates_trigger=5' + config = 'cache_size=50MB,in_memory=true,statistics=(all),log=(enabled=false),' \ + 'eviction_dirty_trigger=10,eviction_updates_trigger=10' return config def test_rollback_to_stable(self): diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable19.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable19.py index 3f10b186b47..c07895c6316 100644 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable19.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable19.py @@ -26,7 +26,6 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import fnmatch, os, shutil, time from helper import simulate_crash_restart from test_rollback_to_stable01 import test_rollback_to_stable_base from wiredtiger import stat, WT_NOTFOUND @@ -56,7 +55,8 @@ class test_rollback_to_stable19(test_rollback_to_stable_base): scenarios = make_scenarios(in_memory_values, key_format_values, restart_options) def conn_config(self): - config = 'cache_size=50MB,statistics=(all),log=(enabled=false),eviction_dirty_trigger=5,eviction_updates_trigger=5' + config = 'cache_size=50MB,statistics=(all),log=(enabled=false),eviction_dirty_trigger=10,' \ + 'eviction_updates_trigger=10' if self.in_memory: config += ',in_memory=true' else: diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable28.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable28.py index 50a8a8fb2fb..42f6861ca74 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable28.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable28.py @@ -26,9 +26,8 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import os, shutil, re +import re from wiredtiger import stat -import wttest from wtdataset import SimpleDataSet from helper import simulate_crash_restart from test_rollback_to_stable01 import test_rollback_to_stable_base @@ -44,8 +43,8 @@ class test_rollback_to_stable28(test_rollback_to_stable_base): # Recovery connection config: The debug mode is only effective on high cache pressure as WiredTiger can potentially decide # to do an update restore evict on a page when the cache pressure requirements are not met. # This means setting eviction target low and cache size high. - conn_recon = ',eviction_updates_trigger=10,eviction_dirty_trigger=5,cache_size=10MB,' + \ - 'debug_mode=(update_restore_evict=true),log=(recover=on)' + conn_recon = ',eviction_updates_trigger=10,eviction_dirty_trigger=5,eviction_dirty_target=1,' \ + 'cache_size=10MB,debug_mode=(update_restore_evict=true),log=(recover=on)' def parse_write_gen(self, uri): meta_cursor = self.session.open_cursor('metadata:') diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable29.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable29.py new file mode 100755 index 00000000000..d40ef12d7c8 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable29.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import os, threading, time +from wtthread import checkpoint_thread +import wiredtiger +from wiredtiger import stat +from wtdataset import SimpleDataSet +from wtscenario import make_scenarios +from helper import copy_wiredtiger_home, simulate_crash_restart +from test_rollback_to_stable01 import test_rollback_to_stable_base + +# test_rollback_to_stable29.py +# Test that the rollback to stable to verify the history store order when an out of order to a tombstone. +class test_rollback_to_stable29(test_rollback_to_stable_base): + conn_config = 'cache_size=25MB,statistics=(all),statistics_log=(json,on_close,wait=1),log=(enabled=true)' + + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + + scenarios = make_scenarios(key_format_values) + + def test_rollback_to_stable(self): + uri = 'table:test_rollback_to_stable29' + nrows = 100 + + # Create our table. + ds = SimpleDataSet(self, uri, 0, key_format=self.key_format, value_format='S',config='log=(enabled=false)') + ds.populate() + + value_a = 'a' * 100 + value_b = 'b' * 100 + value_c = 'c' * 100 + value_d = 'd' * 100 + + # Pin oldest and stable to timestamp 1. + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) + + self.large_updates(uri, value_a, ds, nrows, False, 10) + self.large_removes(uri, ds, nrows, False, 30) + self.large_updates(uri, value_b, ds, nrows, False, 40) + self.check(value_b, uri, nrows, 40) + self.large_updates(uri, value_c, ds, nrows, False, 50) + self.check(value_c, uri, nrows, 50) + self.evict_cursor(uri, nrows, value_c) + + # Insert an out of order update. + self.session.breakpoint() + self.large_updates(uri, value_d, ds, nrows, False, 20) + + self.check(value_a, uri, nrows, 10) + self.check(value_d, uri, nrows, 40) + self.check(value_d, uri, nrows, 50) + self.check(value_d, uri, nrows, 20) + + # Pin stable to timestamp 10. + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(10)) + self.session.checkpoint() + + # Simulate a crash by copying to a new directory(RESTART). + simulate_crash_restart(self, ".", "RESTART") + + self.check(value_a, uri, nrows, 10) + + stat_cursor = self.session.open_cursor('statistics:', None, None) + hs_removed = stat_cursor[stat.conn.txn_rts_hs_removed][2] + stat_cursor.close() + + self.assertGreaterEqual(hs_removed, 3 * nrows) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_search_near01.py b/src/third_party/wiredtiger/test/suite/test_search_near01.py index 6df71b438ff..058704f9d1b 100755 --- a/src/third_party/wiredtiger/test/suite/test_search_near01.py +++ b/src/third_party/wiredtiger/test/suite/test_search_near01.py @@ -100,11 +100,11 @@ class test_search_near01(wttest.WiredTigerTestCase): cursor2.reconfigure("prefix_search=true") cursor2.set_key('aa') - cursor2.search_near() + self.assertEqual(cursor2.search_near(), wiredtiger.WT_NOTFOUND) prefix_skip_count = self.get_stat(stat.conn.cursor_next_skip_lt_100) - # We should've skipped ~26*2 here as we're only looking at the "aa" range * 2. - self.assertGreaterEqual(prefix_skip_count - skip_count, 26*2) + # We should've skipped ~26 here as we're only looking at the "aa" range. + self.assertGreaterEqual(prefix_skip_count - skip_count, 26) skip_count = prefix_skip_count # The prefix code will have come into play at once as we walked to "aba". The prev @@ -114,15 +114,16 @@ class test_search_near01(wttest.WiredTigerTestCase): # Search for a key not at the start. cursor2.set_key('bb') - cursor2.search_near() + self.assertEqual(cursor2.search_near(), wiredtiger.WT_NOTFOUND) - # Assert it to have only incremented the skipped statistic ~26*2 times. + # Assert it to have only incremented the skipped statistic ~26 times. prefix_skip_count = self.get_stat(stat.conn.cursor_next_skip_lt_100) - self.assertGreaterEqual(prefix_skip_count - skip_count, 26*2) + self.assertGreaterEqual(prefix_skip_count - skip_count, 26) skip_count = prefix_skip_count - # Here we should've hit the prefix fast path code twice. Plus the time we already did. - self.assertEqual(self.get_stat(stat.conn.cursor_search_near_prefix_fast_paths), 2+1) + # Here we should have hit the prefix fast path code twice, as we have called prefix + # search near twice, both of which should have early exited when going forwards. + self.assertEqual(self.get_stat(stat.conn.cursor_search_near_prefix_fast_paths), 2) cursor2.close() cursor2 = session2.open_cursor(uri) @@ -192,19 +193,20 @@ class test_search_near01(wttest.WiredTigerTestCase): cursor2.search_near() skip_count = self.get_stat(stat.conn.cursor_next_skip_lt_100) - # This should be equal to roughly key_count * 2 as we're going to traverse most of the - # range forward, and then the whole range backwards. - self.assertGreater(skip_count, key_count * 2) + # This should be slightly greater than key_count as we're going to traverse most of the + # range forwards. + self.assertGreater(skip_count, key_count) + self.assertEqual(self.get_stat(stat.conn.cursor_search_near_prefix_fast_paths), 0) cursor2.reconfigure("prefix_search=true") cursor2.set_key('cc') - cursor2.search_near() - self.assertEqual(self.get_stat(stat.conn.cursor_search_near_prefix_fast_paths), 2) + self.assertEqual(cursor2.search_near(), wiredtiger.WT_NOTFOUND) + self.assertEqual(self.get_stat(stat.conn.cursor_search_near_prefix_fast_paths), 1) # This still isn't visible to our older reader and as such we expect this statistic to - # increment twice. + # increment again. self.unique_insert(cursor2, 'cc', cc_id, keys) - self.assertEqual(self.get_stat(stat.conn.cursor_search_near_prefix_fast_paths), 4) + self.assertEqual(self.get_stat(stat.conn.cursor_search_near_prefix_fast_paths), 2) # In order for prefix key fast pathing to work we rely on some guarantees provided by row # search. Test some of the guarantees. @@ -297,34 +299,38 @@ class test_search_near01(wttest.WiredTigerTestCase): # Search near for the "aa" part of the range. cursor2 = session2.open_cursor(uri) cursor2.set_key('c') - cursor2.search_near() + self.assertEqual(cursor2.search_near(), wiredtiger.WT_NOTFOUND) skip_count = self.get_stat(stat.conn.cursor_next_skip_lt_100, session2) - # This should be equal to roughly key_count * 2 as we're going to traverse the whole - # range forward, and then the whole range backwards. + # This should be equal to roughly key_count as we're going to traverse the whole + # range forwards. self.assertGreater(skip_count, key_count) cursor2.reconfigure("prefix_search=true") cursor2.set_key('c') - cursor2.search_near() + self.assertEqual(cursor2.search_near(), wiredtiger.WT_NOTFOUND) prefix_skip_count = self.get_stat(stat.conn.cursor_next_skip_lt_100, session2) - self.assertEqual(prefix_skip_count - skip_count, 3) + # We expect to traverse one entry and have a buffer to account for anomalies. + self.assertEqual(prefix_skip_count - skip_count, 2) skip_count = prefix_skip_count - self.assertEqual(self.get_stat(stat.conn.cursor_search_near_prefix_fast_paths, session2), 2) + # We early exit here as "cc" is not the last key. + self.assertEqual(self.get_stat(stat.conn.cursor_search_near_prefix_fast_paths, session2), 1) session2.rollback_transaction() session2.begin_transaction('ignore_prepare=true') cursor4 = session2.open_cursor(uri) cursor4.reconfigure("prefix_search=true") cursor4.set_key('c') - cursor4.search_near() + self.assertEqual(cursor4.search_near(), 1) prefix_skip_count = self.get_stat(stat.conn.cursor_next_skip_lt_100, session2) + # We expect to traverse one entry and have a buffer to account for anomalies. self.assertEqual(prefix_skip_count - skip_count, 2) skip_count = prefix_skip_count cursor4.reconfigure("prefix_search=false") cursor4.set_key('c') - cursor4.search_near() + ret = cursor4.search_near() + self.assertTrue(ret == -1 or ret == 1) self.assertEqual(self.get_stat(stat.conn.cursor_next_skip_lt_100, session2) - skip_count, 2) diff --git a/src/third_party/wiredtiger/test/suite/test_search_near04.py b/src/third_party/wiredtiger/test/suite/test_search_near04.py new file mode 100644 index 00000000000..4080ae82449 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_search_near04.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +import wttest +from wtscenario import make_scenarios + +# test_search_near04.py +# This test checks that a search_near call with the prefix key +# configuration will correctly find a key even in cases where the key +# range is split across multiple pages. +class test_search_near04(wttest.WiredTigerTestCase): + key_format_values = [ + ('var_string', dict(key_format='S')), + ('byte_array', dict(key_format='u')), + ] + + scenarios = make_scenarios(key_format_values) + + def check_key(self, key): + if self.key_format == 'u': + return key.encode() + else: + return key + + def test_search_near(self): + uri = 'table:test_search_near' + self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) + + # Make the keys big enough to span over multiple pages. + # key_size can be set to to a lower value so only one page is used and search_near works. + key_size = 200 + + cursor = self.session.open_cursor(uri) + cursor2 = self.session.open_cursor(uri, None, "debug=(release_evict=true)") + + # Basic character array. + l = "abcdefghijklmnopqrstuvwxyz" + + # Insert keys aaa -> aaz with timestamp 200. + prefix = "aa" + self.session.begin_transaction() + for k in range (0, 25): + key = prefix + l[k] + cursor[key * key_size] = key * key_size + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(200)) + + # Insert key aaz with timestamp 50. + self.session.begin_transaction() + key = prefix + "z" + cursor[key * key_size] = key * key_size + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(50)) + + # Evict the whole range. + # If eviction is not performed, things stay in memory and it works fine. + for k in range (0, 26): + cursor2.set_key((prefix + l[k]) * key_size) + self.assertEqual(cursor2.search(), 0) + self.assertEqual(cursor2.reset(), 0) + + # Start a transaction at timestamp 100, aaz should be the only key that is visible. + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(100)) + cursor3 = self.session.open_cursor(uri) + + # Prefix search is disabled by default. + # Search near should always return the only visible key. + expected_key = "aaz" * key_size + cursor3.set_key("aa") + self.assertEqual(cursor3.search_near(), 1) + self.assertEqual(cursor3.get_key(), self.check_key(expected_key)) + + cursor3.set_key("az") + self.assertEqual(cursor3.search_near(), -1) + self.assertEqual(cursor3.get_key(), self.check_key(expected_key)) + + cursor3.set_key("aaz" * key_size) + self.assertEqual(cursor3.search_near(), 0) + self.assertEqual(cursor3.get_key(), self.check_key(expected_key)) + + cursor3.set_key("aazab") + self.assertEqual(cursor3.search_near(), -1) + self.assertEqual(cursor3.get_key(), self.check_key(expected_key)) + + # Enable prefix search. + cursor3.reconfigure("prefix_search=true") + + # The only visible key is aaz. + # If we try to do a search_near() with the prefixes "a" or "aa" without the changes + # introduced in WT-7912, we fail to find the key aaz although it is a valid result. + # This is because we traverse off the page and early exit before seeing the visible + # key that is on another page. However, if we specify "aaz" as a prefix, we are + # able to find that as we are traversing on the same page. + # All three of the prefixes "a", "aa" and "aaz" should lead us to find "aaz". + cursor3.set_key("a") + self.assertEqual(cursor3.search_near(), 1) + self.assertEqual(cursor3.get_key(), self.check_key(expected_key)) + + cursor3.set_key("aa") + self.assertEqual(cursor3.search_near(), 1) + self.assertEqual(cursor3.get_key(), self.check_key(expected_key)) + + cursor3.set_key("aaz") + self.assertEqual(cursor3.search_near(), 1) + self.assertEqual(cursor3.get_key(), self.check_key(expected_key)) + + cursor3.set_key("aaz" * key_size) + self.assertEqual(cursor3.search_near(), 0) + self.assertEqual(cursor3.get_key(), self.check_key(expected_key)) + + cursor3.close() + self.session.commit_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_tiered03.py b/src/third_party/wiredtiger/test/suite/test_tiered03.py index 5a57e95e0ae..2b86474c1d2 100755 --- a/src/third_party/wiredtiger/test/suite/test_tiered03.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered03.py @@ -79,6 +79,9 @@ class test_tiered03(wttest.WiredTigerTestCase): # Test sharing data between a primary and a secondary def test_sharing(self): + # FIXME: WT-8235 Enable the test once file containing transaction ids is supported. + self.skipTest('Sharing the checkpoint file containing transaction ids is not supported') + ds = SimpleDataSet(self, self.uri, 10) ds.populate() ds.check() diff --git a/src/third_party/wiredtiger/test/suite/test_tiered04.py b/src/third_party/wiredtiger/test/suite/test_tiered04.py index 05fbbc44a5e..03d28891b0f 100755 --- a/src/third_party/wiredtiger/test/suite/test_tiered04.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered04.py @@ -181,9 +181,16 @@ class test_tiered04(wttest.WiredTigerTestCase): obj = self.get_stat(stat.conn.tiered_object_size, None) self.assertEqual(obj, self.object_sys_val) - # As we flush each object, we are currently removing the file: object. So N + 1 exists. + # As we flush each object, the next object exists, so N + 1 flushes is the last + # object number that exists (as the file:). + last = 'last=' + str(flush + 1) + # For now all earlier objects exist. So it is always 1 until garbage collection + # starts removing them. + oldest = 'oldest=1' fileuri = self.fileuri_base + str(flush + 1) + '.wtobj' self.check_metadata(self.tiereduri, intl_page) + self.check_metadata(self.tiereduri, last) + self.check_metadata(self.tiereduri, oldest) self.check_metadata(fileuri, intl_page) self.check_metadata(self.objuri, intl_page) diff --git a/src/third_party/wiredtiger/test/suite/test_tiered09.py b/src/third_party/wiredtiger/test/suite/test_tiered09.py new file mode 100755 index 00000000000..e44b828bfd9 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_tiered09.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import os, time, wiredtiger, wttest +from wiredtiger import stat +StorageSource = wiredtiger.StorageSource # easy access to constants + +# test_tiered09.py +# Test tiered storage with sequential connections with different prefixes. +class test_tiered09(wttest.WiredTigerTestCase): + + # If the 'uri' changes all the other names must change with it. + base = 'test_tiered09-000000000' + base2 = 'test_second09-000000000' + fileuri_base = 'file:' + base + obj1file = base + '1.wtobj' + obj1second = base2 + '1.wtobj' + obj2file = base + '2.wtobj' + obj3file = base + '3.wtobj' + objuri = 'object:' + base + '1.wtobj' + tiereduri = "tiered:test_tiered09" + uri = "table:test_tiered09" + uri2 = "table:test_second09" + + auth_token = "test_token" + bucket = "mybucket" + extension_name = "local_store" + prefix1 = "1_" + prefix2 = "2_" + prefix3 = "3_" + retention = 1 + saved_conn = '' + def conn_config(self): + os.mkdir(self.bucket) + self.saved_conn = \ + 'statistics=(all),' + \ + 'tiered_storage=(auth_token=%s,' % self.auth_token + \ + 'bucket=%s,' % self.bucket + \ + 'bucket_prefix=%s,' % self.prefix1 + \ + 'local_retention=%d,' % self.retention + \ + 'name=%s)' % self.extension_name + return self.saved_conn + + # Load the local store extension. + def conn_extensions(self, extlist): + # Windows doesn't support dynamically loaded extension libraries. + if os.name == 'nt': + extlist.skip_if_missing = True + extlist.extension('storage_sources', self.extension_name) + + def check(self, tc, n): + for i in range(0, n): + self.assertEqual(tc[str(i)], str(i)) + tc.set_key(str(n)) + self.assertEquals(tc.search(), wiredtiger.WT_NOTFOUND) + + # Test calling the flush_tier API. + def test_tiered(self): + # Create a table. Add some data. Checkpoint and flush tier. + # Close the connection. Then we want to reopen the connection + # with a different bucket prefix and repeat. Then reopen the + # connection with the original prefix. Then reopen and verify + # we can read all the data. + # + # Verify the files are as we expect also. We expect: + # 1_<tablename>-00000001.wtobj + # 2_<tablename>-00000002.wtobj + # 1_<tablename>-00000003.wtobj + # but we can read and access all data in all objects. + self.session.create(self.uri, 'key_format=S,value_format=S,') + # Add first data. Checkpoint, flush and close the connection. + c = self.session.open_cursor(self.uri) + c["0"] = "0" + self.check(c, 1) + c.close() + self.session.checkpoint() + self.session.flush_tier(None) + self.close_conn() + self.assertTrue(os.path.exists(self.obj1file)) + self.assertTrue(os.path.exists(self.obj2file)) + bucket_obj = self.bucket + '/' + self.prefix1 + self.obj1file + self.assertTrue(os.path.exists(bucket_obj)) + # Since we've closed and reopened the connection we lost the work units + # to drop the local objects. Clean them up now to make sure we can open + # the correct object in the bucket. + localobj = './' + self.obj1file + os.remove(localobj) + + # Reopen the connection with a different prefix this time. + conn_params = self.saved_conn + ',tiered_storage=(bucket_prefix=%s)' % self.prefix2 + self.conn = self.wiredtiger_open('.', conn_params) + self.session = self.conn.open_session() + # Add a second table created while the second prefix is used for the connection. + self.session.create(self.uri2, 'key_format=S,value_format=S,') + # Add first data. Checkpoint, flush and close the connection. + c = self.session.open_cursor(self.uri2) + c["0"] = "0" + self.check(c, 1) + c.close() + # Add more data to original table. + # Checkpoint, flush and close the connection. + c = self.session.open_cursor(self.uri) + c["1"] = "1" + self.check(c, 2) + c.close() + self.session.checkpoint() + self.session.flush_tier(None) + self.close_conn() + # Check each table was created with the correct prefix. + bucket_obj = self.bucket + '/' + self.prefix2 + self.obj1second + self.assertTrue(os.path.exists(bucket_obj)) + bucket_obj = self.bucket + '/' + self.prefix1 + self.obj2file + self.assertTrue(os.path.exists(bucket_obj)) + # Since we've closed and reopened the connection we lost the work units + # to drop the local objects. Clean them up now to make sure we can open + # the correct object in the bucket. + localobj = './' + self.obj2file + os.remove(localobj) + localobj = './' + self.obj1second + + # Reopen with the other prefix and check all data. Even though we're using the + # other prefix, we should find all the data in the object with the original + # prefix. + conn_params = self.saved_conn + ',tiered_storage=(bucket_prefix=%s)' % self.prefix3 + self.conn = self.wiredtiger_open('.', conn_params) + self.session = self.conn.open_session() + c = self.session.open_cursor(self.uri) + self.check(c, 2) + c.close() + c = self.session.open_cursor(self.uri2) + self.check(c, 1) + c.close() + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_tiered10.py b/src/third_party/wiredtiger/test/suite/test_tiered10.py new file mode 100755 index 00000000000..a6b3e5dc56c --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_tiered10.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import os, time, wiredtiger, wttest +from wiredtiger import stat +StorageSource = wiredtiger.StorageSource # easy access to constants + +# test_tiered10.py +# Test tiered storage with simultaneous connections using different +# prefixes to the same bucket directory but different local databases. +class test_tiered10(wttest.WiredTigerTestCase): + + # If the 'uri' changes all the other names must change with it. + base = 'test_tiered10-000000000' + fileuri_base = 'file:' + base + obj1file = base + '1.wtobj' + objuri = 'object:' + base + '1.wtobj' + tiereduri = "tiered:test_tiered10" + uri = "table:test_tiered10" + + auth_token = "test_token" + bucket = "mybucket" + bucket1 = "../" + bucket + bucket2 = "../" + bucket + conn1_dir = "first_dir" + conn2_dir = "second_dir" + extension_name = "local_store" + prefix1 = "1_" + prefix2 = "2_" + retention = 1 + saved_conn = '' + def conn_config(self): + os.mkdir(self.bucket) + os.mkdir(self.conn1_dir) + os.mkdir(self.conn2_dir) + # Use this to create the directories and set up for the others. + dummy_conn = 'create,statistics=(all),' + self.saved_conn = \ + 'create,statistics=(all),' + \ + 'tiered_storage=(auth_token=%s,' % self.auth_token + \ + 'bucket=../%s,' % self.bucket + \ + 'local_retention=%d,' % self.retention + \ + 'name=%s),' % self.extension_name + return dummy_conn + + # Load the local store extension. + def conn_extensions(self, extlist): + # Windows doesn't support dynamically loaded extension libraries. + if os.name == 'nt': + extlist.skip_if_missing = True + extlist.extension('storage_sources', self.extension_name) + + def check(self, tc, base, n): + for i in range(base, n): + self.assertEqual(tc[str(i)], str(i)) + tc.set_key(str(n)) + self.assertEquals(tc.search(), wiredtiger.WT_NOTFOUND) + + # Test calling the flush_tier API. + def test_tiered(self): + # Have two connections running in different directories, but sharing + # the same bucket directory with different prefixes. Each database + # creates an identically named table with different data. Each then + # does a flush tier testing that both databases can coexist in the + # same bucket without conflict. + # + # Then reopen the connections and make sure we can read data correctly. + # + # We open two connections manually so that they both have the same relative + # pathnames. The standard connection is just a dummy for this test. + ext = self.extensionsConfig() + conn1_params = self.saved_conn + ext + ',tiered_storage=(bucket_prefix=%s)' % self.prefix1 + conn1 = self.wiredtiger_open(self.conn1_dir, conn1_params) + session1 = conn1.open_session() + conn2_params = self.saved_conn + ext + ',tiered_storage=(bucket_prefix=%s)' % self.prefix2 + conn2 = self.wiredtiger_open(self.conn2_dir, conn2_params) + session2 = conn2.open_session() + + session1.create(self.uri, 'key_format=S,value_format=S,') + session2.create(self.uri, 'key_format=S,value_format=S,') + + # Add first data. Checkpoint, flush and close the connection. + c1 = session1.open_cursor(self.uri) + c2 = session2.open_cursor(self.uri) + c1["0"] = "0" + c2["20"] = "20" + self.check(c1, 0, 1) + self.check(c2, 20, 1) + c1.close() + c2.close() + session1.checkpoint() + session1.flush_tier(None) + session2.checkpoint() + session2.flush_tier(None) + conn1_obj1 = self.bucket + '/' + self.prefix1 + self.obj1file + conn2_obj1 = self.bucket + '/' + self.prefix2 + self.obj1file + self.assertTrue(os.path.exists(conn1_obj1)) + self.assertTrue(os.path.exists(conn2_obj1)) + conn1.close() + conn2.close() + + # Remove the local copies of the objects before we reopen so that we force + # the system to read from the bucket or bucket cache. + local = self.conn1_dir + '/' + self.obj1file + os.remove(local) + local = self.conn2_dir + '/' + self.obj1file + os.remove(local) + + conn1 = self.wiredtiger_open(self.conn1_dir, conn1_params) + session1 = conn1.open_session() + conn2 = self.wiredtiger_open(self.conn2_dir, conn2_params) + session2 = conn2.open_session() + + c1 = session1.open_cursor(self.uri) + c2 = session2.open_cursor(self.uri) + self.check(c1, 0, 1) + self.check(c2, 20, 1) + c1.close() + c2.close() + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_verbose01.py b/src/third_party/wiredtiger/test/suite/test_verbose01.py new file mode 100755 index 00000000000..7bd7b542803 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_verbose01.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +from suite_subprocess import suite_subprocess +from contextlib import contextmanager +import wiredtiger, wttest +import re + +# Shared base class used by verbose tests. +class test_verbose_base(wttest.WiredTigerTestCase, suite_subprocess): + # The maximum number of lines we will read from stdout in any given context. + nlines = 30000 + + def create_verbose_configuration(self, categories): + if len(categories) == 0: + return '' + return 'verbose=[' + ','.join(categories) + ']' + + @contextmanager + def expect_verbose(self, categories, patterns, expect_output = True): + # Clean the stdout resource before yielding the context to the execution block. We only want to + # capture the verbose output of the using context (ignoring any previous output up to this point). + self.cleanStdout() + # Create a new connection with the given verbose categories. + verbose_config = self.create_verbose_configuration(categories) + conn = self.wiredtiger_open(self.home, verbose_config) + # Yield the connection resource to the execution context, allowing it to perform any necessary + # operations on the connection (for generating the expected verbose output). + yield conn + # Read the contents of stdout to extract our verbose messages. + output = self.readStdout(self.nlines) + # Split the output into their individual messages. We want validate the contents of each message + # to ensure we've only generated verbose messages for the expected categories. + verbose_messages = output.splitlines() + + if expect_output: + self.assertGreater(len(verbose_messages), 0) + else: + self.assertEqual(len(verbose_messages), 0) + + # Test the contents of each verbose message, ensuring it satisfies the expected pattern. + verb_pattern = re.compile('|'.join(patterns)) + for line in verbose_messages: + self.assertTrue(verb_pattern.search(line) != None, 'Unexpected verbose message: ' + line) + + # Close the connection resource and clean up the contents of the stdout file, flushing out the + # verbose output that occurred during the execution of this context. + conn.close() + self.cleanStdout() + +# test_verbose01.py +# Verify basic uses of the verbose configuration API work as intended i.e. passing +# single & multiple valid and invalid verbose categories. These tests are mainly focused on uses +# of the interface prior to the introduction of verbosity levels, ensuring 'legacy'-style +# uses of the interface are still supported. +class test_verbose01(test_verbose_base): + collection_cfg = 'key_format=S,value_format=S' + # Test use cases passing single verbose categories, ensuring we only produce verbose output for the single category. + def test_verbose_single(self): + # Close the initial connection. We will be opening new connections with different verbosity settings throughout + # this test. + self.close_conn() + + # Test passing a single verbose category, 'api'. Ensuring the only verbose output generated is related to + # the 'api' category. + with self.expect_verbose(['api'], ['WT_VERB_API']) as conn: + # Perform a set of simple API operations (table creations and cursor operations) to generate verbose API + # messages. + uri = 'table:test_verbose01_api' + session = conn.open_session() + session.create(uri, self.collection_cfg) + c = session.open_cursor(uri) + c['api'] = 'api' + c.close() + session.close() + + # Test passing another single verbose category, 'compact'. Ensuring the only verbose output generated is related to + # the 'compact' category. + with self.expect_verbose(['compact'], ['WT_VERB_COMPACT']) as conn: + # Create a simple table to invoke compaction on. We aren't doing anything interesting with the table + # such that the data source will be compacted. Rather we want to simply invoke a compaction pass to + # generate verbose messages. + uri = 'table:test_verbose01_compact' + session = conn.open_session() + session.create(uri, self.collection_cfg) + session.compact(uri) + session.close() + + # Test use cases passing multiple verbose categories, ensuring we only produce verbose output for specified categories. + def test_verbose_multiple(self): + self.close_conn() + # Test passing multiple verbose categories, being 'api' & 'version'. Ensuring the only verbose output generated + # is related to those two categories. + with self.expect_verbose(['api','version'], ['WT_VERB_API', 'WT_VERB_VERSION']) as conn: + # Perform a set of simple API operations (table creations and cursor operations) to generate verbose API + # messages. Beyond opening the connection resource, we shouldn't need to do anything special for the version + # category. + uri = 'table:test_verbose01_multiple' + session = conn.open_session() + session.create(uri, self.collection_cfg) + c = session.open_cursor(uri) + c['multiple'] = 'multiple' + c.close() + + # Test use cases passing no verbose categories, ensuring we don't produce unexpected verbose output. + def test_verbose_none(self): + self.close_conn() + # Testing passing an empty set of categories. Ensuring no verbose output is generated. + with self.expect_verbose([], [], False) as conn: + # Perform a set of simple API operations (table creations and cursor operations). Ensuring no verbose messages + # are generated. + uri = 'table:test_verbose01_none' + session = conn.open_session() + session.create(uri, self.collection_cfg) + c = session.open_cursor(uri) + c['none'] = 'none' + c.close() + + # Test use cases passing invalid verbose categories, ensuring the appropriate error message is + # raised. + def test_verbose_invalid(self): + self.close_conn() + self.assertRaisesHavingMessage(wiredtiger.WiredTigerError, + lambda:self.wiredtiger_open(self.home, 'verbose=[test_verbose_invalid]'), + '/\'test_verbose_invalid\' not a permitted choice for key \'verbose\'/') + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_verbose02.py b/src/third_party/wiredtiger/test/suite/test_verbose02.py new file mode 100755 index 00000000000..33ed31de26d --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_verbose02.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +import wiredtiger, wttest +from test_verbose01 import test_verbose_base + +# test_verbose02.py +# Verify basic uses of the verbose configuration API when categories and valid/invalid verbosity +# levels are specified. +class test_verbose02(test_verbose_base): + collection_cfg = 'key_format=S,value_format=S' + + # Test use cases passing single verbose categories, ensuring we only produce verbose output for + # the single category. + def test_verbose_single(self): + # Close the initial connection. We will be opening new connections with different verbosity + # settings throughout this test. + self.close_conn() + + # Test passing a single verbose category, 'api' along with the verbosity level + # WT_VERBOSE_DEBUG (1). Ensuring the only verbose output generated is related to the 'api' + # category. + with self.expect_verbose(['api:1'], ['WT_VERB_API']) as conn: + # Perform a set of simple API operations to generate verbose API messages. + uri = 'table:test_verbose01_api' + session = conn.open_session() + session.create(uri, self.collection_cfg) + c = session.open_cursor(uri) + c['api'] = 'api' + c.close() + session.close() + + # At this time, there is no verbose messages with the category WT_VERB_API and the verbosity + # level WT_VERBOSE_INFO (0), hence we don't expect any output. + with self.expect_verbose(['api:0'], ['WT_VERB_API'], False) as conn: + uri = 'table:test_verbose01_api' + session = conn.open_session() + session.create(uri, self.collection_cfg) + c = session.open_cursor(uri) + c['api'] = 'api' + c.close() + session.close() + + # Test passing another single verbose category, 'compact' with different verbosity levels. + # Since there are verbose message with the category WT_VERB_COMPACT and the verbosity levels + # WT_VERBOSE_INFO (0) and WT_VERBOSE_DEBUG (1), we can test them both. + cfgs = ['compact:0', 'compact:1'] + for cfg in cfgs: + with self.expect_verbose([cfg], ['WT_VERB_COMPACT']) as conn: + # Create a simple table to invoke compaction on. We aren't doing anything + # interesting with the table, we want to simply invoke a compaction pass to generate + # verbose messages. + uri = 'table:test_verbose01_compact' + session = conn.open_session() + session.create(uri, self.collection_cfg) + session.compact(uri) + session.close() + + # Test use cases passing multiple verbose categories, ensuring we only produce verbose output + # for specified categories. + def test_verbose_multiple(self): + self.close_conn() + # Test passing multiple verbose categories, being 'api' & 'version' with different dedicated + # verbosity levels to each category. Ensuring the only verbose output generated is related + # to those two categories. + cfgs = ['api:1,version', 'api,version:1', 'api:1,version:1'] + for cfg in cfgs: + with self.expect_verbose([cfg], ['WT_VERB_API', 'WT_VERB_VERSION']) as conn: + # Perform a set of simple API operations (table creations and cursor operations) to + # generate verbose API messages. Beyond opening the connection resource, we + # shouldn't need to do anything special for the version category. + uri = 'table:test_verbose01_multiple' + session = conn.open_session() + session.create(uri, self.collection_cfg) + c = session.open_cursor(uri) + c['multiple'] = 'multiple' + c.close() + + # Test use cases passing invalid verbosity levels, ensuring the appropriate error message is + # raised. + def test_verbose_level_invalid(self): + self.close_conn() + # Any negative value is invalid. + self.assertRaisesHavingMessage(wiredtiger.WiredTigerError, + lambda:self.wiredtiger_open(self.home, 'verbose=[api:-1]'), + '/Failed to parse verbose option \'api\'/') + # Any value greater than WT_VERBOSE_DEBUG (1) is invalid. + self.assertRaisesHavingMessage(wiredtiger.WiredTigerError, + lambda:self.wiredtiger_open(self.home, 'verbose=[api:2]'), + '/Failed to parse verbose option \'api\'/') + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/wttest.py b/src/third_party/wiredtiger/test/suite/wttest.py index 07e5b0b6610..676a6fe90fa 100755 --- a/src/third_party/wiredtiger/test/suite/wttest.py +++ b/src/third_party/wiredtiger/test/suite/wttest.py @@ -575,6 +575,9 @@ class WiredTigerTestCase(unittest.TestCase): yield self.captureerr.checkAdditionalPattern(self, pat, re_flags) + def readStdout(self, maxchars=10000): + return self.captureout.readFileFrom(self.captureout.filename, self.captureout.expectpos, maxchars) + def ignoreStdoutPatternIfExists(self, pat, re_flags=0): if self.captureout.hasUnexpectedOutput(self): self.captureout.checkAdditionalPattern(self, pat, re_flags) diff --git a/src/third_party/wiredtiger/test/utility/misc.c b/src/third_party/wiredtiger/test/utility/misc.c index c39b189a634..64de4cd0f01 100644 --- a/src/third_party/wiredtiger/test/utility/misc.c +++ b/src/third_party/wiredtiger/test/utility/misc.c @@ -53,10 +53,6 @@ testutil_die(int e, const char *fmt, ...) (void)fflush(stdout); (void)fflush(stderr); - /* Allow test programs to cleanup on fatal error. */ - if (custom_die != NULL) - (*custom_die)(); - fprintf(stderr, "%s: FAILED", progname); if (fmt != NULL) { fprintf(stderr, ": "); @@ -67,9 +63,15 @@ testutil_die(int e, const char *fmt, ...) if (e != 0) fprintf(stderr, ": %s", wiredtiger_strerror(e)); fprintf(stderr, "\n"); - fprintf(stderr, "%s: process aborting\n", progname); + (void)fflush(stderr); - abort(); + /* Allow test programs to cleanup on fatal error. */ + if (custom_die != NULL) + (*custom_die)(); + + /* Drop core. */ + fprintf(stderr, "%s: process aborting\n", progname); + __wt_abort(NULL); } /* @@ -327,7 +329,7 @@ testutil_is_flag_set(const char *flag) */ flag_being_set = res[0] != '0'; - free((void *)res); + __wt_free(NULL, res); return (flag_being_set); } diff --git a/src/third_party/wiredtiger/test/utility/test_util.h b/src/third_party/wiredtiger/test/utility/test_util.h index 54444684ddf..31283084400 100644 --- a/src/third_party/wiredtiger/test/utility/test_util.h +++ b/src/third_party/wiredtiger/test/utility/test_util.h @@ -245,7 +245,8 @@ extern void (*custom_die)(void); #ifdef _WIN32 __declspec(noreturn) #endif - void testutil_die(int, const char *, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); + void testutil_die(int, const char *, ...) WT_GCC_FUNC_ATTRIBUTE((cold)) + WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); void *dcalloc(size_t, size_t); void *dmalloc(size_t); diff --git a/src/third_party/wiredtiger/tools/run_parallel.sh b/src/third_party/wiredtiger/tools/run_parallel.sh index 526dc5e65f0..9a05da66f0c 100755 --- a/src/third_party/wiredtiger/tools/run_parallel.sh +++ b/src/third_party/wiredtiger/tools/run_parallel.sh @@ -39,17 +39,28 @@ echo " num_iter: $num_iter" outf=./outfile.txt for i in $(seq $num_iter); do - echo "Starting iteration $i" >> $outf - echo "Starting iteration $i" + echo "==== Starting iteration $i ====" >> $outf + echo "==== Starting iteration $i ====" + echo "Disk usage and free space for the current drive:" + df -h . + + process_ids=() # start the commands in parallel for((t=1; t<=num_parallel; t++)); do - echo "Starting parallel command $t (of $num_parallel) in iteration $i (of $num_iter)" >> nohup.out.$t - eval nohup $command >> nohup.out.$t 2>&1 & + echo "Starting parallel command $t (of $num_parallel) in iteration $i (of $num_iter)" >> $outf + eval nohup $command > nohup.out.$t 2>&1 & + process_ids[$t]=$! done # Wait for the commands to all complete for((t=1; t<=num_parallel; t++)); do - wait || exit $? + wait ${process_ids[$t]} + err=$? + if [[ $err -ne 0 ]] + then + echo "iteration $i of parallel command $t failed with $err error code" + exit $err + fi done done |
