diff options
| author | Alex Gorrod <alexander.gorrod@mongodb.com> | 2017-07-20 12:20:46 +1000 |
|---|---|---|
| committer | Alex Gorrod <alexander.gorrod@mongodb.com> | 2017-07-20 12:24:11 +1000 |
| commit | 5a2533ebc3606973fbe237228b9bacdcb21a532b (patch) | |
| tree | 66805f578f761446f1cdd6637dcfaa90c60e1388 /src/third_party/wiredtiger | |
| parent | 634435949c4b855b9cc5bfbf5cf481d8158fd996 (diff) | |
| download | mongo-5a2533ebc3606973fbe237228b9bacdcb21a532b.tar.gz | |
Import wiredtiger: 6173a98979715ed727c432c1a31da64ea8a37048 from branch mongodb-3.6
ref: ff10db8811..6173a98979
for: 3.5.11
WT-3039 Enhance logging so new log files no longer need to wait for the previous log file to be synced
WT-3138 Enhance eviction statistics
WT-3140 Enhance json mode statistics to better support per-dhandle statistics
WT-3310 Add support to WT_SESSION::alter to change table log setting
WT-3329 With a uniform workload and a number of small collections, eviction does a poor job of selecting candidates for eviction.
WT-3381 Improve concurrency in the transaction subsystem
WT-3394 Build WiredTiger with gcc7
WT-3396 test/csuite/wt3363_checkpoint_op_races, test_utility cleanups
WT-3397 Coverity lint
WT-3398 in-memory configurations don't support schema_worker operations
WT-3401 Lint and minor cleanup
WT-3402 Move cached overflow records to the update list.
WT-3403 Automated tests timeout running csuite
WT-3409 WiredTiger generations can silently self-deadlock.
WT-3413 Add more aggressive compile warning flags to Jenkins Windows job
WT-3415 setting timestamp in txn_state requires the global write lock
WT-3421 Compilation error unreachable code in test/csuite/wt3363_checkpoint_op_races
WT-3424 additional gcc 7.1 compile warnings.
WT-3425 workgen: add ability to reopen cursors
WT-3426 Add new wtperf configuration to automated performance testing
WT-3438 Don't tune eviction thread count when the count is fixed
Also explicitly disable v2 log records via a configuration setting in MongoDB -
this will later be plumbed into the feature compatability version mechanism.
Diffstat (limited to 'src/third_party/wiredtiger')
128 files changed, 3724 insertions, 1873 deletions
diff --git a/src/third_party/wiredtiger/.gitignore b/src/third_party/wiredtiger/.gitignore index cc2aad047be..082145e66d0 100644 --- a/src/third_party/wiredtiger/.gitignore +++ b/src/third_party/wiredtiger/.gitignore @@ -95,28 +95,31 @@ _wiredtiger.pyd **/examples/c/ex_pack **/examples/c/ex_process **/examples/c/ex_schema +**/examples/c/ex_smoke **/examples/c/ex_stat **/examples/c/ex_sync **/examples/c/ex_thread **/test/bloom/t **/test/checkpoint/t -**/test_scope -**/test_wt1965_col_efficiency -**/test_wt2246_col_append -**/test_wt2323_join_visibility -**/test_wt2403_lsm_workload -**/test_wt2447_join_main_table -**/test_wt2535_insert_race -**/test_wt2592_join_schema -**/test_wt2695_checksum -**/test_wt2719_reconfig -**/test_wt2834_join_bloom_fix -**/test_wt2853_perf -**/test_wt2909_checkpoint_integrity -**/test_wt2999_join_extractor -**/test_wt3120_filesys -**/test_wt3135_search_near_collator -**/test_wt3184_dup_index_collator +**/test/csuite/test_rwlock +**/test/csuite/test_scope +**/test/csuite/test_wt1965_col_efficiency +**/test/csuite/test_wt2246_col_append +**/test/csuite/test_wt2323_join_visibility +**/test/csuite/test_wt2403_lsm_workload +**/test/csuite/test_wt2447_join_main_table +**/test/csuite/test_wt2535_insert_race +**/test/csuite/test_wt2592_join_schema +**/test/csuite/test_wt2695_checksum +**/test/csuite/test_wt2719_reconfig +**/test/csuite/test_wt2834_join_bloom_fix +**/test/csuite/test_wt2853_perf +**/test/csuite/test_wt2909_checkpoint_integrity +**/test/csuite/test_wt2999_join_extractor +**/test/csuite/test_wt3120_filesys +**/test/csuite/test_wt3135_search_near_collator +**/test/csuite/test_wt3184_dup_index_collator +**/test/csuite/test_wt3363_checkpoint_op_races **/test/cursor_order/cursor_order **/test/fops/t **/test/format/s_dumpcmp diff --git a/src/third_party/wiredtiger/SConstruct b/src/third_party/wiredtiger/SConstruct index 748bf18db00..501dae2ad9f 100644 --- a/src/third_party/wiredtiger/SConstruct +++ b/src/third_party/wiredtiger/SConstruct @@ -67,10 +67,11 @@ var.Add('CPPPATH', 'C Preprocessor include path', [ ]) var.Add('CFLAGS', 'C Compiler Flags', [ - "/wd4090", # Ignore warning about mismatched const qualifiers - "/wd4996", # Ignore deprecated functions "/W3", # Warning level 3 "/WX", # Warnings are fatal + "/wd4090", # Ignore warning about mismatched const qualifiers + "/wd4996", # Ignore deprecated functions + "/we4100", # Complain about unreferenced format parameter "/Z7", # Generate debugging symbols "/TC", # Compile as C code #"/Od", # Disable optimization @@ -295,6 +296,7 @@ wtbin = env.Program("wt", [ "src/utilities/util_cpyright.c", "src/utilities/util_compact.c", "src/utilities/util_create.c", + "src/utilities/util_downgrade.c", "src/utilities/util_drop.c", "src/utilities/util_dump.c", "src/utilities/util_list.c", @@ -336,6 +338,7 @@ if GetOption("lang-python"): ]) # Ignore warnings in swig-generated code. pythonEnv['CFLAGS'].remove("/WX") + pythonEnv['CFLAGS'].remove("/we4100") swiglib = pythonEnv.SharedLibrary('_wiredtiger', [ 'lang\python\wiredtiger.i'], diff --git a/src/third_party/wiredtiger/bench/workgen/runner/small_btree_reopen.py b/src/third_party/wiredtiger/bench/workgen/runner/small_btree_reopen.py new file mode 100644 index 00000000000..252dd02aae9 --- /dev/null +++ b/src/third_party/wiredtiger/bench/workgen/runner/small_btree_reopen.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2017 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +from runner import * +from wiredtiger import * +from workgen import * + +context = Context() +conn = wiredtiger_open("WT_TEST", "create,cache_size=500MB") +s = conn.open_session() +tname = "file:test.wt" +s.create(tname, 'key_format=S,value_format=S') +table = Table(tname) +table.options.key_size = 20 +table.options.value_size = 100 + +op = Operation(Operation.OP_INSERT, table) +thread = Thread(op * 500000) +pop_workload = Workload(context, thread) +print('populate:') +pop_workload.run(conn) + +op = Operation(Operation.OP_SEARCH, table) +op._config = 'reopen' +t = Thread(op) +workload = Workload(context, t * 8) +workload.options.run_time = 120 +workload.options.report_interval = 5 +print('read workload:') +workload.run(conn) diff --git a/src/third_party/wiredtiger/bench/workgen/workgen.cxx b/src/third_party/wiredtiger/bench/workgen/workgen.cxx index 880b8ca6467..1a0735f9adc 100644 --- a/src/third_party/wiredtiger/bench/workgen/workgen.cxx +++ b/src/third_party/wiredtiger/bench/workgen/workgen.cxx @@ -534,7 +534,7 @@ void ThreadRunner::op_create_all(Operation *op, size_t &keysize, size_t &valuesize) { tint_t tint; - op->size_check(); + op->create_all(); if (op->_optype != Operation::OP_NONE) { op->kv_compute_max(true); if (OP_HAS_VALUE(op)) @@ -579,6 +579,7 @@ uint64_t ThreadRunner::op_get_key_recno(Operation *op, tint_t tint) { uint64_t recno_count; uint32_t rand; + (void)op; recno_count = _icontext->_recno[tint]; if (recno_count == 0) // The file has no entries, returning 0 forces a WT_NOTFOUND return. @@ -590,13 +591,15 @@ uint64_t ThreadRunner::op_get_key_recno(Operation *op, tint_t tint) { int ThreadRunner::op_run(Operation *op) { Track *track; tint_t tint = op->_table._internal->_tint; - WT_CURSOR *cursor = _cursors[tint]; + WT_CURSOR *cursor; WT_DECL_RET; uint64_t recno; - bool measure_latency; + bool measure_latency, own_cursor; - recno = 0; track = NULL; + cursor = NULL; + recno = 0; + own_cursor = false; if (_throttle != NULL) { if (_throttle_ops >= _throttle_limit && !_in_transaction) { WT_ERR(_throttle->throttle(_throttle_ops, @@ -636,6 +639,13 @@ int ThreadRunner::op_run(Operation *op) { recno = 0; break; } + if ((op->_flags & WORKGEN_OP_REOPEN) != 0 && + op->_optype != Operation::OP_NONE) { + WT_ERR(_session->open_cursor(_session, op->_table._uri.c_str(), NULL, + NULL, &cursor)); + own_cursor = true; + } else + cursor = _cursors[tint]; measure_latency = track != NULL && track->ops != 0 && track->track_latency() && @@ -676,6 +686,7 @@ int ThreadRunner::op_run(Operation *op) { ASSERT(false); } if (ret != 0) { + ASSERT(ret == WT_NOTFOUND); track = &_stats.not_found; ret = 0; // WT_NOTFOUND allowed. } @@ -694,6 +705,8 @@ int ThreadRunner::op_run(Operation *op) { i != op->_group->end(); i++) WT_ERR(op_run(&*i)); err: + if (own_cursor) + WT_TRET(cursor->close(cursor)); if (op->_transaction != NULL) { if (ret != 0 || op->_transaction->_rollback) WT_TRET(_session->rollback_transaction(_session, NULL)); @@ -716,8 +729,8 @@ Throttle::Throttle(ThreadRunner &runner, double throttle, _burst(throttle_burst), _next_div(), _ops_delta(0), _ops_prev(0), _ops_per_div(0), _ms_per_div(0), _started(false) { ts_clear(_next_div); - _ms_per_div = ceill(1000.0 / THROTTLE_PER_SEC); - _ops_per_div = ceill(_throttle / THROTTLE_PER_SEC); + _ms_per_div = (uint64_t)ceill(1000.0 / THROTTLE_PER_SEC); + _ops_per_div = (uint64_t)ceill(_throttle / THROTTLE_PER_SEC); } Throttle::~Throttle() {} @@ -835,36 +848,37 @@ void Thread::describe(std::ostream &os) const { } Operation::Operation() : - _optype(OP_NONE), _table(), _key(), _value(), _transaction(NULL), - _group(NULL), _repeatgroup(0), + _optype(OP_NONE), _table(), _key(), _value(), _config(), _transaction(NULL), + _group(NULL), _repeatgroup(0), _flags(0), _keysize(0), _valuesize(0), _keymax(0), _valuemax(0) { } Operation::Operation(OpType optype, Table table, Key key, Value value) : - _optype(optype), _table(table), _key(key), _value(value), - _transaction(NULL), _group(NULL), _repeatgroup(0), + _optype(optype), _table(table), _key(key), _value(value), _config(), + _transaction(NULL), _group(NULL), _repeatgroup(0), _flags(0), _keysize(0), _valuesize(0), _keymax(0), _valuemax(0) { size_check(); } Operation::Operation(OpType optype, Table table, Key key) : - _optype(optype), _table(table), _key(key), _value(), _transaction(NULL), - _group(NULL), _repeatgroup(0), + _optype(optype), _table(table), _key(key), _value(), _config(), + _transaction(NULL), _group(NULL), _repeatgroup(0), _flags(0), _keysize(0), _valuesize(0), _keymax(0), _valuemax(0) { size_check(); } Operation::Operation(OpType optype, Table table) : - _optype(optype), _table(table), _key(), _value(), _transaction(NULL), - _group(NULL), _repeatgroup(0), + _optype(optype), _table(table), _key(), _value(), _config(), + _transaction(NULL), _group(NULL), _repeatgroup(0), _flags(0), _keysize(0), _valuesize(0), _keymax(0), _valuemax(0) { size_check(); } Operation::Operation(const Operation &other) : _optype(other._optype), _table(other._table), _key(other._key), - _value(other._value), _transaction(other._transaction), - _group(other._group), _repeatgroup(other._repeatgroup), + _value(other._value), _config(other._config), + _transaction(other._transaction), _group(other._group), + _repeatgroup(other._repeatgroup), _flags(other._flags), _keysize(other._keysize), _valuesize(other._valuesize), _keymax(other._keymax), _valuemax(other._valuemax) { // Creation and destruction of _group and _transaction is managed @@ -890,6 +904,18 @@ Operation& Operation::operator=(const Operation &other) { return (*this); } +void Operation::create_all() { + size_check(); + + _flags = 0; + if (!_config.empty()) { + if (_config == "reopen") + _flags |= WORKGEN_OP_REOPEN; + else + THROW("operation has illegal config: \"" << _config << "\""); + } +} + void Operation::describe(std::ostream &os) const { os << "Operation: " << _optype; if (_optype != OP_NONE) { @@ -897,9 +923,10 @@ void Operation::describe(std::ostream &os) const { os << ", "; _key.describe(os); os << ", "; _value.describe(os); } - if (_transaction != NULL) { + if (!_config.empty()) + os << ", '" << _config; + if (_transaction != NULL) os << ", ["; _transaction->describe(os); os << "]"; - } if (_group != NULL) { os << ", group[" << _repeatgroup << "]: {"; bool first = true; @@ -1648,4 +1675,4 @@ int WorkloadRunner::run_all() { return (ret); } -}; +} diff --git a/src/third_party/wiredtiger/bench/workgen/workgen.h b/src/third_party/wiredtiger/bench/workgen/workgen.h index c7be8ee0035..a12e4dc4c89 100644 --- a/src/third_party/wiredtiger/bench/workgen/workgen.h +++ b/src/third_party/wiredtiger/bench/workgen/workgen.h @@ -33,6 +33,7 @@ namespace workgen { struct ContextInternal; +struct OperationInternal; struct TableInternal; struct Thread; struct Transaction; @@ -245,11 +246,15 @@ struct Operation { Table _table; Key _key; Value _value; + std::string _config; Transaction *_transaction; std::vector<Operation> *_group; int _repeatgroup; #ifndef SWIG +#define WORKGEN_OP_REOPEN 0x0001 // reopen cursor for each op + uint32_t _flags; + int _keysize; // derived from Key._size and Table.options.key_size int _valuesize; uint64_t _keymax; @@ -266,6 +271,7 @@ struct Operation { void describe(std::ostream &os) const; #ifndef SWIG Operation& operator=(const Operation &other); + void create_all(); void get_static_counts(Stats &stats, int multiplier); void kv_compute_max(bool); void kv_gen(bool, uint64_t, char *) const; @@ -408,4 +414,4 @@ struct Workload { int run(WT_CONNECTION *conn); }; -}; +} diff --git a/src/third_party/wiredtiger/bench/workgen/workgen.swig b/src/third_party/wiredtiger/bench/workgen/workgen.swig index 0f74942169c..a33705f4350 100644 --- a/src/third_party/wiredtiger/bench/workgen/workgen.swig +++ b/src/third_party/wiredtiger/bench/workgen/workgen.swig @@ -158,7 +158,7 @@ WorkgenFrozenClass(WorkloadOptions) def __add__(self, other): if not isinstance(other, Operation): raise Exception('Operation.__sum__ requires an Operation') - if self._group == None or self._repeatgroup != 1 or self._transaction != None: + if self._group == None or self._repeatgroup != 1 or self._transaction != None or self._config != '': op = Operation() op._group = OpList([self, other]) op._repeatgroup = 1 diff --git a/src/third_party/wiredtiger/bench/workgen/workgen_int.h b/src/third_party/wiredtiger/bench/workgen/workgen_int.h index 9283aea1d7b..a8d008a3bc5 100644 --- a/src/third_party/wiredtiger/bench/workgen/workgen_int.h +++ b/src/third_party/wiredtiger/bench/workgen/workgen_int.h @@ -203,4 +203,4 @@ private: WorkloadRunner& operator=(const WorkloadRunner &other); // disallowed }; -}; +} diff --git a/src/third_party/wiredtiger/bench/wtperf/config.c b/src/third_party/wiredtiger/bench/wtperf/config.c index c5a3dd40032..1a4f46f5c37 100644 --- a/src/third_party/wiredtiger/bench/wtperf/config.c +++ b/src/third_party/wiredtiger/bench/wtperf/config.c @@ -798,7 +798,7 @@ config_sanity(WTPERF *wtperf) workp->table_index >= (int32_t)opts->table_count) { fprintf(stderr, "Workload table index %" PRId32 - " is larger than table count %" PRId32, + " is larger than table count %" PRIu32, workp->table_index, opts->table_count); return (EINVAL); } diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/update-only-btree.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/update-only-btree.wtperf new file mode 100644 index 00000000000..ea37a4d13d1 --- /dev/null +++ b/src/third_party/wiredtiger/bench/wtperf/runners/update-only-btree.wtperf @@ -0,0 +1,10 @@ +# wtperf options file: btree with updates, in memory. The goal here is to +# have a workload that uses a lot of transaction IDs, so stresses the ID +# allocation/release code performance. +conn_config="cache_size=5G" +table_config="type=file" +icount=5000000 +report_interval=5 +run_time=120 +populate_threads=1 +threads=((count=16,updates=1)) diff --git a/src/third_party/wiredtiger/bench/wtperf/wtperf.c b/src/third_party/wiredtiger/bench/wtperf/wtperf.c index a8d3f135280..9e18df9a0e3 100644 --- a/src/third_party/wiredtiger/bench/wtperf/wtperf.c +++ b/src/third_party/wiredtiger/bench/wtperf/wtperf.c @@ -1750,7 +1750,7 @@ execute_workload(WTPERF *wtperf) lprintf(wtperf, 0, 1, "Starting workload #%u: %" PRId64 " threads, inserts=%" PRId64 ", reads=%" PRId64 ", updates=%" PRId64 - ", truncate=%" PRId64 ", throttle=%" PRId64, + ", truncate=%" PRId64 ", throttle=%" PRIu64, i + 1, workp->threads, workp->insert, workp->read, workp->update, workp->truncate, workp->throttle); diff --git a/src/third_party/wiredtiger/build_posix/Make.base b/src/third_party/wiredtiger/build_posix/Make.base index e5228fac885..08d64440ff0 100644 --- a/src/third_party/wiredtiger/build_posix/Make.base +++ b/src/third_party/wiredtiger/build_posix/Make.base @@ -22,6 +22,7 @@ wt_SOURCES =\ src/utilities/util_cpyright.c \ src/utilities/util_compact.c \ src/utilities/util_create.c \ + src/utilities/util_downgrade.c \ src/utilities/util_drop.c \ src/utilities/util_dump.c \ src/utilities/util_list.c \ diff --git a/src/third_party/wiredtiger/build_posix/aclocal/strict.m4 b/src/third_party/wiredtiger/build_posix/aclocal/strict.m4 index 8c15a22d575..f9509df26bf 100644 --- a/src/third_party/wiredtiger/build_posix/aclocal/strict.m4 +++ b/src/third_party/wiredtiger/build_posix/aclocal/strict.m4 @@ -18,7 +18,6 @@ AC_DEFUN([AM_GCC_WARNINGS], [ w="$w -Wjump-misses-init" w="$w -Wmissing-declarations" w="$w -Wmissing-field-initializers" - w="$w -Wmissing-parameter-type" w="$w -Wmissing-prototypes" w="$w -Wnested-externs" w="$w -Wold-style-definition" @@ -33,7 +32,6 @@ AC_DEFUN([AM_GCC_WARNINGS], [ w="$w -Wundef" w="$w -Wuninitialized" w="$w -Wunreachable-code" - w="$w -Wunsafe-loop-optimizations" w="$w -Wunused" w="$w -Wwrite-strings" @@ -48,10 +46,33 @@ AC_DEFUN([AM_GCC_WARNINGS], [ # Additional warning messages. case "$1" in [*4.7.[0-9]*]) # gcc4.7 - w="$w -Wno-c11-extensions";; + w="$w -Wno-c11-extensions" + w="$w -Wunsafe-loop-optimizations";; [*6.[0-9].[0-9]*]) # gcc6.X w="$w -Wduplicated-cond" - w="$w -Wmisleading-indentation";; + w="$w -Wformat-signedness" + w="$w -Wjump-misses-init" + w="$w -Wlogical-op" + w="$w -Wredundant-decls" + w="$w -Wunsafe-loop-optimizations" + w="$w -Wunused-const-variable=2" + w="$w -Wunused-macros" + w="$w -Wvariadic-macros";; + [*7.[0-9].[0-9]*]) # gcc7.X + w="$w -Walloca" + w="$w -Walloc-zero" + w="$w -Wduplicated-branches" + w="$w -Wduplicated-cond" + w="$w -Wformat-overflow=2" + w="$w -Wformat-signedness" + w="$w -Wformat-truncation=2" + w="$w -Wjump-misses-init" + w="$w -Wlogical-op" + w="$w -Wredundant-decls" + w="$w -Wrestrict" + w="$w -Wunused-const-variable=2" + w="$w -Wunused-macros" + w="$w -Wvariadic-macros";; esac wt_cv_strict_warnings="$w" diff --git a/src/third_party/wiredtiger/build_win/wiredtiger_config.h b/src/third_party/wiredtiger/build_win/wiredtiger_config.h index 90cd40f1fae..b118cfa9882 100644 --- a/src/third_party/wiredtiger/build_win/wiredtiger_config.h +++ b/src/third_party/wiredtiger/build_win/wiredtiger_config.h @@ -151,9 +151,6 @@ /* Default alignment of buffers used for I/O */ #define WT_BUFFER_ALIGNMENT_DEFAULT 0 -/* Size of a transaction timestamp in bytes */ -#define WT_TIMESTAMP_SIZE 8 - /* Enable large inode numbers on Mac OS X 10.5. */ /* #ifndef _DARWIN_USE_64_BIT_INODE */ /* # define _DARWIN_USE_64_BIT_INODE 1 */ diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index 4a4ffae535d..9e873198f33 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -131,6 +131,14 @@ file_runtime_config = [ do not ever evict the object's pages from cache. Not compatible with LSM tables; see @ref tuning_cache_resident for more information''', type='boolean'), + Config('log', '', r''' + the transaction log configuration for this object. Only valid if + log is enabled in ::wiredtiger_open''', + type='category', subconfig=[ + Config('enabled', 'true', r''' + if false, this object has checkpoint-level durability''', + type='boolean'), + ]), ] # Per-file configuration @@ -252,14 +260,6 @@ file_config = format_meta + file_runtime_config + [ Config('leaf_item_max', '0', r''' historic term for leaf_key_max and leaf_value_max''', min=0, undoc=True), - Config('log', '', r''' - the transaction log configuration for this object. Only valid if - log is enabled in ::wiredtiger_open''', - type='category', subconfig=[ - Config('enabled', 'true', r''' - if false, this object has checkpoint-level durability''', - type='boolean'), - ]), Config('memory_page_max', '5MB', r''' the maximum size a page can grow to in memory before being reconciled to disk. The specified size will be adjusted to a lower @@ -401,6 +401,12 @@ connection_runtime_config = [ above 0 configures periodic checkpoints''', min='0', max='100000'), ]), + Config('compatibility', '', r''' + set compatibility version of database''', + type='category', subconfig=[ + Config('release', '', r''' + compatibility release version string'''), + ]), Config('diagnostic_timing_stress', '', r''' enable insertion of code that interrupts the usual timing of operations with a goal of uncovering race conditions and unexpected diff --git a/src/third_party/wiredtiger/dist/filelist b/src/third_party/wiredtiger/dist/filelist index a81d693cb4e..309339ac877 100644 --- a/src/third_party/wiredtiger/dist/filelist +++ b/src/third_party/wiredtiger/dist/filelist @@ -95,6 +95,7 @@ src/evict/evict_stat.c src/log/log.c src/log/log_auto.c src/log/log_slot.c +src/log/log_sys.c src/lsm/lsm_cursor.c src/lsm/lsm_cursor_bulk.c src/lsm/lsm_manager.c diff --git a/src/third_party/wiredtiger/dist/log.py b/src/third_party/wiredtiger/dist/log.py index ac35d884eac..81dc8bd35d7 100644 --- a/src/third_party/wiredtiger/dist/log.py +++ b/src/third_party/wiredtiger/dist/log.py @@ -10,6 +10,8 @@ tmp_file = '__tmp' # Map log record types to: # (C type, pack type, printf format, printf arg(s), list of setup functions) field_types = { + 'WT_LSN' : ('WT_LSN *', 'II', '%" PRIu32 "%" PRIu32 "', + 'arg->l.file, arg->l.offset', [ '' ]), 'string' : ('const char *', 'S', '%s', 'arg', [ '' ]), 'item' : ('WT_ITEM *', 'u', '%s', 'escaped', [ 'WT_ERR(__logrec_make_json_str(session, &escaped, &arg));', diff --git a/src/third_party/wiredtiger/dist/log_data.py b/src/third_party/wiredtiger/dist/log_data.py index bbfb51708bf..1782d2aadf0 100644 --- a/src/third_party/wiredtiger/dist/log_data.py +++ b/src/third_party/wiredtiger/dist/log_data.py @@ -40,6 +40,10 @@ rectypes = [ # Debugging message in the log LogRecordType('message', 'message', [('string', 'message')]), + + # System record + LogRecordType('system', 'system', [ + ('WT_LSN','prev_lsn'), ('item', 'unused')]), ] class LogOperationType: diff --git a/src/third_party/wiredtiger/dist/s_funcs.list b/src/third_party/wiredtiger/dist/s_funcs.list index b73767cad13..42eacf9c068 100644 --- a/src/third_party/wiredtiger/dist/s_funcs.list +++ b/src/third_party/wiredtiger/dist/s_funcs.list @@ -7,7 +7,6 @@ __bit_ffs __bit_nclr __ovfl_discard_dump __ovfl_reuse_dump -__ovfl_txnc_dump __wt_bloom_drop __wt_bloom_get __wt_bulk_insert_fix diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index e0899bfeedf..723f6d18858 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -690,6 +690,7 @@ extern extlist fadvise fallocate +fallthrough fblocks fc fclose @@ -803,6 +804,7 @@ initn initsize initval inline +inlined inmem inmemory insertK @@ -1033,6 +1035,7 @@ prepended prepending presize presync +prevlsn primary's printf printlog @@ -1238,6 +1241,7 @@ unregister unsized unterminated untyped +unvisited upd update's upg @@ -1261,6 +1265,7 @@ valuep valuev vanishingly variable's +variadic vectorized versa vfprintf @@ -1284,6 +1289,7 @@ wb wcursor whitespace wiredTiger +wiredTigerTables wiredtiger workFactor workgen diff --git a/src/third_party/wiredtiger/dist/stat_data.py b/src/third_party/wiredtiger/dist/stat_data.py index 7b919848003..2e344e0fbef 100644 --- a/src/third_party/wiredtiger/dist/stat_data.py +++ b/src/third_party/wiredtiger/dist/stat_data.py @@ -205,12 +205,12 @@ connection_stats = [ CacheStat('cache_eviction_empty_score', 'eviction empty score', 'no_clear,no_scale'), CacheStat('cache_eviction_fail', 'pages selected for eviction unable to be evicted'), CacheStat('cache_eviction_force', 'pages evicted because they exceeded the in-memory maximum count'), - CacheStat('cache_eviction_force_time', 'pages evicted because they exceeded the in-memory maximum time (usecs)'), CacheStat('cache_eviction_force_delete', 'pages evicted because they had chains of deleted items count'), CacheStat('cache_eviction_force_delete_time', 'pages evicted because they had chains of deleted items time (usecs)'), CacheStat('cache_eviction_force_fail', 'failed eviction of pages that exceeded the in-memory maximum count'), CacheStat('cache_eviction_force_fail_time', 'failed eviction of pages that exceeded the in-memory maximum time (usecs)'), CacheStat('cache_eviction_force_retune', 'force re-tuning of eviction workers once in a while'), + CacheStat('cache_eviction_force_time', 'pages evicted because they exceeded the in-memory maximum time (usecs)'), CacheStat('cache_eviction_get_ref', 'eviction calls to get a page'), CacheStat('cache_eviction_get_ref_empty', 'eviction calls to get a page found queue empty'), CacheStat('cache_eviction_get_ref_empty2', 'eviction calls to get a page found queue empty after locking'), @@ -230,10 +230,22 @@ connection_stats = [ CacheStat('cache_eviction_split_leaf', 'leaf pages split during eviction'), CacheStat('cache_eviction_stable_state_workers', 'eviction worker thread stable number', 'no_clear'), CacheStat('cache_eviction_state', 'eviction state', 'no_clear,no_scale'), + CacheStat('cache_eviction_target_page_ge128', 'eviction walk target pages histogram - 128 and higher'), + CacheStat('cache_eviction_target_page_lt10', 'eviction walk target pages histogram - 0-9'), + CacheStat('cache_eviction_target_page_lt32', 'eviction walk target pages histogram - 10-31'), + CacheStat('cache_eviction_target_page_lt64', 'eviction walk target pages histogram - 32-63'), + CacheStat('cache_eviction_target_page_lt128', 'eviction walk target pages histogram - 64-128'), CacheStat('cache_eviction_walk', 'pages walked for eviction'), + CacheStat('cache_eviction_walk_from_root', 'eviction walks started from root of tree'), + CacheStat('cache_eviction_walk_passes', 'eviction passes of a file'), + CacheStat('cache_eviction_walk_saved_pos', 'eviction walks started from saved location in tree'), CacheStat('cache_eviction_walks_abandoned', 'eviction walks abandoned'), CacheStat('cache_eviction_walks_active', 'files with active eviction walks', 'no_clear,no_scale'), + CacheStat('cache_eviction_walks_ended', 'eviction walks reached end of tree'), + CacheStat('cache_eviction_walks_gave_up_no_targets', 'eviction walks gave up because they saw too many pages and found no candidates'), + CacheStat('cache_eviction_walks_gave_up_ratio', 'eviction walks gave up because they saw too many pages and found too few candidates'), CacheStat('cache_eviction_walks_started', 'files with new eviction walks started'), + CacheStat('cache_eviction_walks_stopped', 'eviction walks gave up because they restarted their walk twice'), CacheStat('cache_eviction_worker_created', 'eviction worker thread created'), CacheStat('cache_eviction_worker_evicting', 'eviction worker thread evicting pages'), CacheStat('cache_eviction_worker_removed', 'eviction worker thread removed'), @@ -244,7 +256,6 @@ connection_stats = [ CacheStat('cache_inmem_splittable', 'in-memory page passed criteria to be split'), CacheStat('cache_lookaside_insert', 'lookaside table insert calls'), CacheStat('cache_lookaside_remove', 'lookaside table remove calls'), - CacheStat('cache_overflow_value', 'overflow values cached in memory', 'no_scale'), CacheStat('cache_overhead', 'percentage overhead', 'no_clear,no_scale'), CacheStat('cache_pages_dirty', 'tracked dirty pages in the cache', 'no_clear,no_scale'), CacheStat('cache_pages_inuse', 'pages currently held in the cache', 'no_clear,no_scale'), @@ -323,6 +334,7 @@ connection_stats = [ LogStat('log_compress_write_fails', 'log records not compressed'), LogStat('log_compress_writes', 'log records compressed'), LogStat('log_flush', 'log flush operations'), + LogStat('log_force_ckpt_sleep', 'force checkpoint calls slept'), LogStat('log_force_write', 'log force write operations'), LogStat('log_force_write_skip', 'log force write operations skipped'), LogStat('log_max_filesize', 'maximum log file size', 'no_clear,no_scale,size'), @@ -510,11 +522,24 @@ dsrc_stats = [ CacheStat('cache_eviction_fail', 'data source pages selected for eviction unable to be evicted'), CacheStat('cache_eviction_hazard', 'hazard pointer blocked page eviction'), CacheStat('cache_eviction_internal', 'internal pages evicted'), + CacheStat('cache_eviction_pages_seen', 'pages seen by eviction walk'), CacheStat('cache_eviction_split_internal', 'internal pages split during eviction'), CacheStat('cache_eviction_split_leaf', 'leaf pages split during eviction'), + CacheStat('cache_eviction_target_page_ge128', 'eviction walk target pages histogram - 128 and higher'), + CacheStat('cache_eviction_target_page_lt10', 'eviction walk target pages histogram - 0-9'), + CacheStat('cache_eviction_target_page_lt32', 'eviction walk target pages histogram - 10-31'), + CacheStat('cache_eviction_target_page_lt64', 'eviction walk target pages histogram - 32-63'), + CacheStat('cache_eviction_target_page_lt128', 'eviction walk target pages histogram - 64-128'), + CacheStat('cache_eviction_walk_from_root', 'eviction walks started from root of tree'), + CacheStat('cache_eviction_walk_passes', 'eviction walk passes of a file'), + CacheStat('cache_eviction_walk_saved_pos', 'eviction walks started from saved location in tree'), + CacheStat('cache_eviction_walks_abandoned', 'eviction walks abandoned'), + CacheStat('cache_eviction_walks_ended', 'eviction walks reached end of tree'), + CacheStat('cache_eviction_walks_gave_up_no_targets', 'eviction walks gave up because they saw too many pages and found no candidates'), + CacheStat('cache_eviction_walks_gave_up_ratio', 'eviction walks gave up because they saw too many pages and found too few candidates'), + CacheStat('cache_eviction_walks_stopped', 'eviction walks gave up because they restarted their walk twice'), CacheStat('cache_inmem_split', 'in-memory page splits'), CacheStat('cache_inmem_splittable', 'in-memory page passed criteria to be split'), - CacheStat('cache_overflow_value', 'overflow values cached in memory', 'no_scale'), CacheStat('cache_pages_requested', 'pages requested from the cache'), CacheStat('cache_read', 'pages read into cache'), CacheStat('cache_read_lookaside', 'pages read into cache requiring lookaside entries'), @@ -526,6 +551,8 @@ dsrc_stats = [ ########################################## # Cache content statistics ########################################## + CacheWalkStat('cache_state_avg_unvisited_age', 'Average time in cache for pages that have not been visited by the eviction server', 'no_clear,no_scale'), + CacheWalkStat('cache_state_avg_visited_age', 'Average time in cache for pages that have been visited by the eviction server', 'no_clear,no_scale'), CacheWalkStat('cache_state_avg_written_size', 'Average on-disk page image size seen', 'no_clear,no_scale'), CacheWalkStat('cache_state_gen_avg_gap', 'Average difference between current eviction generation when the page was last considered', 'no_clear,no_scale'), CacheWalkStat('cache_state_gen_current', 'Current eviction generation', 'no_clear,no_scale'), @@ -544,6 +571,7 @@ dsrc_stats = [ CacheWalkStat('cache_state_root_entries', 'Entries in the root page', 'no_clear,no_scale'), CacheWalkStat('cache_state_root_size', 'Size of the root page', 'no_clear,no_scale'), CacheWalkStat('cache_state_smaller_alloc_size', 'On-disk page image sizes smaller than a single allocation unit', 'no_clear,no_scale'), + CacheWalkStat('cache_state_unvisited_count', 'Number of pages never visited by eviction server', 'no_clear,no_scale'), ########################################## # Compression statistics diff --git a/src/third_party/wiredtiger/examples/c/ex_backup.c b/src/third_party/wiredtiger/examples/c/ex_backup.c index 9cd9d5cf2a0..bf40d7573d8 100644 --- a/src/third_party/wiredtiger/examples/c/ex_backup.c +++ b/src/third_party/wiredtiger/examples/c/ex_backup.c @@ -48,7 +48,7 @@ static int compare_backups(int i) { int ret; - char buf[1024], msg[8]; + char buf[1024], msg[32]; /* * We run 'wt dump' on both the full backup directory and the @@ -83,7 +83,7 @@ compare_backups(int i) full_out, i, incr_out, i); ret = system(buf); if (i == 0) - (void)strncpy(msg, "MAIN", sizeof(msg)); + (void)snprintf(msg, sizeof(msg), "%s", "MAIN"); else (void)snprintf(msg, sizeof(msg), "%d", i); printf( diff --git a/src/third_party/wiredtiger/examples/c/ex_encrypt.c b/src/third_party/wiredtiger/examples/c/ex_encrypt.c index b6277ede084..3fdfa9a6d33 100644 --- a/src/third_party/wiredtiger/examples/c/ex_encrypt.c +++ b/src/third_party/wiredtiger/examples/c/ex_encrypt.c @@ -427,7 +427,7 @@ main(int argc, char *argv[]) WT_CURSOR *c1, *c2, *nc; WT_SESSION *session; int i, ret; - char keybuf[16], valbuf[16]; + char keybuf[32], valbuf[32]; char *key1, *key2, *key3, *val1, *val2, *val3; home = example_setup(argc, argv); diff --git a/src/third_party/wiredtiger/examples/c/ex_log.c b/src/third_party/wiredtiger/examples/c/ex_log.c index 3cf6ff6df23..1912ffca8a6 100644 --- a/src/third_party/wiredtiger/examples/c/ex_log.c +++ b/src/third_party/wiredtiger/examples/c/ex_log.c @@ -269,7 +269,7 @@ main(int argc, char *argv[]) WT_CURSOR *cursor; WT_SESSION *session; int count_min, i, record_count; - char cmd_buf[256], k[16], v[16]; + char cmd_buf[256], k[32], v[32]; (void)argc; /* Unused variable */ (void)testutil_set_progname(argv); diff --git a/src/third_party/wiredtiger/examples/c/ex_sync.c b/src/third_party/wiredtiger/examples/c/ex_sync.c index 2dcd332c51f..975ba3b928c 100644 --- a/src/third_party/wiredtiger/examples/c/ex_sync.c +++ b/src/third_party/wiredtiger/examples/c/ex_sync.c @@ -43,7 +43,7 @@ main(int argc, char *argv[]) WT_CURSOR *cursor; WT_SESSION *session; int i, record_count, ret; - char k[16], v[16]; + char k[32], v[32]; const char *conf; home = example_setup(argc, argv); diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index e6068334d42..ee82256992f 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -1,5 +1,5 @@ { - "commit": "ff10db881161bbd1bc23e40ac385ff0de18f68ff", + "commit": "6173a98979715ed727c432c1a31da64ea8a37048", "github": "wiredtiger/wiredtiger.git", "vendor": "wiredtiger", "branch": "mongodb-3.6" diff --git a/src/third_party/wiredtiger/src/async/async_op.c b/src/third_party/wiredtiger/src/async/async_op.c index d4ca754b95f..dc8a3cdf348 100644 --- a/src/third_party/wiredtiger/src/async/async_op.c +++ b/src/third_party/wiredtiger/src/async/async_op.c @@ -7,6 +7,7 @@ */ #include "wt_internal.h" + /* * __async_get_key -- * WT_ASYNC_OP->get_key implementation for op handles. diff --git a/src/third_party/wiredtiger/src/async/async_worker.c b/src/third_party/wiredtiger/src/async/async_worker.c index 57ebe5d8bb1..2626a21435f 100644 --- a/src/third_party/wiredtiger/src/async/async_worker.c +++ b/src/third_party/wiredtiger/src/async/async_worker.c @@ -215,7 +215,7 @@ __async_worker_execop(WT_SESSION_IMPL *session, WT_ASYNC_OP_IMPL *op, break; case WT_AOP_NONE: WT_RET_MSG(session, EINVAL, - "Unknown async optype %d", op->optype); + "Unknown async optype %d", (int)op->optype); } return (0); } diff --git a/src/third_party/wiredtiger/src/block/block_compact.c b/src/third_party/wiredtiger/src/block/block_compact.c index e7b9beafb01..76c4b1de3ec 100644 --- a/src/third_party/wiredtiger/src/block/block_compact.c +++ b/src/third_party/wiredtiger/src/block/block_compact.c @@ -8,7 +8,9 @@ #include "wt_internal.h" +#ifdef HAVE_VERBOSE static void __block_dump_avail(WT_SESSION_IMPL *, WT_BLOCK *, bool); +#endif /* * __wt_block_compact_start -- @@ -78,9 +80,11 @@ __wt_block_compact_skip(WT_SESSION_IMPL *session, WT_BLOCK *block, bool *skipp) __wt_spin_lock(session, &block->live_lock); +#ifdef HAVE_VERBOSE /* Dump the current state of the file. */ if (WT_VERBOSE_ISSET(session, WT_VERB_COMPACT)) __block_dump_avail(session, block, true); +#endif /* Sum the available bytes in the initial 80% and 90% of the file. */ avail_eighty = avail_ninety = 0; @@ -195,6 +199,7 @@ __wt_block_compact_page_skip(WT_SESSION_IMPL *session, return (0); } +#ifdef HAVE_VERBOSE /* * __block_dump_avail -- * Dump out the avail list so we can see what compaction will look like. @@ -271,3 +276,4 @@ __block_dump_avail(WT_SESSION_IMPL *session, WT_BLOCK *block, bool start) (uintmax_t)((v * 100) / (wt_off_t)el->bytes)); } } +#endif diff --git a/src/third_party/wiredtiger/src/block/block_vrfy.c b/src/third_party/wiredtiger/src/block/block_vrfy.c index 1058f16bde6..d2ca561fe82 100644 --- a/src/third_party/wiredtiger/src/block/block_vrfy.c +++ b/src/third_party/wiredtiger/src/block/block_vrfy.c @@ -33,7 +33,7 @@ int __wt_block_verify_start(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_CKPT *ckptbase, const char *cfg[]) { - WT_CKPT *ckpt; + WT_CKPT *ckpt, *t; WT_CONFIG_ITEM cval; wt_off_t size; @@ -50,14 +50,12 @@ __wt_block_verify_start(WT_SESSION_IMPL *session, * checkpoint we have is fake, there's no work to do. Don't complain, * that's not our problem to solve. */ - WT_CKPT_FOREACH(ckptbase, ckpt) - ; - for (;; --ckpt) { - if (ckpt->name != NULL && !F_ISSET(ckpt, WT_CKPT_FAKE)) - break; - if (ckpt == ckptbase) - return (0); - } + ckpt = NULL; + WT_CKPT_FOREACH(ckptbase, t) + if (t->name != NULL && !F_ISSET(t, WT_CKPT_FAKE)) + ckpt = t; + if (ckpt == NULL) + return (0); /* Set the size of the file to the size of the last checkpoint. */ WT_RET(__verify_set_file_size(session, block, ckpt)); @@ -387,6 +385,8 @@ __verify_filefrag_add(WT_SESSION_IMPL *session, WT_BLOCK *block, { uint64_t f, frag, frags, i; + WT_UNUSED(type); /* !HAVE_VERBOSE */ + __wt_verbose(session, WT_VERB_VERIFY, "add file block%s%s%s at %" PRIuMAX "-%" PRIuMAX " (%" PRIuMAX ")", type == NULL ? "" : " (", diff --git a/src/third_party/wiredtiger/src/btree/bt_debug.c b/src/third_party/wiredtiger/src/btree/bt_debug.c index 394ac6c7b84..c0aaf3f42d9 100644 --- a/src/third_party/wiredtiger/src/btree/bt_debug.c +++ b/src/third_party/wiredtiger/src/btree/bt_debug.c @@ -445,18 +445,28 @@ static char * __debug_tree_shape_info(WT_PAGE *page) { uint64_t v; - static char buf[32]; + static char buf[128]; + const char *unit; v = page->memory_footprint; - if (v >= WT_GIGABYTE) - (void)__wt_snprintf(buf, sizeof(buf), - "(%p %" PRIu64 "G)", (void *)page, v / WT_GIGABYTE); - else if (v >= WT_MEGABYTE) - (void)__wt_snprintf(buf, sizeof(buf), - "(%p %" PRIu64 "M)", (void *)page, v / WT_MEGABYTE); - else - (void)__wt_snprintf(buf, sizeof(buf), - "(%p %" PRIu64 ")", (void *)page, v); + + if (v > WT_GIGABYTE) { + v /= WT_GIGABYTE; + unit = "G"; + } else if (v > WT_MEGABYTE) { + v /= WT_MEGABYTE; + unit = "M"; + } else if (v > WT_KILOBYTE) { + v /= WT_KILOBYTE; + unit = "K"; + } else { + unit = "B"; + } + + (void)__wt_snprintf(buf, sizeof(buf), "(%p, %" PRIu64 + "%s, evict gen %" PRIu64 ", create gen %" PRIu64 ")", + (void *)page, v, unit, + page->evict_pass_gen, page->cache_create_gen); return (buf); } @@ -982,7 +992,7 @@ __debug_row_skip(WT_DBG *ds, WT_INSERT_HEAD *head) static int __debug_update(WT_DBG *ds, WT_UPDATE *upd, bool hexbyte) { - for (; upd != NULL; upd = upd->next) + for (; upd != NULL; upd = upd->next) { if (upd->type == WT_UPDATE_DELETED) WT_RET(ds->f(ds, "\tvalue {deleted}\n")); else if (upd->type == WT_UPDATE_RESERVED) @@ -995,6 +1005,30 @@ __debug_update(WT_DBG *ds, WT_UPDATE *upd, bool hexbyte) } else WT_RET(__debug_item(ds, "value", WT_UPDATE_DATA(upd), upd->size)); + WT_RET(ds->f(ds, "\t" "txn id %" PRIu64, upd->txnid)); + +#ifdef HAVE_TIMESTAMPS + if (!__wt_timestamp_iszero(upd->timestamp)) { +#if WT_TIMESTAMP_SIZE == 8 + { + uint64_t ts; + __wt_timestamp_set( + (uint8_t *)&ts, (uint8_t *)&upd->timestamp[0]); + ts = __wt_bswap64(ts); + WT_RET(ds->f(ds, ", stamp %" PRIu64, ts)); + } +#else + { + int i; + WT_RET(ds->f(ds, ", stamp 0x")); + for (i = 0; i < WT_TIMESTAMP_SIZE; ++i) + WT_RET(ds->f(ds, "%" PRIx8, upd->timestamp[i])); + } +#endif + } +#endif + WT_RET(ds->f(ds, "\n")); + } return (0); } diff --git a/src/third_party/wiredtiger/src/btree/bt_discard.c b/src/third_party/wiredtiger/src/btree/bt_discard.c index bfa8eb25aac..bc9356e2669 100644 --- a/src/third_party/wiredtiger/src/btree/bt_discard.c +++ b/src/third_party/wiredtiger/src/btree/bt_discard.c @@ -249,7 +249,8 @@ __free_page_modify(WT_SESSION_IMPL *session, WT_PAGE *page) /* Free the overflow on-page, reuse and transaction-cache skiplists. */ __wt_ovfl_reuse_free(session, page); - __wt_ovfl_txnc_free(session, page); + if (mod->ovfl_track != NULL) + __wt_free(session, mod->ovfl_track->remove); __wt_ovfl_discard_free(session, page); __wt_free(session, page->modify->ovfl_track); diff --git a/src/third_party/wiredtiger/src/btree/bt_handle.c b/src/third_party/wiredtiger/src/btree/bt_handle.c index 06fbd6b74c7..5fad76849dc 100644 --- a/src/third_party/wiredtiger/src/btree/bt_handle.c +++ b/src/third_party/wiredtiger/src/btree/bt_handle.c @@ -279,6 +279,9 @@ __btree_conf(WT_SESSION_IMPL *session, WT_CKPT *ckpt) bool fixed; const char **cfg, *enc_cfg[] = { NULL, NULL }; + WT_UNUSED(maj_version); /* !HAVE_VERBOSE */ + WT_UNUSED(min_version); /* !HAVE_VERBOSE */ + btree = S2BT(session); cfg = btree->dhandle->cfg; conn = S2C(session); @@ -290,7 +293,7 @@ __btree_conf(WT_SESSION_IMPL *session, WT_CKPT *ckpt) WT_RET(__wt_config_gets(session, cfg, "version.minor", &cval)); min_version = cval.val; __wt_verbose(session, WT_VERB_VERSION, - "%" PRIu64 ".%" PRIu64, maj_version, min_version); + "%" PRId64 ".%" PRId64, maj_version, min_version); } /* Get the file ID. */ diff --git a/src/third_party/wiredtiger/src/btree/bt_ovfl.c b/src/third_party/wiredtiger/src/btree/bt_ovfl.c index 76d1bfd121c..a0b1ff65006 100644 --- a/src/third_party/wiredtiger/src/btree/bt_ovfl.c +++ b/src/third_party/wiredtiger/src/btree/bt_ovfl.c @@ -48,6 +48,9 @@ __wt_ovfl_read(WT_SESSION_IMPL *session, WT_PAGE *page, WT_CELL_UNPACK *unpack, WT_ITEM *store) { WT_DECL_RET; + WT_OVFL_TRACK *track; + WT_UPDATE *upd; + size_t i; /* * If no page specified, there's no need to lock and there's no cache @@ -61,16 +64,24 @@ __wt_ovfl_read(WT_SESSION_IMPL *session, * WT_CELL_VALUE_OVFL_RM cells: If reconciliation deleted an overflow * value, but there was still a reader in the system that might need it, * the on-page cell type will have been reset to WT_CELL_VALUE_OVFL_RM - * and we will be passed a page so we can look-aside into the cache of - * such values. + * and we will be passed a page so we can check the on-page cell. * * Acquire the overflow lock, and retest the on-page cell's value inside * the lock. */ __wt_readlock(session, &S2BT(session)->ovfl_lock); - ret = __wt_cell_type_raw(unpack->cell) == WT_CELL_VALUE_OVFL_RM ? - __wt_ovfl_txnc_search(page, unpack->data, unpack->size, store) : - __ovfl_read(session, unpack->data, unpack->size, store); + if (__wt_cell_type_raw(unpack->cell) == WT_CELL_VALUE_OVFL_RM) { + track = page->modify->ovfl_track; + for (upd = NULL, i = 0; i < track->remove_next; ++i) + if (track->remove[i].cell == unpack->cell) { + upd = track->remove[i].upd; + break; + } + WT_ASSERT(session, i < track->remove_next); + store->data = WT_UPDATE_DATA(upd); + store->size = upd->size; + } else + ret = __ovfl_read(session, unpack->data, unpack->size, store); __wt_readunlock(session, &S2BT(session)->ovfl_lock); return (ret); @@ -98,8 +109,7 @@ __ovfl_cache_col_visible( * the value. */ if (__wt_cell_rle(unpack) == 1 && - upd != NULL && /* Sanity: upd should always be set. */ - __wt_txn_upd_visible_all(session, upd)) + WT_UPDATE_DATA_VALUE(upd) && __wt_txn_upd_visible_all(session, upd)) return (true); return (false); } @@ -109,51 +119,99 @@ __ovfl_cache_col_visible( * row-store: check for a globally visible update. */ static bool -__ovfl_cache_row_visible(WT_SESSION_IMPL *session, WT_PAGE *page, WT_ROW *rip) +__ovfl_cache_row_visible(WT_SESSION_IMPL *session, WT_UPDATE *upd) { - WT_UPDATE *upd; - /* Check to see if there's a globally visible update. */ - for (upd = WT_ROW_UPDATE(page, rip); upd != NULL; upd = upd->next) - if (__wt_txn_upd_visible_all(session, upd)) + for (; upd != NULL; upd = upd->next) + if (WT_UPDATE_DATA_VALUE(upd) && + __wt_txn_upd_visible_all(session, upd)) return (true); return (false); } /* - * __ovfl_cache -- - * Cache a deleted overflow value. + * __ovfl_cache_append_update -- + * Append an overflow value to the update list. */ static int -__ovfl_cache(WT_SESSION_IMPL *session, WT_PAGE *page, WT_CELL_UNPACK *unpack) +__ovfl_cache_append_update(WT_SESSION_IMPL *session, WT_PAGE *page, + WT_UPDATE *upd_list, WT_CELL_UNPACK *unpack, WT_UPDATE **updp) { WT_DECL_ITEM(tmp); WT_DECL_RET; - size_t addr_size; - const uint8_t *addr; + WT_UPDATE *append, *upd; + size_t size; - addr = unpack->data; - addr_size = unpack->size; + *updp = NULL; + /* Read the overflow value. */ WT_RET(__wt_scr_alloc(session, 1024, &tmp)); + WT_ERR(__ovfl_read(session, unpack->data, unpack->size, tmp)); + + /* + * Create an update entry with no transaction ID to ensure global + * visibility, append it to the update list. + * + * We don't need locks or barriers in this function: any thread reading + * the update list will see our newly appended record or not, it doesn't + * matter until the on-page cell is set to WT_CELL_VALUE_OVFL_RM. That + * involves atomic operations which will act as our barrier. Regardless, + * we update the page footprint as part of this operation, which acts as + * a barrier as well. + */ + WT_ERR(__wt_update_alloc( + session, tmp, &append, &size, WT_UPDATE_STANDARD)); + append->txnid = WT_TXN_NONE; + for (upd = upd_list; upd->next != NULL; upd = upd->next) + ; + WT_PUBLISH(upd->next, append); + + __wt_cache_page_inmem_incr(session, page, size); - /* Enter the value into the overflow cache. */ - WT_ERR(__ovfl_read(session, addr, addr_size, tmp)); - WT_ERR(__wt_ovfl_txnc_add( - session, page, addr, addr_size, tmp->data, tmp->size)); + *updp = append; err: __wt_scr_free(session, &tmp); return (ret); } /* - * __wt_ovfl_cache -- - * Handle deletion of an overflow value. + * __ovfl_cache -- + * Cache an overflow value. + */ +static int +__ovfl_cache(WT_SESSION_IMPL *session, + WT_PAGE *page, WT_UPDATE *upd_list, WT_CELL_UNPACK *unpack) +{ + WT_OVFL_TRACK *track; + WT_UPDATE *upd; + + /* Append a copy of the overflow value to the update list. */ + WT_RET(__ovfl_cache_append_update( + session, page, upd_list, unpack, &upd)); + + /* Allocating tracking structures as necessary. */ + if (page->modify->ovfl_track == NULL) + WT_RET(__wt_ovfl_track_init(session, page)); + track = page->modify->ovfl_track; + + /* Add the value's information to the update list. */ + WT_RET(__wt_realloc_def(session, + &track->remove_allocated, track->remove_next + 1, &track->remove)); + track->remove[track->remove_next].cell = unpack->cell; + track->remove[track->remove_next].upd = upd; + ++track->remove_next; + + return (0); +} + +/* + * __wt_ovfl_remove -- + * Remove an overflow value. */ int -__wt_ovfl_cache(WT_SESSION_IMPL *session, - WT_PAGE *page, void *cookie, WT_CELL_UNPACK *vpack) +__wt_ovfl_remove(WT_SESSION_IMPL *session, + WT_PAGE *page, WT_UPDATE *upd_list, WT_CELL_UNPACK *unpack) { bool visible; @@ -195,10 +253,10 @@ __wt_ovfl_cache(WT_SESSION_IMPL *session, */ switch (page->type) { case WT_PAGE_COL_VAR: - visible = __ovfl_cache_col_visible(session, cookie, vpack); + visible = __ovfl_cache_col_visible(session, upd_list, unpack); break; case WT_PAGE_ROW_LEAF: - visible = __ovfl_cache_row_visible(session, page, cookie); + visible = __ovfl_cache_row_visible(session, upd_list); break; WT_ILLEGAL_VALUE(session); } @@ -207,18 +265,15 @@ __wt_ovfl_cache(WT_SESSION_IMPL *session, * If there's no globally visible update, there's a reader in the system * that might try and read the old value, cache it. */ - if (!visible) { - WT_RET(__ovfl_cache(session, page, vpack)); - WT_STAT_CONN_INCR(session, cache_overflow_value); - WT_STAT_DATA_INCR(session, cache_overflow_value); - } + if (!visible) + WT_RET(__ovfl_cache(session, page, upd_list, unpack)); /* * Queue the on-page cell to be set to WT_CELL_VALUE_OVFL_RM and the * underlying overflow value's blocks to be freed when reconciliation * completes. */ - return (__wt_ovfl_discard_add(session, page, vpack->cell)); + return (__wt_ovfl_discard_add(session, page, unpack->cell)); } /* diff --git a/src/third_party/wiredtiger/src/btree/bt_page.c b/src/third_party/wiredtiger/src/btree/bt_page.c index ca5f05fe3dc..d7643e9ad26 100644 --- a/src/third_party/wiredtiger/src/btree/bt_page.c +++ b/src/third_party/wiredtiger/src/btree/bt_page.c @@ -116,6 +116,7 @@ err: if ((pindex = WT_INTL_INDEX_GET_SAFE(page)) != NULL) { __wt_cache_page_inmem_incr(session, page, size); (void)__wt_atomic_add64(&cache->bytes_read, size); (void)__wt_atomic_add64(&cache->pages_inmem, 1); + page->cache_create_gen = cache->evict_pass_gen; *pagep = page; return (0); diff --git a/src/third_party/wiredtiger/src/btree/bt_read.c b/src/third_party/wiredtiger/src/btree/bt_read.c index 5f8cf3a45e1..6a89f505c31 100644 --- a/src/third_party/wiredtiger/src/btree/bt_read.c +++ b/src/third_party/wiredtiger/src/btree/bt_read.c @@ -710,7 +710,7 @@ __btree_verbose_lookaside_read(WT_SESSION_IMPL *session) if (__wt_atomic_casv64(&conn->las_verb_gen_read, ckpt_gen_last, ckpt_gen_current)) { __wt_verbose(session, WT_VERB_LOOKASIDE, - "Read from lookaside file triggered."); + "%s", "Read from lookaside file triggered."); } } #else diff --git a/src/third_party/wiredtiger/src/btree/bt_slvg.c b/src/third_party/wiredtiger/src/btree/bt_slvg.c index eb39301abc7..1f686d904ec 100644 --- a/src/third_party/wiredtiger/src/btree/bt_slvg.c +++ b/src/third_party/wiredtiger/src/btree/bt_slvg.c @@ -1885,6 +1885,8 @@ __slvg_row_build_leaf( uint32_t i, skip_start, skip_stop; int cmp; + WT_UNUSED(ss); /* !HAVE_VERBOSE */ + btree = S2BT(session); page = NULL; diff --git a/src/third_party/wiredtiger/src/btree/row_srch.c b/src/third_party/wiredtiger/src/btree/row_srch.c index 76bebde7de7..3a9a6eb0f9b 100644 --- a/src/third_party/wiredtiger/src/btree/row_srch.c +++ b/src/third_party/wiredtiger/src/btree/row_srch.c @@ -591,13 +591,13 @@ leaf_match: cbt->compare = 0; */ if (base == 0) { cbt->compare = 1; - cbt->slot = WT_ROW_SLOT(page, page->pg_row); + cbt->slot = 0; F_SET(cbt, WT_CBT_SEARCH_SMALLEST); ins_head = WT_ROW_INSERT_SMALLEST(page); } else { cbt->compare = -1; - cbt->slot = WT_ROW_SLOT(page, page->pg_row + (base - 1)); + cbt->slot = base - 1; ins_head = WT_ROW_INSERT_SLOT(page, cbt->slot); } diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index 30b2b3e0e4d..dda27fec57c 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -55,6 +55,12 @@ static const WT_CONFIG_CHECK }; static const WT_CONFIG_CHECK + confchk_wiredtiger_open_compatibility_subconfigs[] = { + { "release", "string", NULL, NULL, NULL, 0 }, + { NULL, NULL, NULL, NULL, NULL, 0 } +}; + +static const WT_CONFIG_CHECK confchk_wiredtiger_open_eviction_subconfigs[] = { { "threads_max", "int", NULL, "min=1,max=20", NULL, 0 }, { "threads_min", "int", NULL, "min=1,max=20", NULL, 0 }, @@ -117,6 +123,9 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = { { "checkpoint", "category", NULL, NULL, confchk_wiredtiger_open_checkpoint_subconfigs, 2 }, + { "compatibility", "category", + NULL, NULL, + confchk_wiredtiger_open_compatibility_subconfigs, 1 }, { "diagnostic_timing_stress", "list", NULL, "choices=[\"checkpoint_slow\"]", NULL, 0 }, @@ -179,11 +188,20 @@ static const WT_CONFIG_CHECK confchk_WT_CURSOR_reconfigure[] = { { NULL, NULL, NULL, NULL, NULL, 0 } }; +static const WT_CONFIG_CHECK + confchk_WT_SESSION_create_log_subconfigs[] = { + { "enabled", "boolean", NULL, NULL, NULL, 0 }, + { NULL, NULL, NULL, NULL, NULL, 0 } +}; + static const WT_CONFIG_CHECK confchk_WT_SESSION_alter[] = { { "access_pattern_hint", "string", NULL, "choices=[\"none\",\"random\",\"sequential\"]", NULL, 0 }, { "cache_resident", "boolean", NULL, NULL, NULL, 0 }, + { "log", "category", + NULL, NULL, + confchk_WT_SESSION_create_log_subconfigs, 1 }, { NULL, NULL, NULL, NULL, NULL, 0 } }; @@ -230,12 +248,6 @@ static const WT_CONFIG_CHECK }; static const WT_CONFIG_CHECK - confchk_WT_SESSION_create_log_subconfigs[] = { - { "enabled", "boolean", NULL, NULL, NULL, 0 }, - { NULL, NULL, NULL, NULL, NULL, 0 } -}; - -static const WT_CONFIG_CHECK confchk_WT_SESSION_create_lsm_subconfigs[] = { { "auto_throttle", "boolean", NULL, NULL, NULL, 0 }, { "bloom", "boolean", NULL, NULL, NULL, 0 }, @@ -712,6 +724,9 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { NULL, NULL, confchk_wiredtiger_open_checkpoint_subconfigs, 2 }, { "checkpoint_sync", "boolean", NULL, NULL, NULL, 0 }, + { "compatibility", "category", + NULL, NULL, + confchk_wiredtiger_open_compatibility_subconfigs, 1 }, { "config_base", "boolean", NULL, NULL, NULL, 0 }, { "create", "boolean", NULL, NULL, NULL, 0 }, { "diagnostic_timing_stress", "list", @@ -803,6 +818,9 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { NULL, NULL, confchk_wiredtiger_open_checkpoint_subconfigs, 2 }, { "checkpoint_sync", "boolean", NULL, NULL, NULL, 0 }, + { "compatibility", "category", + NULL, NULL, + confchk_wiredtiger_open_compatibility_subconfigs, 1 }, { "config_base", "boolean", NULL, NULL, NULL, 0 }, { "create", "boolean", NULL, NULL, NULL, 0 }, { "diagnostic_timing_stress", "list", @@ -895,6 +913,9 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { NULL, NULL, confchk_wiredtiger_open_checkpoint_subconfigs, 2 }, { "checkpoint_sync", "boolean", NULL, NULL, NULL, 0 }, + { "compatibility", "category", + NULL, NULL, + confchk_wiredtiger_open_compatibility_subconfigs, 1 }, { "diagnostic_timing_stress", "list", NULL, "choices=[\"checkpoint_slow\"]", NULL, 0 }, @@ -981,6 +1002,9 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { NULL, NULL, confchk_wiredtiger_open_checkpoint_subconfigs, 2 }, { "checkpoint_sync", "boolean", NULL, NULL, NULL, 0 }, + { "compatibility", "category", + NULL, NULL, + confchk_wiredtiger_open_compatibility_subconfigs, 1 }, { "diagnostic_timing_stress", "list", NULL, "choices=[\"checkpoint_slow\"]", NULL, 0 }, @@ -1099,18 +1123,18 @@ static const WT_CONFIG_ENTRY config_entries[] = { { "WT_CONNECTION.reconfigure", "async=(enabled=false,ops_max=1024,threads=2),cache_overhead=8," "cache_size=100MB,checkpoint=(log_size=0,wait=0)," - "diagnostic_timing_stress=,error_prefix=,eviction=(threads_max=8," - "threads_min=1),eviction_checkpoint_target=5," - "eviction_dirty_target=5,eviction_dirty_trigger=20," - "eviction_target=80,eviction_trigger=95," - "file_manager=(close_handle_minimum=250,close_idle_time=30," + "compatibility=(release=),diagnostic_timing_stress=,error_prefix=" + ",eviction=(threads_max=8,threads_min=1)," + "eviction_checkpoint_target=5,eviction_dirty_target=5," + "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" + ",file_manager=(close_handle_minimum=250,close_idle_time=30," "close_scan_interval=10),log=(archive=true,prealloc=true," "zero_fill=false),lsm_manager=(merge=true,worker_thread_max=4)," "lsm_merge=true,shared_cache=(chunk=10MB,name=,quota=0,reserve=0," "size=500MB),statistics=none,statistics_log=(json=false," "on_close=false,sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," "verbose=", - confchk_WT_CONNECTION_reconfigure, 20 + confchk_WT_CONNECTION_reconfigure, 21 }, { "WT_CONNECTION.set_file_system", "", @@ -1129,8 +1153,8 @@ static const WT_CONFIG_ENTRY config_entries[] = { confchk_WT_CURSOR_reconfigure, 2 }, { "WT_SESSION.alter", - "access_pattern_hint=none,cache_resident=false", - confchk_WT_SESSION_alter, 2 + "access_pattern_hint=none,cache_resident=false,log=(enabled=true)", + confchk_WT_SESSION_alter, 3 }, { "WT_SESSION.begin_transaction", "isolation=,name=,priority=0,read_timestamp=,snapshot=,sync=", @@ -1318,10 +1342,10 @@ static const WT_CONFIG_ENTRY config_entries[] = { "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1" ",builtin_extension_config=,cache_overhead=8,cache_size=100MB," "checkpoint=(log_size=0,wait=0),checkpoint_sync=true," - "config_base=true,create=false,diagnostic_timing_stress=," - "direct_io=,encryption=(keyid=,name=,secretkey=),error_prefix=," - "eviction=(threads_max=8,threads_min=1)," - "eviction_checkpoint_target=5,eviction_dirty_target=5," + "compatibility=(release=),config_base=true,create=false," + "diagnostic_timing_stress=,direct_io=,encryption=(keyid=,name=," + "secretkey=),error_prefix=,eviction=(threads_max=8,threads_min=1)" + ",eviction_checkpoint_target=5,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" ",exclusive=false,extensions=,file_extend=," "file_manager=(close_handle_minimum=250,close_idle_time=30," @@ -1336,16 +1360,16 @@ static const WT_CONFIG_ENTRY config_entries[] = { ",wait=0),transaction_sync=(enabled=false,method=fsync)," "use_environment=true,use_environment_priv=false,verbose=," "write_through=", - confchk_wiredtiger_open, 41 + confchk_wiredtiger_open, 42 }, { "wiredtiger_open_all", "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1" ",builtin_extension_config=,cache_overhead=8,cache_size=100MB," "checkpoint=(log_size=0,wait=0),checkpoint_sync=true," - "config_base=true,create=false,diagnostic_timing_stress=," - "direct_io=,encryption=(keyid=,name=,secretkey=),error_prefix=," - "eviction=(threads_max=8,threads_min=1)," - "eviction_checkpoint_target=5,eviction_dirty_target=5," + "compatibility=(release=),config_base=true,create=false," + "diagnostic_timing_stress=,direct_io=,encryption=(keyid=,name=," + "secretkey=),error_prefix=,eviction=(threads_max=8,threads_min=1)" + ",eviction_checkpoint_target=5,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" ",exclusive=false,extensions=,file_extend=," "file_manager=(close_handle_minimum=250,close_idle_time=30," @@ -1360,15 +1384,16 @@ static const WT_CONFIG_ENTRY config_entries[] = { ",wait=0),transaction_sync=(enabled=false,method=fsync)," "use_environment=true,use_environment_priv=false,verbose=," "version=(major=0,minor=0),write_through=", - confchk_wiredtiger_open_all, 42 + confchk_wiredtiger_open_all, 43 }, { "wiredtiger_open_basecfg", "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1" ",builtin_extension_config=,cache_overhead=8,cache_size=100MB," "checkpoint=(log_size=0,wait=0),checkpoint_sync=true," - "diagnostic_timing_stress=,direct_io=,encryption=(keyid=,name=," - "secretkey=),error_prefix=,eviction=(threads_max=8,threads_min=1)" - ",eviction_checkpoint_target=5,eviction_dirty_target=5," + "compatibility=(release=),diagnostic_timing_stress=,direct_io=," + "encryption=(keyid=,name=,secretkey=),error_prefix=," + "eviction=(threads_max=8,threads_min=1)," + "eviction_checkpoint_target=5,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" ",extensions=,file_extend=,file_manager=(close_handle_minimum=250" ",close_idle_time=30,close_scan_interval=10),hazard_max=1000," @@ -1381,15 +1406,16 @@ static const WT_CONFIG_ENTRY config_entries[] = { ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\"" ",wait=0),transaction_sync=(enabled=false,method=fsync),verbose=," "version=(major=0,minor=0),write_through=", - confchk_wiredtiger_open_basecfg, 36 + confchk_wiredtiger_open_basecfg, 37 }, { "wiredtiger_open_usercfg", "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1" ",builtin_extension_config=,cache_overhead=8,cache_size=100MB," "checkpoint=(log_size=0,wait=0),checkpoint_sync=true," - "diagnostic_timing_stress=,direct_io=,encryption=(keyid=,name=," - "secretkey=),error_prefix=,eviction=(threads_max=8,threads_min=1)" - ",eviction_checkpoint_target=5,eviction_dirty_target=5," + "compatibility=(release=),diagnostic_timing_stress=,direct_io=," + "encryption=(keyid=,name=,secretkey=),error_prefix=," + "eviction=(threads_max=8,threads_min=1)," + "eviction_checkpoint_target=5,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" ",extensions=,file_extend=,file_manager=(close_handle_minimum=250" ",close_idle_time=30,close_scan_interval=10),hazard_max=1000," @@ -1402,7 +1428,7 @@ static const WT_CONFIG_ENTRY config_entries[] = { ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\"" ",wait=0),transaction_sync=(enabled=false,method=fsync),verbose=," "write_through=", - confchk_wiredtiger_open_usercfg, 35 + confchk_wiredtiger_open_usercfg, 36 }, { NULL, NULL, NULL, 0 } }; diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index cb0aa17791e..09eceac0a3b 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -190,6 +190,45 @@ __wt_conn_remove_collator(WT_SESSION_IMPL *session) } /* + * __conn_compat_config -- + * Configure compatibility version. + */ +static int +__conn_compat_config(WT_SESSION_IMPL *session, const char **cfg) +{ + WT_CONFIG_ITEM cval; + WT_CONNECTION_IMPL *conn; + uint16_t patch; + + conn = S2C(session); + WT_RET(__wt_config_gets(session, cfg, + "compatibility.release", &cval)); + if (cval.len != 0) { + /* + * Accept either a major.minor release string or a + * major.minor.patch release string. We ignore the patch + * value, but allow it in the string. + */ + if (sscanf(cval.str, "%" SCNu16 ".%" SCNu16, + &conn->compat_major, &conn->compat_minor) != 2 && + sscanf(cval.str, "%" SCNu16 ".%" SCNu16 ".%" SCNu16, + &conn->compat_major, &conn->compat_minor, &patch) != 3) + WT_RET_MSG(session, + EINVAL, "illegal compatibility release"); + if (conn->compat_major > WIREDTIGER_VERSION_MAJOR) + WT_RET_MSG(session, EINVAL, "unknown major version"); + if (conn->compat_major == WIREDTIGER_VERSION_MAJOR && + conn->compat_minor > WIREDTIGER_VERSION_MINOR) + WT_RET_MSG(session, + EINVAL, "illegal compatibility version"); + } else { + conn->compat_major = WIREDTIGER_VERSION_MAJOR; + conn->compat_minor = WIREDTIGER_VERSION_MINOR; + } + return (0); +} + +/* * __compressor_confchk -- * Validate the compressor. */ @@ -1134,6 +1173,7 @@ __conn_reconfigure(WT_CONNECTION *wt_conn, const char *config) cfg[1] = config; /* Second, reconfigure the system. */ + WT_ERR(__conn_compat_config(session, cfg)); WT_ERR(__conn_statistics_config(session, cfg)); WT_ERR(__wt_async_reconfig(session, cfg)); WT_ERR(__wt_cache_config(session, true, cfg)); @@ -2018,6 +2058,7 @@ __conn_write_base_config(WT_SESSION_IMPL *session, const char *cfg[]) * merge the rest to be written. */ WT_ERR(__wt_config_merge(session, cfg + 1, + "compatibility=(release=)," "config_base=," "create=," "encryption=(secretkey=)," @@ -2287,6 +2328,11 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, WT_ERR(__conn_load_extensions(session, cfg, true)); /* + * Set compatibility versions early so that any subsystem sees it. + */ + WT_ERR(__conn_compat_config(session, cfg)); + + /* * If the application didn't configure its own file system, configure * one of ours. Check to ensure we have a valid file system. */ diff --git a/src/third_party/wiredtiger/src/conn/conn_cache_pool.c b/src/third_party/wiredtiger/src/conn/conn_cache_pool.c index adc2e2bffc3..55393a86ba1 100644 --- a/src/third_party/wiredtiger/src/conn/conn_cache_pool.c +++ b/src/third_party/wiredtiger/src/conn/conn_cache_pool.c @@ -372,8 +372,8 @@ __wt_conn_cache_pool_destroy(WT_SESSION_IMPL *session) } if (!F_ISSET(cp, WT_CACHE_POOL_ACTIVE)) { - __wt_verbose( - session, WT_VERB_SHARED_CACHE, "Destroying cache pool"); + __wt_verbose(session, + WT_VERB_SHARED_CACHE, "%s", "Destroying cache pool"); __wt_spin_lock(session, &__wt_process.spinlock); /* * We have been holding the pool lock - no connections could @@ -401,7 +401,7 @@ __wt_conn_cache_pool_destroy(WT_SESSION_IMPL *session) /* Notify other participants if we were managing */ if (FLD_ISSET(cache->pool_flags, WT_CACHE_POOL_MANAGER)) { cp->pool_managed = 0; - __wt_verbose(session, WT_VERB_SHARED_CACHE, + __wt_verbose(session, WT_VERB_SHARED_CACHE, "%s", "Shutting down shared cache manager connection"); } } @@ -585,8 +585,8 @@ __cache_pool_adjust(WT_SESSION_IMPL *session, if (WT_VERBOSE_ISSET(session, WT_VERB_SHARED_CACHE)) { __wt_verbose(session, - WT_VERB_SHARED_CACHE, "Cache pool distribution: "); - __wt_verbose(session, WT_VERB_SHARED_CACHE, + WT_VERB_SHARED_CACHE, "%s", "Cache pool distribution: "); + __wt_verbose(session, WT_VERB_SHARED_CACHE, "%s", "\t" "cache (MB), pressure, skips, busy, %% full:"); } @@ -731,7 +731,7 @@ __cache_pool_adjust(WT_SESSION_IMPL *session, cp->currently_used -= adjustment; } __wt_verbose(session, WT_VERB_SHARED_CACHE, - "Allocated %s%" PRId64 " to %s", + "Allocated %s%" PRIu64 " to %s", grow ? "" : "-", adjustment, entry->home); /* @@ -778,7 +778,7 @@ __wt_cache_pool_server(void *arg) if (__wt_atomic_cas8(&cp->pool_managed, 0, 1)) { FLD_SET(cache->pool_flags, WT_CACHE_POOL_MANAGER); __wt_verbose(session, WT_VERB_SHARED_CACHE, - "Cache pool switched manager thread"); + "%s", "Cache pool switched manager thread"); } /* diff --git a/src/third_party/wiredtiger/src/conn/conn_log.c b/src/third_party/wiredtiger/src/conn/conn_log.c index 37acbe4a1a4..a6290e4ed92 100644 --- a/src/third_party/wiredtiger/src/conn/conn_log.c +++ b/src/third_party/wiredtiger/src/conn/conn_log.c @@ -40,6 +40,114 @@ __logmgr_sync_cfg(WT_SESSION_IMPL *session, const char **cfg) } /* + * __logmgr_force_ckpt -- + * Force a checkpoint out, waiting for the checkpoint LSN in the log + * is up to the given log number. + */ +static int +__logmgr_force_ckpt(WT_SESSION_IMPL *session, uint32_t lognum) +{ + WT_CONNECTION_IMPL *conn; + WT_LOG *log; + WT_SESSION_IMPL *tmp_session; + int yield; + + conn = S2C(session); + log = conn->log; + yield = 0; + WT_RET(__wt_open_internal_session(conn, + "compatibility-reconfig", true, 0, &tmp_session)); + while (log->ckpt_lsn.l.file < lognum) { + /* + * Force a checkpoint to be written in the new log file and + * force the archiving of all previous log files. We do the + * checkpoint in the loop because the checkpoint LSN in the + * log record could still reflect the previous log file in + * cases such as the write LSN has not yet advanced into the + * new log file due to another group of threads still in + * progress with their slot copies or writes. + */ + WT_RET(tmp_session->iface.checkpoint( + &tmp_session->iface, "force=1")); + WT_RET(WT_SESSION_CHECK_PANIC(tmp_session)); + /* + * Only sleep in the rare case that we had to come through + * this loop more than once. + */ + if (yield++) { + WT_STAT_CONN_INCR(session, log_force_ckpt_sleep); + __wt_sleep(0, WT_THOUSAND); + } + } + WT_RET(tmp_session->iface.close(&tmp_session->iface, NULL)); + return (0); +} + +/* + * __logmgr_version -- + * Set up the versions in the log manager. + */ +static int +__logmgr_version(WT_SESSION_IMPL *session, bool reconfig) +{ + WT_CONNECTION_IMPL *conn; + WT_LOG *log; + bool downgrade; + uint32_t first_record, lognum; + uint16_t new_version; + + conn = S2C(session); + log = conn->log; + if (log == NULL) + return (0); + + /* + * Set the log file format versions based on compatibility versions + * set in the connection. We must set this before we call log_open + * to open or create a log file. + * + * Since the log version changed at a major release number we only need + * to check the major number, not the minor number in the compatibility + * setting. + */ + if (conn->compat_major < WT_LOG_V2) { + new_version = 1; + first_record = WT_LOG_END_HEADER; + downgrade = true; + } else { + new_version = WT_LOG_VERSION; + first_record = WT_LOG_END_HEADER + log->allocsize; + downgrade = false; + } + + /* + * If the version is the same, there is nothing to do. + */ + if (log->log_version == new_version) + return (0); + /* + * If we are reconfiguring and at a new version we need to force + * the log file to advance so that we write out a log file at the + * correct version. When we are downgrading we must force a checkpoint + * and finally archive, even if disabled, so that all new version log + * files are gone. + * + * All of the version changes must be handled with locks on reconfigure + * because other threads may be changing log files, using pre-allocated + * files. + */ + /* + * Set the version. If it is a live change the logging subsystem will + * do other work as well to move to a new log file. + */ + WT_RET(__wt_log_set_version(session, new_version, + first_record, downgrade, reconfig, &lognum)); + if (reconfig && FLD_ISSET(conn->log_flags, WT_CONN_LOG_DOWNGRADED)) + WT_RET(__logmgr_force_ckpt(session, lognum)); + return (0); +} + +/* * __logmgr_config -- * Parse and setup the logging server options. */ @@ -187,7 +295,8 @@ __wt_logmgr_reconfig(WT_SESSION_IMPL *session, const char **cfg) { bool dummy; - return (__logmgr_config(session, cfg, &dummy, true)); + WT_RET(__logmgr_config(session, cfg, &dummy, true)); + return (__logmgr_version(session, true)); } /* @@ -329,19 +438,17 @@ err: __wt_err(session, ret, "log pre-alloc server error"); * currently running. */ int -__wt_log_truncate_files( - WT_SESSION_IMPL *session, WT_CURSOR *cursor, const char *cfg[]) +__wt_log_truncate_files(WT_SESSION_IMPL *session, WT_CURSOR *cursor, bool force) { WT_CONNECTION_IMPL *conn; WT_DECL_RET; WT_LOG *log; uint32_t backup_file; - WT_UNUSED(cfg); conn = S2C(session); if (!FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED)) return (0); - if (F_ISSET(conn, WT_CONN_SERVER_LOG) && + if (!force && F_ISSET(conn, WT_CONN_SERVER_LOG) && FLD_ISSET(conn->log_flags, WT_CONN_LOG_ARCHIVE)) WT_RET_MSG(session, EINVAL, "Attempt to archive manually while a server is running"); @@ -349,8 +456,10 @@ __wt_log_truncate_files( log = conn->log; backup_file = 0; - if (cursor != NULL) + if (cursor != NULL) { + WT_ASSERT(session, force == false); backup_file = WT_CURSOR_BACKUP_ID(cursor); + } WT_ASSERT(session, backup_file <= log->alloc_lsn.l.file); __wt_verbose(session, WT_VERB_LOG, "log_truncate_files: Archive once up to %" PRIu32, backup_file); @@ -828,7 +937,7 @@ __log_server(void *arg) session, &log->log_archive_lock); WT_ERR(ret); } else - __wt_verbose(session, WT_VERB_LOG, + __wt_verbose(session, WT_VERB_LOG, "%s", "log_archive: Blocked due to open " "log cursor holding archive lock"); } @@ -875,6 +984,7 @@ __wt_logmgr_create(WT_SESSION_IMPL *session, const char *cfg[]) WT_RET(__wt_calloc_one(session, &conn->log)); log = conn->log; WT_RET(__wt_spin_init(session, &log->log_lock, "log")); + WT_RET(__wt_spin_init(session, &log->log_fs_lock, "log files")); WT_RET(__wt_spin_init(session, &log->log_slot_lock, "log slot")); WT_RET(__wt_spin_init(session, &log->log_sync_lock, "log sync")); WT_RET(__wt_spin_init(session, &log->log_writelsn_lock, @@ -898,6 +1008,8 @@ __wt_logmgr_create(WT_SESSION_IMPL *session, const char *cfg[]) WT_INIT_LSN(&log->write_lsn); WT_INIT_LSN(&log->write_start_lsn); log->fileid = 0; + WT_RET(__logmgr_version(session, false)); + WT_RET(__wt_cond_alloc(session, "log sync", &log->log_sync_cond)); WT_RET(__wt_cond_alloc(session, "log write", &log->log_write_cond)); WT_RET(__wt_log_open(session)); @@ -1050,6 +1162,7 @@ __wt_logmgr_destroy(WT_SESSION_IMPL *session) __wt_cond_destroy(session, &conn->log->log_write_cond); __wt_rwlock_destroy(session, &conn->log->log_archive_lock); __wt_spin_destroy(session, &conn->log->log_lock); + __wt_spin_destroy(session, &conn->log->log_fs_lock); __wt_spin_destroy(session, &conn->log->log_slot_lock); __wt_spin_destroy(session, &conn->log->log_sync_lock); __wt_spin_destroy(session, &conn->log->log_writelsn_lock); diff --git a/src/third_party/wiredtiger/src/conn/conn_stat.c b/src/third_party/wiredtiger/src/conn/conn_stat.c index f38d81a7f7a..4559a72a46e 100644 --- a/src/third_party/wiredtiger/src/conn/conn_stat.c +++ b/src/third_party/wiredtiger/src/conn/conn_stat.c @@ -221,6 +221,101 @@ err: __stat_sources_free(session, &sources); } /* + * __statlog_print_header -- + * Write the header for statistics when running in JSON mode. + */ +static int +__statlog_print_header(WT_SESSION_IMPL *session) +{ + WT_CONNECTION_IMPL *conn; + + conn = S2C(session); + + if (!FLD_ISSET(conn->stat_flags, WT_STAT_JSON)) + return (0); + + /* + * This flag is required in order to generate correct JSON when printing + * out stats for individual tables. When we are about to print the first + * table's stats we must print out the wiredTigerTables header once + * only and add a correct closing brace when we finish the tables + * section. To do this we maintain a flag variable to note when we have + * printed the first table. Unfortunately, the mechanism which we use + * to print stats for each table does not allow passing of variables + * by reference, this necessitates the use of a variable on the + * connection. The variable is safe as the JSON printing logic is only + * performed by the single threaded stat server. + */ + conn->stat_json_tables = false; + WT_RET(__wt_fprintf(session, conn->stat_fs, + "{\"version\":\"%s\",\"localTime\":\"%s\"", + WIREDTIGER_VERSION_STRING, conn->stat_stamp)); + return (0); +} + +/* + * __statlog_print_table_name -- + * Write the header for the wiredTigerTables section of statistics if + * running in JSON mode and the header has not been written this round, + * then print the name of the table. + */ +static int +__statlog_print_table_name( + WT_SESSION_IMPL *session, const char *name, bool conn_stats) +{ + WT_CONNECTION_IMPL *conn; + + conn = S2C(session); + + /* + * If printing the connection stats, write that header and we are done. + */ + if (conn_stats) { + WT_RET(__wt_fprintf( + session, conn->stat_fs, ",\"wiredTiger\":{")); + return (0); + } + + /* + * If this is the first table we are printing stats for print the header + * for the wiredTigerTables section. Otherwise print a comma as this + * is a subsequent table. + */ + if (conn->stat_json_tables) + WT_RET(__wt_fprintf(session, conn->stat_fs,",")); + else { + conn->stat_json_tables = true; + WT_RET(__wt_fprintf(session, + conn->stat_fs,",\"wiredTigerTables\":{")); + } + WT_RET(__wt_fprintf(session, conn->stat_fs, "\"%s\":{", name)); + return (0); +} + +/* + * __statlog_print_footer -- + * Write the footer for statistics when running in JSON mode. + */ +static int +__statlog_print_footer(WT_SESSION_IMPL *session) +{ + WT_CONNECTION_IMPL *conn; + + conn = S2C(session); + + if (!FLD_ISSET(conn->stat_flags, WT_STAT_JSON)) + return (0); + + /* If we have printed a tables stats, then close that section. */ + if (conn->stat_json_tables) { + WT_RET(__wt_fprintf(session, conn->stat_fs, "}")); + conn->stat_json_tables = false; + } + WT_RET(__wt_fprintf(session, conn->stat_fs, "}\n")); + return (0); +} + +/* * __statlog_dump -- * Dump out handle/connection statistics. */ @@ -264,14 +359,10 @@ __statlog_dump(WT_SESSION_IMPL *session, const char *name, bool conn_stats) goto err; } - if (FLD_ISSET(conn->stat_flags, WT_STAT_JSON)) { - WT_ERR(__wt_fprintf(session, conn->stat_fs, - "{\"version\":\"%s\",\"localTime\":\"%s\"", - WIREDTIGER_VERSION_STRING, conn->stat_stamp)); - WT_ERR(__wt_fprintf( - session, conn->stat_fs, ",\"wiredTiger\":{")); - while ((ret = cursor->next(cursor)) == 0) { - WT_ERR(cursor->get_value(cursor, &desc, &valstr, &val)); + WT_ERR(__statlog_print_table_name(session, name, conn_stats)); + while ((ret = cursor->next(cursor)) == 0) { + WT_ERR(cursor->get_value(cursor, &desc, &valstr, &val)); + if (FLD_ISSET(conn->stat_flags, WT_STAT_JSON)) { /* Check if we are starting a new section. */ endprefix = strchr(desc, ':'); prefixlen = WT_PTRDIFF(endprefix, desc); @@ -291,18 +382,15 @@ __statlog_dump(WT_SESSION_IMPL *session, const char *name, bool conn_stats) "%s\"%s\":%" PRId64, groupfirst ? "" : ",", endprefix + 2, val)); groupfirst = false; - } - WT_ERR_NOTFOUND_OK(ret); - WT_ERR(__wt_fprintf(session, conn->stat_fs, "}}}\n")); - } else { - while ((ret = cursor->next(cursor)) == 0) { - WT_ERR(cursor->get_value(cursor, &desc, &valstr, &val)); + } else { WT_ERR(__wt_fprintf(session, conn->stat_fs, "%s %" PRId64 " %s %s\n", conn->stat_stamp, val, name, desc)); } - WT_ERR_NOTFOUND_OK(ret); } + WT_ERR_NOTFOUND_OK(ret); + if (FLD_ISSET(conn->stat_flags, WT_STAT_JSON)) + WT_ERR(__wt_fprintf(session, conn->stat_fs, "}}")); err: __wt_scr_free(session, &tmp); if (cursor != NULL) @@ -437,6 +525,7 @@ __statlog_log_one(WT_SESSION_IMPL *session, WT_ITEM *path, WT_ITEM *tmp) if (strftime(tmp->mem, tmp->memsize, conn->stat_format, tm) == 0) WT_RET_MSG(session, ENOMEM, "strftime timestamp conversion"); conn->stat_stamp = tmp->mem; + WT_RET(__statlog_print_header(session)); /* Dump the connection statistics. */ WT_RET(__statlog_dump(session, conn->home, true)); @@ -459,6 +548,7 @@ __statlog_log_one(WT_SESSION_IMPL *session, WT_ITEM *path, WT_ITEM *tmp) */ if (conn->stat_sources != NULL) WT_RET(__statlog_lsm_apply(session)); + WT_RET(__statlog_print_footer(session)); /* Flush. */ return (__wt_fflush(session, conn->stat_fs)); diff --git a/src/third_party/wiredtiger/src/cursor/cur_backup.c b/src/third_party/wiredtiger/src/cursor/cur_backup.c index 60750b88900..490c3813cad 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_backup.c +++ b/src/third_party/wiredtiger/src/cursor/cur_backup.c @@ -210,6 +210,8 @@ __backup_start( cb->list = NULL; cb->list_next = 0; + WT_RET(__wt_inmem_unsupported_op(session, "backup cursor")); + /* * Single thread hot backups: we're holding the schema lock, so we * know we'll serialize with other attempts to start a hot backup. diff --git a/src/third_party/wiredtiger/src/cursor/cur_log.c b/src/third_party/wiredtiger/src/cursor/cur_log.c index 38e9d4a1784..d1166c5d402 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_log.c +++ b/src/third_party/wiredtiger/src/cursor/cur_log.c @@ -303,7 +303,6 @@ __curlog_close(WT_CURSOR *cursor) cl = (WT_CURSOR_LOG *)cursor; conn = S2C(session); - WT_ASSERT(session, FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED)); if (F_ISSET(cl, WT_CURLOG_ARCHIVE_LOCK)) __wt_readunlock(session, &conn->log->log_archive_lock); @@ -355,9 +354,6 @@ __wt_curlog_open(WT_SESSION_IMPL *session, WT_STATIC_ASSERT(offsetof(WT_CURSOR_LOG, iface) == 0); conn = S2C(session); - if (!FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED)) - WT_RET_MSG(session, EINVAL, - "Cannot open a log cursor without logging enabled"); log = conn->log; cl = NULL; @@ -378,15 +374,17 @@ __wt_curlog_open(WT_SESSION_IMPL *session, WT_ERR(__wt_cursor_init(cursor, uri, NULL, cfg, cursorp)); - /* - * The user may be trying to read a log record they just wrote. - * Log records may be buffered, so force out any now. - */ - WT_ERR(__wt_log_force_write(session, 1, NULL)); + if (log != NULL) { + /* + * The user may be trying to read a log record they just wrote. + * Log records may be buffered, so force out any now. + */ + WT_ERR(__wt_log_force_write(session, 1, NULL)); - /* Log cursors block archiving. */ - __wt_readlock(session, &log->log_archive_lock); - F_SET(cl, WT_CURLOG_ARCHIVE_LOCK); + /* Log cursors block archiving. */ + __wt_readlock(session, &log->log_archive_lock); + F_SET(cl, WT_CURLOG_ARCHIVE_LOCK); + } if (0) { err: WT_TRET(__curlog_close(cursor)); diff --git a/src/third_party/wiredtiger/src/docs/upgrading.dox b/src/third_party/wiredtiger/src/docs/upgrading.dox index 8640991e7cd..fc0fc4e022a 100644 --- a/src/third_party/wiredtiger/src/docs/upgrading.dox +++ b/src/third_party/wiredtiger/src/docs/upgrading.dox @@ -1,6 +1,29 @@ /*! @page upgrading Upgrading WiredTiger applications -@section version_293 Upgrading to Version 2.9.3 +@section version_300 Upgrading to Version 3.0.0 +<dl> + +<dt>WiredTiger on-disk log file format change</dt> +<dd> +The WiredTiger on-disk file format for write-ahead log files has changed +to include a new internal system log record. +</dd> + +<dt>Addition of compatibility configuration setting</dt> +<dd> +There is now a compatibility setting that allows the user to upgrade or +downgrade the log files generated so that they can be run with an older +release. +</dd> + +<dt>Cached overflow record statistics</dt> +<dd> +The \c cache_overflow_value ("overflow values cached in memory") statistic +has been removed, it no longer has any meaning. +</dd> + +</dl><hr> +@section version_292 Upgrading to Version 2.9.2 <dl> <dt>Logging subsystem statistics</dt> @@ -12,11 +35,7 @@ are no longer present. They were duplicates of \c log_writes and been added. </dd> -</dl><hr> -@section version_292 Upgrading to Version 2.9.2 -<dl> - -<dt>WiredTiger utility now supports truncate</dt> +<dt>WiredTiger Utility now supports truncate</dt> <dd> The WiredTiger utility \c wt can now \c truncate objects, removing all contents from the specified object. diff --git a/src/third_party/wiredtiger/src/evict/evict_lru.c b/src/third_party/wiredtiger/src/evict/evict_lru.c index 4f4ecdf286c..1f26949c94f 100644 --- a/src/third_party/wiredtiger/src/evict/evict_lru.c +++ b/src/third_party/wiredtiger/src/evict/evict_lru.c @@ -305,12 +305,14 @@ __wt_evict_thread_run(WT_SESSION_IMPL *session, WT_THREAD *thread) F_ISSET(thread, WT_THREAD_RUN)) __wt_yield(); else { - __wt_verbose(session, WT_VERB_EVICTSERVER, "sleeping"); + __wt_verbose(session, + WT_VERB_EVICTSERVER, "%s", "sleeping"); /* Don't rely on signals: check periodically. */ __wt_cond_auto_wait(session, cache->evict_cond, did_work, NULL); - __wt_verbose(session, WT_VERB_EVICTSERVER, "waking"); + __wt_verbose(session, + WT_VERB_EVICTSERVER, "%s", "waking"); } } else WT_ERR(__evict_lru_pages(session, false)); @@ -351,8 +353,8 @@ __wt_evict_thread_stop(WT_SESSION_IMPL *session, WT_THREAD *thread) */ WT_ASSERT(session, F_ISSET(conn, WT_CONN_CLOSING | WT_CONN_RECOVERING)); - __wt_verbose( - session, WT_VERB_EVICTSERVER, "cache eviction thread exiting"); + __wt_verbose(session, + WT_VERB_EVICTSERVER, "%s", "cache eviction thread exiting"); if (0) { err: WT_PANIC_MSG(session, ret, "cache eviction thread error"); @@ -515,7 +517,7 @@ __wt_evict_destroy(WT_SESSION_IMPL *session) __wt_evict_server_wake(session); __wt_verbose( - session, WT_VERB_EVICTSERVER, "waiting for helper threads"); + session, WT_VERB_EVICTSERVER, "%s", "waiting for helper threads"); /* * We call the destroy function still holding the write lock. @@ -740,7 +742,7 @@ __evict_pass(WT_SESSION_IMPL *session) WT_STAT_CONN_INCR(session, cache_eviction_slow); __wt_verbose(session, WT_VERB_EVICTSERVER, - "unable to reach eviction goal"); + "%s", "unable to reach eviction goal"); break; } else { if (cache->evict_aggressive_score > 0) { @@ -779,6 +781,7 @@ __evict_clear_walk(WT_SESSION_IMPL *session) return (0); WT_STAT_CONN_INCR(session, cache_eviction_walks_abandoned); + WT_STAT_DATA_INCR(session, cache_eviction_walks_abandoned); /* * Clear evict_ref before releasing it in case that forces eviction (we @@ -957,6 +960,13 @@ __evict_tune_workers(WT_SESSION_IMPL *session) conn = S2C(session); cache = conn->cache; + /* + * If we have a fixed number of eviction threads, there is no value in + * calculating if we should do any tuning. + */ + if (conn->evict_threads_max == conn->evict_threads_min) + return; + WT_ASSERT(session, conn->evict_threads.threads[0]->session == session); pgs_evicted_cur = 0; @@ -1106,8 +1116,8 @@ __evict_tune_workers(WT_SESSION_IMPL *session) &conn->evict_threads, false); WT_STAT_CONN_INCR(session, cache_eviction_worker_created); - __wt_verbose(session, WT_VERB_EVICTSERVER, - "added worker thread"); + __wt_verbose(session, + WT_VERB_EVICTSERVER, "%s", "added worker thread"); } conn->evict_tune_last_action_time = current_time; } @@ -1641,26 +1651,16 @@ __evict_walk_file(WT_SESSION_IMPL *session, QUEUE_FILLS_PER_PASS; /* - * Randomly walk trees with a small fraction of the cache in case there - * are so many trees that none of them use enough of the cache to be - * allocated slots. - * - * The chance of walking a tree is equal to the chance that a random - * byte in cache belongs to the tree, weighted by how many times we - * want to fill queues during a pass through all the trees in cache. + * Walk trees with a small fraction of the cache in case there are so + * many trees that none of them use enough of the cache to be allocated + * slots. Only skip a tree if it has no bytes of interest. */ if (target_pages == 0) { - if (F_ISSET(cache, WT_CACHE_EVICT_CLEAN)) { - btree_inuse = __wt_btree_bytes_evictable(session); - cache_inuse = __wt_cache_bytes_inuse(cache); - } else { - btree_inuse = __wt_btree_dirty_leaf_inuse(session); - cache_inuse = __wt_cache_dirty_leaf_inuse(cache); - } - if (btree_inuse == 0 || cache_inuse == 0) - return (0); - if (__wt_random64(&session->rnd) % cache_inuse > - btree_inuse * QUEUE_FILLS_PER_PASS) + btree_inuse = F_ISSET(cache, WT_CACHE_EVICT_CLEAN) ? + __wt_btree_bytes_evictable(session) : + __wt_btree_dirty_leaf_inuse(session); + + if (btree_inuse == 0) return (0); } @@ -1678,6 +1678,32 @@ __evict_walk_file(WT_SESSION_IMPL *session, if (F_ISSET(session->dhandle, WT_DHANDLE_DEAD) || target_pages > remaining_slots) target_pages = remaining_slots; + + /* + * These statistics generate a histogram of the number of pages targeted + * for eviction each round. The range of values here start at + * MIN_PAGES_PER_TREE as this is the smallest number of pages we can + * target, unless there are fewer slots available. The aim is to cover + * the likely ranges of target pages in as few statistics as possible to + * reduce the overall overhead. + */ + if (target_pages < MIN_PAGES_PER_TREE) { + WT_STAT_CONN_INCR(session, cache_eviction_target_page_lt10); + WT_STAT_DATA_INCR(session, cache_eviction_target_page_lt10); + } else if (target_pages < 32) { + WT_STAT_CONN_INCR(session, cache_eviction_target_page_lt32); + WT_STAT_DATA_INCR(session, cache_eviction_target_page_lt32); + } else if (target_pages < 64) { + WT_STAT_CONN_INCR(session, cache_eviction_target_page_lt64); + WT_STAT_DATA_INCR(session, cache_eviction_target_page_lt64); + } else if (target_pages < 128) { + WT_STAT_CONN_INCR(session, cache_eviction_target_page_lt128); + WT_STAT_DATA_INCR(session, cache_eviction_target_page_lt128); + } else { + WT_STAT_CONN_INCR(session, cache_eviction_target_page_ge128); + WT_STAT_DATA_INCR(session, cache_eviction_target_page_ge128); + } + end = start + target_pages; /* @@ -1690,6 +1716,14 @@ __evict_walk_file(WT_SESSION_IMPL *session, !F_ISSET(cache, WT_CACHE_EVICT_CLEAN)) min_pages *= 10; + if (btree->evict_ref == NULL) { + WT_STAT_CONN_INCR(session, cache_eviction_walk_from_root); + WT_STAT_DATA_INCR(session, cache_eviction_walk_from_root); + } else { + WT_STAT_CONN_INCR(session, cache_eviction_walk_saved_pos); + WT_STAT_DATA_INCR(session, cache_eviction_walk_saved_pos); + } + walk_flags = WT_READ_CACHE | WT_READ_NO_EVICT | WT_READ_NO_GEN | WT_READ_NO_WAIT; @@ -1760,12 +1794,37 @@ __evict_walk_file(WT_SESSION_IMPL *session, btree->evict_start_type = (btree->evict_start_type + 1) % WT_EVICT_WALK_START_NUM; + /* + * We differentiate the reasons we gave up on this walk + * and increment the stats accordingly. + */ + if (pages_queued == 0) { + WT_STAT_CONN_INCR(session, + cache_eviction_walks_gave_up_no_targets); + WT_STAT_DATA_INCR(session, + cache_eviction_walks_gave_up_no_targets); + } else { + WT_STAT_CONN_INCR(session, + cache_eviction_walks_gave_up_ratio); + WT_STAT_DATA_INCR(session, + cache_eviction_walks_gave_up_ratio); + } break; } if (ref == NULL) { - if (++restarts == 2) + WT_STAT_CONN_INCR( + session, cache_eviction_walks_ended); + WT_STAT_DATA_INCR( + session, cache_eviction_walks_ended); + + if (++restarts == 2) { + WT_STAT_CONN_INCR( + session, cache_eviction_walks_stopped); + WT_STAT_DATA_INCR( + session, cache_eviction_walks_stopped); break; + } WT_STAT_CONN_INCR( session, cache_eviction_walks_started); continue; @@ -1919,6 +1978,9 @@ fast: /* If the page can't be evicted, give up. */ WT_STAT_CONN_INCRV(session, cache_eviction_walk, refs_walked); WT_STAT_CONN_INCRV(session, cache_eviction_pages_seen, pages_seen); + WT_STAT_DATA_INCRV(session, cache_eviction_pages_seen, pages_seen); + WT_STAT_CONN_INCRV(session, cache_eviction_walk_passes, 1); + WT_STAT_DATA_INCRV(session, cache_eviction_walk_passes, 1); return (0); } diff --git a/src/third_party/wiredtiger/src/evict/evict_stat.c b/src/third_party/wiredtiger/src/evict/evict_stat.c index 276e737ebbb..63d0aff60f1 100644 --- a/src/third_party/wiredtiger/src/evict/evict_stat.c +++ b/src/third_party/wiredtiger/src/evict/evict_stat.c @@ -16,28 +16,30 @@ static void __evict_stat_walk(WT_SESSION_IMPL *session) { WT_BTREE *btree; + WT_CACHE *cache; WT_PAGE *page; WT_REF *next_walk; - uint64_t dsk_size, gen_gap, size; - uint64_t written_size_cnt, written_size_sum; - uint64_t gen_gap_cnt, gen_gap_max, gen_gap_sum; - uint64_t max_pagesize, min_written_size; - uint64_t num_memory, num_queued, num_not_queueable, num_smaller_allocsz; - uint64_t pages_clean, pages_dirty, pages_internal, pages_leaf; - uint64_t seen_count, walk_count; + uint64_t dsk_size, gen_gap, gen_gap_max, gen_gap_sum, max_pagesize; + uint64_t min_written_size, num_memory, num_not_queueable, num_queued; + uint64_t num_smaller_allocsz, pages_clean, pages_dirty, pages_internal; + uint64_t pages_leaf, seen_count, size, visited_count; + uint64_t visited_age_gap_sum, unvisited_count, unvisited_age_gap_sum; + uint64_t walk_count, written_size_cnt, written_size_sum; btree = S2BT(session); + cache = S2C(session)->cache; next_walk = NULL; - written_size_cnt = written_size_sum = 0; - gen_gap_cnt = gen_gap_max = gen_gap_sum = 0; - max_pagesize = 0; - num_memory = num_queued = num_not_queueable = num_smaller_allocsz = 0; - pages_clean = pages_dirty = pages_internal = pages_leaf = 0; - seen_count = walk_count = 0; + dsk_size = gen_gap = gen_gap_max = gen_gap_sum = max_pagesize = 0; + num_memory = num_not_queueable = num_queued = 0; + num_smaller_allocsz = pages_clean = pages_dirty = pages_internal = 0; + pages_leaf = seen_count = size = visited_count = 0; + visited_age_gap_sum = unvisited_count = unvisited_age_gap_sum = 0; + walk_count = written_size_cnt = written_size_sum = 0; min_written_size = UINT64_MAX; while (__wt_tree_walk_count(session, &next_walk, &walk_count, - WT_READ_CACHE | WT_READ_NO_EVICT | WT_READ_NO_WAIT) == 0 && + WT_READ_CACHE | WT_READ_NO_EVICT | + WT_READ_NO_GEN | WT_READ_NO_WAIT) == 0 && next_walk != NULL) { ++seen_count; page = next_walk->page; @@ -78,19 +80,29 @@ __evict_stat_walk(WT_SESSION_IMPL *session) if (__wt_ref_is_root(next_walk)) continue; - gen_gap = - S2C(session)->cache->evict_pass_gen - page->evict_pass_gen; - if (gen_gap > gen_gap_max) - gen_gap_max = gen_gap; - gen_gap_sum += gen_gap; - ++gen_gap_cnt; + if (page->evict_pass_gen == 0) { + unvisited_age_gap_sum += + (cache->evict_pass_gen - page->cache_create_gen); + ++unvisited_count; + } else { + visited_age_gap_sum += + (cache->evict_pass_gen - page->cache_create_gen); + gen_gap = cache->evict_pass_gen - page->evict_pass_gen; + if (gen_gap > gen_gap_max) + gen_gap_max = gen_gap; + gen_gap_sum += gen_gap; + ++visited_count; + } } + WT_STAT_DATA_SET(session, cache_state_gen_avg_gap, + visited_count == 0 ? 0 : gen_gap_sum / visited_count); + WT_STAT_DATA_SET(session, cache_state_avg_unvisited_age, + unvisited_count == 0 ? 0 : unvisited_age_gap_sum / unvisited_count); + WT_STAT_DATA_SET(session, cache_state_avg_visited_age, + visited_count == 0 ? 0 : visited_age_gap_sum / visited_count); WT_STAT_DATA_SET(session, cache_state_avg_written_size, written_size_cnt == 0 ? 0 : written_size_sum / written_size_cnt); - WT_STAT_DATA_SET(session, cache_state_gen_avg_gap, - gen_gap_cnt == 0 ? 0 : gen_gap_sum / gen_gap_cnt); - WT_STAT_DATA_SET(session, cache_state_gen_max_gap, gen_gap_max); WT_STAT_DATA_SET(session, cache_state_max_pagesize, max_pagesize); WT_STAT_DATA_SET(session, @@ -98,8 +110,6 @@ __evict_stat_walk(WT_SESSION_IMPL *session) WT_STAT_DATA_SET(session, cache_state_memory, num_memory); WT_STAT_DATA_SET(session, cache_state_queued, num_queued); WT_STAT_DATA_SET(session, cache_state_not_queueable, num_not_queueable); - WT_STAT_DATA_SET(session, - cache_state_smaller_alloc_size, num_smaller_allocsz); WT_STAT_DATA_SET(session, cache_state_pages, walk_count); WT_STAT_DATA_SET(session, cache_state_pages_clean, pages_clean); WT_STAT_DATA_SET(session, cache_state_pages_dirty, pages_dirty); @@ -107,6 +117,10 @@ __evict_stat_walk(WT_SESSION_IMPL *session) WT_STAT_DATA_SET(session, cache_state_pages_leaf, pages_leaf); WT_STAT_DATA_SET(session, cache_state_refs_skipped, walk_count - seen_count); + WT_STAT_DATA_SET(session, + cache_state_smaller_alloc_size, num_smaller_allocsz); + WT_STAT_DATA_SET(session, + cache_state_unvisited_count, unvisited_count); } /* diff --git a/src/third_party/wiredtiger/src/include/api.h b/src/third_party/wiredtiger/src/include/api.h index 372ba063cd3..60ed31b64e8 100644 --- a/src/third_party/wiredtiger/src/include/api.h +++ b/src/third_party/wiredtiger/src/include/api.h @@ -13,7 +13,7 @@ (s)->dhandle = (dh); \ (s)->name = (s)->lastop = #h "." #n; \ WT_ERR(WT_SESSION_CHECK_PANIC(s)); \ - __wt_verbose((s), WT_VERB_API, "CALL: " #h ":" #n) + __wt_verbose((s), WT_VERB_API, "%s", "CALL: " #h ":" #n) #define API_CALL_NOCONF(s, h, n, dh) do { \ API_SESSION_INIT(s, h, n, dh) diff --git a/src/third_party/wiredtiger/src/include/bitstring.i b/src/third_party/wiredtiger/src/include/bitstring.i index d3dc3bebd0f..a9ec91d49ff 100644 --- a/src/third_party/wiredtiger/src/include/bitstring.i +++ b/src/third_party/wiredtiger/src/include/bitstring.i @@ -171,16 +171,18 @@ __bit_ffc(uint8_t *bitf, uint64_t nbits, uint64_t *retp) if (nbits == 0) return (-1); - for (byte = 0, - stopbyte = __bit_byte(nbits - 1); byte <= stopbyte; ++byte) + for (byte = 0, stopbyte = __bit_byte(nbits - 1);; ++byte) { if (bitf[byte] != 0xff) { value = byte << 3; for (lb = bitf[byte]; lb & 0x01; ++value, lb >>= 1) ; break; } + if (byte == stopbyte) + return (-1); + } - if (byte > stopbyte || value >= nbits) + if (value >= nbits) return (-1); *retp = value; @@ -201,16 +203,18 @@ __bit_ffs(uint8_t *bitf, uint64_t nbits, uint64_t *retp) if (nbits == 0) return (-1); - for (byte = 0, - stopbyte = __bit_byte(nbits - 1); byte <= stopbyte; ++byte) + for (byte = 0, stopbyte = __bit_byte(nbits - 1);; ++byte) { if (bitf[byte] != 0) { value = byte << 3; for (lb = bitf[byte]; !(lb & 0x01); ++value, lb >>= 1) ; break; } + if (byte == stopbyte) + return (-1); + } - if (byte > stopbyte || value >= nbits) + if (value >= nbits) return (-1); *retp = value; @@ -227,13 +231,6 @@ __bit_getv(uint8_t *bitf, uint64_t entry, uint8_t width) uint8_t value; uint64_t bit; -#define __BIT_GET(len, mask) \ - case len: \ - if (__bit_test(bitf, bit)) \ - value |= (mask); \ - ++bit \ - /* FALLTHROUGH */ - value = 0; bit = entry * width; @@ -241,17 +238,50 @@ __bit_getv(uint8_t *bitf, uint64_t entry, uint8_t width) * Fast-path single bytes, do repeated tests for the rest: we could * slice-and-dice instead, but the compiler is probably going to do * a better job than I will. + * + * The Berkeley version of this file uses a #define to compress this + * case statement. This code expands the case statement because gcc7 + * complains about implicit fallthrough and doesn't support explicit + * fallthrough comments in macros. */ switch (width) { case 8: return (bitf[__bit_byte(bit)]); - __BIT_GET(7, 0x40); - __BIT_GET(6, 0x20); - __BIT_GET(5, 0x10); - __BIT_GET(4, 0x08); - __BIT_GET(3, 0x04); - __BIT_GET(2, 0x02); - __BIT_GET(1, 0x01); + case 7: + if (__bit_test(bitf, bit)) + value |= 0x40; + ++bit; + /* FALLTHROUGH */ + case 6: + if (__bit_test(bitf, bit)) + value |= 0x20; + ++bit; + /* FALLTHROUGH */ + case 5: + if (__bit_test(bitf, bit)) + value |= 0x10; + ++bit; + /* FALLTHROUGH */ + case 4: + if (__bit_test(bitf, bit)) + value |= 0x08; + ++bit; + /* FALLTHROUGH */ + case 3: + if (__bit_test(bitf, bit)) + value |= 0x04; + ++bit; + /* FALLTHROUGH */ + case 2: + if (__bit_test(bitf, bit)) + value |= 0x02; + ++bit; + /* FALLTHROUGH */ + case 1: + if (__bit_test(bitf, bit)) + value |= 0x01; + ++bit; + break; } return (value); } @@ -276,32 +306,70 @@ __bit_setv(uint8_t *bitf, uint64_t entry, uint8_t width, uint8_t value) { uint64_t bit; -#define __BIT_SET(len, mask) \ - case len: \ - if (value & (mask)) \ - __bit_set(bitf, bit); \ - else \ - __bit_clear(bitf, bit); \ - ++bit \ - /* FALLTHROUGH */ - bit = entry * width; /* * Fast-path single bytes, do repeated tests for the rest: we could * slice-and-dice instead, but the compiler is probably going to do * a better job than I will. + * + * The Berkeley version of this file uses a #define to compress this + * case statement. This code expands the case statement because gcc7 + * complains about implicit fallthrough and doesn't support explicit + * fallthrough comments in macros. */ switch (width) { case 8: bitf[__bit_byte(bit)] = value; return; - __BIT_SET(7, 0x40); - __BIT_SET(6, 0x20); - __BIT_SET(5, 0x10); - __BIT_SET(4, 0x08); - __BIT_SET(3, 0x04); - __BIT_SET(2, 0x02); - __BIT_SET(1, 0x01); + case 7: + if (value & 0x40) + __bit_set(bitf, bit); + else + __bit_clear(bitf, bit); + ++bit; + /* FALLTHROUGH */ + case 6: + if (value & 0x20) + __bit_set(bitf, bit); + else + __bit_clear(bitf, bit); + ++bit; + /* FALLTHROUGH */ + case 5: + if (value & 0x10) + __bit_set(bitf, bit); + else + __bit_clear(bitf, bit); + ++bit; + /* FALLTHROUGH */ + case 4: + if (value & 0x08) + __bit_set(bitf, bit); + else + __bit_clear(bitf, bit); + ++bit; + /* FALLTHROUGH */ + case 3: + if (value & 0x04) + __bit_set(bitf, bit); + else + __bit_clear(bitf, bit); + ++bit; + /* FALLTHROUGH */ + case 2: + if (value & 0x02) + __bit_set(bitf, bit); + else + __bit_clear(bitf, bit); + ++bit; + /* FALLTHROUGH */ + case 1: + if (value & 0x01) + __bit_set(bitf, bit); + else + __bit_clear(bitf, bit); + ++bit; + break; } } diff --git a/src/third_party/wiredtiger/src/include/btmem.h b/src/third_party/wiredtiger/src/include/btmem.h index 54a03e5762c..e8d3307b013 100644 --- a/src/third_party/wiredtiger/src/include/btmem.h +++ b/src/third_party/wiredtiger/src/include/btmem.h @@ -162,32 +162,6 @@ struct __wt_ovfl_reuse { }; /* - * Overflow tracking for cached values: When a page is reconciled, we write new - * K/V overflow items, and discard previous underlying blocks. If there's a - * transaction in the system that needs to read the previous value, we have to - * cache the old value until no running transaction needs it. - */ -struct __wt_ovfl_txnc { - uint64_t current; /* Maximum transaction ID at store */ - - uint32_t value_offset; /* Overflow value offset */ - uint32_t value_size; /* Overflow value size */ - uint8_t addr_offset; /* Overflow addr offset */ - uint8_t addr_size; /* Overflow addr size */ - - /* - * The untyped address immediately follows the WT_OVFL_TXNC - * structure, the untyped value immediately follows the address. - */ -#define WT_OVFL_TXNC_ADDR(p) \ - ((void *)((uint8_t *)(p) + (p)->addr_offset)) -#define WT_OVFL_TXNC_VALUE(p) \ - ((void *)((uint8_t *)(p) + (p)->value_offset)) - - WT_OVFL_TXNC *next[0]; /* Forward-linked skip list */ -}; - -/* * Lookaside table support: when a page is being reconciled for eviction and has * updates that might be required by earlier readers in the system, the updates * are written into a lookaside table, and restored as necessary if the page is @@ -403,18 +377,20 @@ struct __wt_page_modify { WT_OVFL_REUSE *ovfl_reuse[WT_SKIP_MAXDEPTH]; /* - * Overflow value address/byte-string pairs cached until no - * running transaction will possibly read them. - */ - WT_OVFL_TXNC *ovfl_txnc[WT_SKIP_MAXDEPTH]; - - /* * Overflow key/value addresses to be discarded from the block * manager after reconciliation completes successfully. */ WT_CELL **discard; size_t discard_entries; size_t discard_allocated; + + /* Cached overflow value cell/update address pairs. */ + struct { + WT_CELL *cell; + WT_UPDATE *upd; + } *remove; + size_t remove_allocated; + uint32_t remove_next; } *ovfl_track; #define WT_PAGE_LOCK(s, p) \ @@ -644,6 +620,7 @@ struct __wt_page { /* This is the 64 byte boundary, try to keep hot fields above here. */ + uint64_t cache_create_gen; /* Page create timestamp */ uint64_t evict_pass_gen; /* Eviction pass generation */ }; @@ -921,6 +898,10 @@ WT_PACKED_STRUCT_BEGIN(__wt_update) #define WT_UPDATE_RESERVED 2 uint8_t type; /* type (one byte to conserve memory) */ + /* The update includes a complete value. */ +#define WT_UPDATE_DATA_VALUE(upd) \ + ((upd)->type == WT_UPDATE_STANDARD || (upd)->type == WT_UPDATE_DELETED) + /* The untyped value immediately follows the WT_UPDATE structure. */ #define WT_UPDATE_DATA(upd) \ ((void *)((uint8_t *)(upd) + sizeof(WT_UPDATE))) diff --git a/src/third_party/wiredtiger/src/include/btree.h b/src/third_party/wiredtiger/src/include/btree.h index e0f7570b869..4d2a4f0c4d9 100644 --- a/src/third_party/wiredtiger/src/include/btree.h +++ b/src/third_party/wiredtiger/src/include/btree.h @@ -165,24 +165,25 @@ struct __wt_btree { WT_SPINLOCK flush_lock; /* Lock to flush the tree's pages */ /* Flags values up to 0xff are reserved for WT_DHANDLE_* */ -#define WT_BTREE_BULK 0x000100 /* Bulk-load handle */ -#define WT_BTREE_CLOSED 0x000200 /* Handle closed */ -#define WT_BTREE_IGNORE_CACHE 0x000400 /* Cache-resident object */ -#define WT_BTREE_IN_MEMORY 0x000800 /* Cache-resident object */ -#define WT_BTREE_LOOKASIDE 0x001000 /* Look-aside table */ -#define WT_BTREE_NO_CHECKPOINT 0x002000 /* Disable checkpoints */ -#define WT_BTREE_NO_LOGGING 0x004000 /* Disable logging */ -#define WT_BTREE_REBALANCE 0x008000 /* Handle is for rebalance */ -#define WT_BTREE_SALVAGE 0x010000 /* Handle is for salvage */ -#define WT_BTREE_SKIP_CKPT 0x020000 /* Handle skipped checkpoint */ -#define WT_BTREE_UPGRADE 0x040000 /* Handle is for upgrade */ -#define WT_BTREE_VERIFY 0x080000 /* Handle is for verify */ +#define WT_BTREE_ALTER 0x000100 /* Handle is for alter */ +#define WT_BTREE_BULK 0x000200 /* Bulk-load handle */ +#define WT_BTREE_CLOSED 0x000400 /* Handle closed */ +#define WT_BTREE_IGNORE_CACHE 0x000800 /* Cache-resident object */ +#define WT_BTREE_IN_MEMORY 0x001000 /* Cache-resident object */ +#define WT_BTREE_LOOKASIDE 0x002000 /* Look-aside table */ +#define WT_BTREE_NO_CHECKPOINT 0x004000 /* Disable checkpoints */ +#define WT_BTREE_NO_LOGGING 0x008000 /* Disable logging */ +#define WT_BTREE_REBALANCE 0x010000 /* Handle is for rebalance */ +#define WT_BTREE_SALVAGE 0x020000 /* Handle is for salvage */ +#define WT_BTREE_SKIP_CKPT 0x040000 /* Handle skipped checkpoint */ +#define WT_BTREE_UPGRADE 0x080000 /* Handle is for upgrade */ +#define WT_BTREE_VERIFY 0x100000 /* Handle is for verify */ uint32_t flags; }; /* Flags that make a btree handle special (not for normal use). */ #define WT_BTREE_SPECIAL_FLAGS \ - (WT_BTREE_BULK | WT_BTREE_REBALANCE | \ + (WT_BTREE_ALTER | WT_BTREE_BULK | WT_BTREE_REBALANCE | \ WT_BTREE_SALVAGE | WT_BTREE_UPGRADE | WT_BTREE_VERIFY) /* diff --git a/src/third_party/wiredtiger/src/include/btree_cmp.i b/src/third_party/wiredtiger/src/include/btree_cmp.i index c1354a7ea4b..db7af8daaed 100644 --- a/src/third_party/wiredtiger/src/include/btree_cmp.i +++ b/src/third_party/wiredtiger/src/include/btree_cmp.i @@ -217,23 +217,61 @@ __wt_lex_compare_short(const WT_ITEM *user_item, const WT_ITEM *tree_item) /* * The maximum packed uint64_t is 9B, catch row-store objects using * packed record numbers as keys. + * + * Don't use a #define to compress this case statement: gcc7 complains + * about implicit fallthrough and doesn't support explicit fallthrough + * comments in macros. */ #define WT_COMPARE_SHORT_MAXLEN 9 -#undef WT_COMPARE_SHORT -#define WT_COMPARE_SHORT(n) \ - case n: \ - if (*userp != *treep) \ - break; \ - ++userp, ++treep switch (len) { - WT_COMPARE_SHORT(9); - WT_COMPARE_SHORT(8); - WT_COMPARE_SHORT(7); - WT_COMPARE_SHORT(6); - WT_COMPARE_SHORT(5); - WT_COMPARE_SHORT(4); - WT_COMPARE_SHORT(3); - WT_COMPARE_SHORT(2); + case 9: + if (*userp != *treep) + break; + ++userp; + ++treep; + /* FALLTHROUGH */ + case 8: + if (*userp != *treep) + break; + ++userp; + ++treep; + /* FALLTHROUGH */ + case 7: + if (*userp != *treep) + break; + ++userp; + ++treep; + /* FALLTHROUGH */ + case 6: + if (*userp != *treep) + break; + ++userp; + ++treep; + /* FALLTHROUGH */ + case 5: + if (*userp != *treep) + break; + ++userp; + ++treep; + /* FALLTHROUGH */ + case 4: + if (*userp != *treep) + break; + ++userp; + ++treep; + /* FALLTHROUGH */ + case 3: + if (*userp != *treep) + break; + ++userp; + ++treep; + /* FALLTHROUGH */ + case 2: + if (*userp != *treep) + break; + ++userp; + ++treep; + /* FALLTHROUGH */ case 1: if (*userp != *treep) break; diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h index 94abe48ffe5..7bd4a818495 100644 --- a/src/third_party/wiredtiger/src/include/connection.h +++ b/src/third_party/wiredtiger/src/include/connection.h @@ -184,6 +184,9 @@ struct __wt_connection_impl { const char *error_prefix; /* Database error prefix */ int is_new; /* Connection created database */ + uint16_t compat_major; /* Compatibility major version */ + uint16_t compat_minor; /* Compatibility minor version */ + WT_EXTENSION_API extension_api; /* Extension API */ /* Configuration */ @@ -302,18 +305,22 @@ struct __wt_connection_impl { WT_CONDVAR *stat_cond; /* Statistics log wait mutex */ const char *stat_format; /* Statistics log timestamp format */ WT_FSTREAM *stat_fs; /* Statistics log stream */ + /* Statistics log json table printing state flag */ + bool stat_json_tables; char *stat_path; /* Statistics log path format */ char **stat_sources; /* Statistics log list of objects */ const char *stat_stamp; /* Statistics log entry timestamp */ uint64_t stat_usecs; /* Statistics log period */ -#define WT_CONN_LOG_ARCHIVE 0x01 /* Archive is enabled */ -#define WT_CONN_LOG_ENABLED 0x02 /* Logging is enabled */ -#define WT_CONN_LOG_EXISTED 0x04 /* Log files found */ -#define WT_CONN_LOG_RECOVER_DIRTY 0x08 /* Recovering unclean */ -#define WT_CONN_LOG_RECOVER_DONE 0x10 /* Recovery completed */ -#define WT_CONN_LOG_RECOVER_ERR 0x20 /* Error if recovery required */ -#define WT_CONN_LOG_ZERO_FILL 0x40 /* Manually zero files */ +#define WT_CONN_LOG_ARCHIVE 0x001 /* Archive is enabled */ +#define WT_CONN_LOG_DOWNGRADED 0x002 /* Running older version */ +#define WT_CONN_LOG_ENABLED 0x004 /* Logging is enabled */ +#define WT_CONN_LOG_EXISTED 0x008 /* Log files found */ +#define WT_CONN_LOG_FORCE_DOWNGRADE 0x010 /* Force downgrade */ +#define WT_CONN_LOG_RECOVER_DIRTY 0x020 /* Recovering unclean */ +#define WT_CONN_LOG_RECOVER_DONE 0x040 /* Recovery completed */ +#define WT_CONN_LOG_RECOVER_ERR 0x080 /* Error if recovery required */ +#define WT_CONN_LOG_ZERO_FILL 0x100 /* Manually zero files */ uint32_t log_flags; /* Global logging configuration */ WT_CONDVAR *log_cond; /* Log server wait mutex */ WT_SESSION_IMPL *log_session; /* Log server session */ diff --git a/src/third_party/wiredtiger/src/include/error.h b/src/third_party/wiredtiger/src/include/error.h index 465ab4fa859..2d302fb6879 100644 --- a/src/third_party/wiredtiger/src/include/error.h +++ b/src/third_party/wiredtiger/src/include/error.h @@ -124,3 +124,28 @@ #define WT_ASSERT(session, exp) \ WT_UNUSED(session) #endif + +/* + * __wt_verbose -- + * Display a verbose message. + * + * Not an inlined function because you can't inline functions taking variadic + * arguments and we don't want to make a function call in production systems + * just to find out a verbose flag isn't set. + * + * The macro must take a format string and at least one additional argument, + * there's no portable way to remove the comma before an empty __VA_ARGS__ + * value. + */ +#ifdef HAVE_VERBOSE +#define __wt_verbose(session, flag, fmt, ...) do { \ + if (WT_VERBOSE_ISSET(session, flag)) \ + __wt_verbose_worker(session, fmt, __VA_ARGS__); \ +} while (0) +#else +#define __wt_verbose(session, flag, fmt, ...) do { \ + WT_UNUSED(session); \ + WT_UNUSED(flag); \ + WT_UNUSED(fmt); \ +} while (0) +#endif diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index 59e853d75f4..8b48fd587bd 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -148,7 +148,7 @@ extern const char *__wt_cell_type_string(uint8_t type); extern const char *__wt_page_addr_string(WT_SESSION_IMPL *session, WT_REF *ref, WT_ITEM *buf); extern const char *__wt_addr_string(WT_SESSION_IMPL *session, const uint8_t *addr, size_t addr_size, WT_ITEM *buf); extern int __wt_ovfl_read(WT_SESSION_IMPL *session, WT_PAGE *page, WT_CELL_UNPACK *unpack, WT_ITEM *store) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_ovfl_cache(WT_SESSION_IMPL *session, WT_PAGE *page, void *cookie, WT_CELL_UNPACK *vpack) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_ovfl_remove(WT_SESSION_IMPL *session, WT_PAGE *page, WT_UPDATE *upd_list, WT_CELL_UNPACK *unpack) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_ovfl_discard(WT_SESSION_IMPL *session, WT_CELL *cell) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_page_alloc(WT_SESSION_IMPL *session, uint8_t type, uint32_t alloc_entries, bool alloc_refs, WT_PAGE **pagep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_page_inmem(WT_SESSION_IMPL *session, WT_REF *ref, const void *image, size_t memsize, uint32_t flags, WT_PAGE **pagep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -267,7 +267,7 @@ extern int __wt_conn_dhandle_discard(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ extern int __wt_connection_init(WT_CONNECTION_IMPL *conn) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_connection_destroy(WT_CONNECTION_IMPL *conn); extern int __wt_logmgr_reconfig(WT_SESSION_IMPL *session, const char **cfg) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_log_truncate_files( WT_SESSION_IMPL *session, WT_CURSOR *cursor, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_log_truncate_files(WT_SESSION_IMPL *session, WT_CURSOR *cursor, bool force) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_log_wrlsn(WT_SESSION_IMPL *session, int *yield); extern int __wt_logmgr_create(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_logmgr_open(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -371,6 +371,8 @@ extern void __wt_log_written_reset(WT_SESSION_IMPL *session); extern int __wt_log_get_all_files(WT_SESSION_IMPL *session, char ***filesp, u_int *countp, uint32_t *maxid, bool active_only) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_log_extract_lognum( WT_SESSION_IMPL *session, const char *name, uint32_t *id) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_log_reset(WT_SESSION_IMPL *session, uint32_t lognum) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_log_fill(WT_SESSION_IMPL *session, WT_MYSLOT *myslot, bool force, WT_ITEM *record, WT_LSN *lsnp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_log_set_version(WT_SESSION_IMPL *session, uint16_t version, uint32_t first_rec, bool downgrade, bool live_chg, uint32_t *lognump) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_log_acquire(WT_SESSION_IMPL *session, uint64_t recsize, WT_LOGSLOT *slot) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_log_allocfile( WT_SESSION_IMPL *session, uint32_t lognum, const char *dest) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_log_remove(WT_SESSION_IMPL *session, const char *file_prefix, uint32_t lognum) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -412,6 +414,8 @@ extern int __wt_log_slot_destroy(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTR extern void __wt_log_slot_join(WT_SESSION_IMPL *session, uint64_t mysize, uint32_t flags, WT_MYSLOT *myslot); extern int64_t __wt_log_slot_release(WT_MYSLOT *myslot, int64_t size); extern void __wt_log_slot_free(WT_SESSION_IMPL *session, WT_LOGSLOT *slot); +extern int __wt_log_system_record( WT_SESSION_IMPL *session, WT_FH *log_fh, WT_LSN *lsn) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_log_recover_system(WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end, WT_LSN *lsnp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_clsm_request_switch(WT_CURSOR_LSM *clsm) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_clsm_await_switch(WT_CURSOR_LSM *clsm) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_clsm_init_merge( WT_CURSOR *cursor, u_int start_chunk, uint32_t start_id, u_int nchunks) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -443,7 +447,6 @@ extern void __wt_lsm_tree_release(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tre extern void __wt_lsm_tree_throttle( WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, bool decrease_only); extern int __wt_lsm_tree_switch(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_lsm_tree_retire_chunks(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, u_int start_chunk, u_int nchunks) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_lsm_tree_alter( WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_lsm_tree_drop( WT_SESSION_IMPL *session, const char *name, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_lsm_tree_rename(WT_SESSION_IMPL *session, const char *olduri, const char *newuri, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_lsm_tree_truncate( WT_SESSION_IMPL *session, const char *name, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -541,14 +544,12 @@ extern int __wt_ext_unpack_item(WT_EXTENSION_API *wt_api, WT_PACK_STREAM *ps, WT extern int __wt_ext_unpack_int(WT_EXTENSION_API *wt_api, WT_PACK_STREAM *ps, int64_t *ip) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_ext_unpack_str(WT_EXTENSION_API *wt_api, WT_PACK_STREAM *ps, const char **sp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_ext_unpack_uint(WT_EXTENSION_API *wt_api, WT_PACK_STREAM *ps, uint64_t *up) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_ovfl_track_init(WT_SESSION_IMPL *session, WT_PAGE *page) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_ovfl_discard_add(WT_SESSION_IMPL *session, WT_PAGE *page, WT_CELL *cell) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_ovfl_discard_free(WT_SESSION_IMPL *session, WT_PAGE *page); extern int __wt_ovfl_reuse_search(WT_SESSION_IMPL *session, WT_PAGE *page, uint8_t **addrp, size_t *addr_sizep, const void *value, size_t value_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_ovfl_reuse_add(WT_SESSION_IMPL *session, WT_PAGE *page, const uint8_t *addr, size_t addr_size, const void *value, size_t value_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_ovfl_reuse_free(WT_SESSION_IMPL *session, WT_PAGE *page); -extern int __wt_ovfl_txnc_search( WT_PAGE *page, const uint8_t *addr, size_t addr_size, WT_ITEM *store) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_ovfl_txnc_add(WT_SESSION_IMPL *session, WT_PAGE *page, const uint8_t *addr, size_t addr_size, const void *value, size_t value_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern void __wt_ovfl_txnc_free(WT_SESSION_IMPL *session, WT_PAGE *page); extern int __wt_ovfl_track_wrapup(WT_SESSION_IMPL *session, WT_PAGE *page) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_ovfl_track_wrapup_err(WT_SESSION_IMPL *session, WT_PAGE *page) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_reconcile(WT_SESSION_IMPL *session, WT_REF *ref, WT_SALVAGE_COOKIE *salvage, uint32_t flags, bool *lookaside_retryp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -559,7 +560,7 @@ extern int __wt_bulk_insert_row(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk) extern int __wt_bulk_insert_fix( WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk, bool deleted) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_bulk_insert_fix_bitmap(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_bulk_insert_var( WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk, bool deleted) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_schema_alter(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_alter(WT_SESSION_IMPL *session, const char *newcfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_direct_io_size_check(WT_SESSION_IMPL *session, const char **cfg, const char *config_name, uint32_t *allocsizep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_schema_colgroup_source(WT_SESSION_IMPL *session, WT_TABLE *table, const char *cgname, const char *config, WT_ITEM *buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_schema_index_source(WT_SESSION_IMPL *session, WT_TABLE *table, const char *idxname, const char *config, WT_ITEM *buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -631,6 +632,7 @@ extern int __wt_eventv(WT_SESSION_IMPL *session, bool msg_event, int error, cons extern void __wt_err(WT_SESSION_IMPL *session, int error, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4))) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern void __wt_errx(WT_SESSION_IMPL *session, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 2, 3))); extern int __wt_ext_err_printf( WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_verbose_worker(WT_SESSION_IMPL *session, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 2, 3))) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)); extern int __wt_msg(WT_SESSION_IMPL *session, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 2, 3))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_ext_msg_printf( WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern const char *__wt_ext_strerror(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, int error); @@ -646,6 +648,7 @@ __wt_assert(WT_SESSION_IMPL *session, WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern int __wt_panic(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_illegal_value(WT_SESSION_IMPL *session, const char *name) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_inmem_unsupported_op(WT_SESSION_IMPL *session, const char *tag) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_object_unsupported(WT_SESSION_IMPL *session, const char *uri) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_bad_object_type(WT_SESSION_IMPL *session, const char *uri) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_unexpected_object_type( WT_SESSION_IMPL *session, const char *uri, const char *expect) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -705,7 +708,6 @@ extern uint32_t __wt_rduppo2(uint32_t n, uint32_t po2); extern void __wt_random_init(WT_RAND_STATE volatile *rnd_state) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern void __wt_random_init_seed( WT_SESSION_IMPL *session, WT_RAND_STATE volatile *rnd_state) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern uint32_t __wt_random(WT_RAND_STATE volatile *rnd_state) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); -extern uint64_t __wt_random64(WT_RAND_STATE volatile *rnd_state) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern int __wt_buf_grow_worker(WT_SESSION_IMPL *session, WT_ITEM *buf, size_t size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_buf_fmt(WT_SESSION_IMPL *session, WT_ITEM *buf, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_buf_catfmt(WT_SESSION_IMPL *session, WT_ITEM *buf, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -789,3 +791,7 @@ extern int __wt_txn_global_query_timestamp( WT_SESSION_IMPL *session, char *hex_ extern int __wt_txn_update_pinned_timestamp(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_txn_global_set_timestamp(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_txn_set_timestamp(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_txn_set_commit_timestamp(WT_SESSION_IMPL *session); +extern void __wt_txn_clear_commit_timestamp(WT_SESSION_IMPL *session); +extern void __wt_txn_set_read_timestamp(WT_SESSION_IMPL *session); +extern void __wt_txn_clear_read_timestamp(WT_SESSION_IMPL *session); diff --git a/src/third_party/wiredtiger/src/include/gcc.h b/src/third_party/wiredtiger/src/include/gcc.h index 21eaaaef049..043d9b44356 100644 --- a/src/third_party/wiredtiger/src/include/gcc.h +++ b/src/third_party/wiredtiger/src/include/gcc.h @@ -21,7 +21,7 @@ * dist/s_prototypes to create extern.h. */ #define WT_GCC_FUNC_ATTRIBUTE(x) -#define WT_GCC_FUNC_DECL_ATTRIBUTE(x) __attribute__(x) +#define WT_GCC_FUNC_DECL_ATTRIBUTE(x) __attribute__(x) /* * Atomic writes: diff --git a/src/third_party/wiredtiger/src/include/log.h b/src/third_party/wiredtiger/src/include/log.h index e7bc28cd220..e6acdd1e834 100644 --- a/src/third_party/wiredtiger/src/include/log.h +++ b/src/third_party/wiredtiger/src/include/log.h @@ -42,7 +42,7 @@ union __wt_lsn { #define WT_ZERO_LSN(l) WT_SET_LSN((l), 0, 0) /* - * Initialize LSN is (1,0). We only need to shift the 1 for comparison. + * Test for initial LSN. We only need to shift the 1 for comparison. */ #define WT_IS_INIT_LSN(l) ((l)->file_offset == ((uint64_t)1 << 32)) /* @@ -52,6 +52,10 @@ union __wt_lsn { #define WT_IS_MAX_LSN(lsn) \ ((lsn)->l.file == UINT32_MAX && \ ((lsn)->l.offset == INT32_MAX || (lsn)->l.offset == UINT32_MAX)) +/* + * Test for zero LSN. + */ +#define WT_IS_ZERO_LSN(l) ((l)->file_offset == 0) /* * Both of the macros below need to change if the content of __wt_lsn @@ -198,10 +202,11 @@ struct __wt_myslot { uint32_t flags; /* Flags */ }; -#define WT_LOG_FIRST_RECORD log->allocsize +#define WT_LOG_END_HEADER log->allocsize struct __wt_log { uint32_t allocsize; /* Allocation alignment size */ + uint32_t first_record; /* Offset of first record in file */ wt_off_t log_written; /* Amount of log written this period */ /* * Log file information @@ -215,6 +220,8 @@ struct __wt_log { WT_FH *log_close_fh; /* Logging file handle to close */ WT_LSN log_close_lsn; /* LSN needed to close */ + uint16_t log_version; /* Version of log file */ + /* * System LSNs */ @@ -232,6 +239,7 @@ struct __wt_log { * Synchronization resources */ WT_SPINLOCK log_lock; /* Locked: Logging fields */ + WT_SPINLOCK log_fs_lock; /* Locked: tmp, prep and log files */ WT_SPINLOCK log_slot_lock; /* Locked: Consolidation array */ WT_SPINLOCK log_sync_lock; /* Locked: Single-thread fsync */ WT_SPINLOCK log_writelsn_lock; /* Locked: write LSN */ @@ -261,8 +269,9 @@ struct __wt_log { uint64_t write_calls; /* Calls to log_write */ #endif -#define WT_LOG_OPENED 0x01 /* Log subsystem successfully open */ -#define WT_LOG_TRUNCATE_NOTSUP 0x02 /* File system truncate not supported */ +#define WT_LOG_FORCE_NEWFILE 0x01 /* Force switch to new log file */ +#define WT_LOG_OPENED 0x02 /* Log subsystem successfully open */ +#define WT_LOG_TRUNCATE_NOTSUP 0x04 /* File system truncate not supported */ uint32_t flags; }; @@ -303,12 +312,22 @@ __wt_log_record_byteswap(WT_LOG_RECORD *record) struct __wt_log_desc { #define WT_LOG_MAGIC 0x101064 uint32_t log_magic; /* 00-03: Magic number */ -#define WT_LOG_MAJOR_VERSION 1 - uint16_t majorv; /* 04-05: Major version */ -#define WT_LOG_MINOR_VERSION 0 - uint16_t minorv; /* 06-07: Minor version */ +#define WT_LOG_VERSION 2 + uint16_t version; /* 04-05: Log version */ + uint16_t unused; /* 06-07: Unused */ uint64_t log_size; /* 08-15: Log file size */ }; +/* + * This is the log version that introduced the system record. + */ +#define WT_LOG_VERSION_SYSTEM 2 + +/* + * WiredTiger release version where log format version changed. + * We only have to check the major version for now. It is minor + * version 0 once release numbers move on. + */ +#define WT_LOG_V2 3 /* * __wt_log_desc_byteswap -- @@ -320,8 +339,8 @@ __wt_log_desc_byteswap(WT_LOG_DESC *desc) { #ifdef WORDS_BIGENDIAN desc->log_magic = __wt_bswap32(desc->log_magic); - desc->majorv = __wt_bswap16(desc->majorv); - desc->minorv = __wt_bswap16(desc->minorv); + desc->version = __wt_bswap16(desc->version); + desc->unused = __wt_bswap16(desc->unused); desc->log_size = __wt_bswap64(desc->log_size); #else WT_UNUSED(desc); diff --git a/src/third_party/wiredtiger/src/include/misc.i b/src/third_party/wiredtiger/src/include/misc.i index 36a1e1f18eb..bb785a63072 100644 --- a/src/third_party/wiredtiger/src/include/misc.i +++ b/src/third_party/wiredtiger/src/include/misc.i @@ -80,39 +80,6 @@ __wt_time_check_monotonic(WT_SESSION_IMPL *session, struct timespec *tsp) } /* - * __wt_verbose -- - * Verbose message. - * - * Inline functions are not parsed for external prototypes, so in cases where we - * want GCC attributes attached to the functions, we have to do so explicitly. - */ -static inline void -__wt_verbose(WT_SESSION_IMPL *session, int flag, const char *fmt, ...) -WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4))); - -/* - * __wt_verbose -- - * Verbose message. - */ -static inline void -__wt_verbose(WT_SESSION_IMPL *session, int flag, const char *fmt, ...) -{ -#ifdef HAVE_VERBOSE - va_list ap; - - if (WT_VERBOSE_ISSET(session, flag)) { - va_start(ap, fmt); - WT_IGNORE_RET(__wt_eventv(session, true, 0, NULL, 0, fmt, ap)); - va_end(ap); - } -#else - WT_UNUSED(session); - WT_UNUSED(flag); - WT_UNUSED(fmt); -#endif -} - -/* * __wt_snprintf -- * snprintf convenience function, ignoring the returned size. */ diff --git a/src/third_party/wiredtiger/src/include/stat.h b/src/third_party/wiredtiger/src/include/stat.h index 7d7d701590a..b340b278684 100644 --- a/src/third_party/wiredtiger/src/include/stat.h +++ b/src/third_party/wiredtiger/src/include/stat.h @@ -303,13 +303,25 @@ struct __wt_connection_stats { int64_t cache_eviction_get_ref_empty2; int64_t cache_eviction_aggressive_set; int64_t cache_eviction_empty_score; + int64_t cache_eviction_walk_passes; int64_t cache_eviction_queue_empty; int64_t cache_eviction_queue_not_empty; int64_t cache_eviction_server_evicting; int64_t cache_eviction_server_slept; int64_t cache_eviction_slow; int64_t cache_eviction_state; + int64_t cache_eviction_target_page_lt10; + int64_t cache_eviction_target_page_lt32; + int64_t cache_eviction_target_page_ge128; + int64_t cache_eviction_target_page_lt64; + int64_t cache_eviction_target_page_lt128; int64_t cache_eviction_walks_abandoned; + int64_t cache_eviction_walks_stopped; + int64_t cache_eviction_walks_gave_up_no_targets; + int64_t cache_eviction_walks_gave_up_ratio; + int64_t cache_eviction_walks_ended; + int64_t cache_eviction_walk_from_root; + int64_t cache_eviction_walk_saved_pos; int64_t cache_eviction_active_workers; int64_t cache_eviction_worker_created; int64_t cache_eviction_worker_evicting; @@ -336,7 +348,6 @@ struct __wt_connection_stats { int64_t cache_eviction_dirty; int64_t cache_eviction_app_dirty; int64_t cache_read_overflow; - int64_t cache_overflow_value; int64_t cache_eviction_deepen; int64_t cache_write_lookaside; int64_t cache_pages_inuse; @@ -414,6 +425,7 @@ struct __wt_connection_stats { int64_t lock_table_read_count; int64_t lock_table_write_count; int64_t log_slot_switch_busy; + int64_t log_force_ckpt_sleep; int64_t log_bytes_payload; int64_t log_bytes_written; int64_t log_zero_fills; @@ -570,6 +582,19 @@ struct __wt_dsrc_stats { int64_t cache_bytes_write; int64_t cache_eviction_checkpoint; int64_t cache_eviction_fail; + int64_t cache_eviction_walk_passes; + int64_t cache_eviction_target_page_lt10; + int64_t cache_eviction_target_page_lt32; + int64_t cache_eviction_target_page_ge128; + int64_t cache_eviction_target_page_lt64; + int64_t cache_eviction_target_page_lt128; + int64_t cache_eviction_walks_abandoned; + int64_t cache_eviction_walks_stopped; + int64_t cache_eviction_walks_gave_up_no_targets; + int64_t cache_eviction_walks_gave_up_ratio; + int64_t cache_eviction_walks_ended; + int64_t cache_eviction_walk_from_root; + int64_t cache_eviction_walk_saved_pos; int64_t cache_eviction_hazard; int64_t cache_inmem_splittable; int64_t cache_inmem_split; @@ -578,18 +603,20 @@ struct __wt_dsrc_stats { int64_t cache_eviction_split_leaf; int64_t cache_eviction_dirty; int64_t cache_read_overflow; - int64_t cache_overflow_value; int64_t cache_eviction_deepen; int64_t cache_write_lookaside; int64_t cache_read; int64_t cache_read_lookaside; int64_t cache_pages_requested; + int64_t cache_eviction_pages_seen; int64_t cache_write; int64_t cache_write_restore; int64_t cache_bytes_dirty; int64_t cache_eviction_clean; int64_t cache_state_gen_avg_gap; int64_t cache_state_avg_written_size; + int64_t cache_state_avg_visited_age; + int64_t cache_state_avg_unvisited_age; int64_t cache_state_pages_clean; int64_t cache_state_gen_current; int64_t cache_state_pages_dirty; @@ -599,6 +626,7 @@ struct __wt_dsrc_stats { int64_t cache_state_gen_max_gap; int64_t cache_state_max_pagesize; int64_t cache_state_min_written_size; + int64_t cache_state_unvisited_count; int64_t cache_state_smaller_alloc_size; int64_t cache_state_memory; int64_t cache_state_queued; diff --git a/src/third_party/wiredtiger/src/include/txn.h b/src/third_party/wiredtiger/src/include/txn.h index bf2d9aa21ef..e4cc0b04046 100644 --- a/src/third_party/wiredtiger/src/include/txn.h +++ b/src/third_party/wiredtiger/src/include/txn.h @@ -74,9 +74,6 @@ struct __wt_txn_state { volatile uint64_t pinned_id; volatile uint64_t metadata_pinned; - WT_DECL_TIMESTAMP(commit_timestamp) - WT_DECL_TIMESTAMP(read_timestamp) - WT_CACHE_LINE_PAD_END }; @@ -103,6 +100,14 @@ struct __wt_txn_global { /* Protects the active transaction states. */ WT_RWLOCK rwlock; + /* List of transactions sorted by commit timestamp. */ + WT_RWLOCK commit_timestamp_rwlock; + TAILQ_HEAD(__wt_txn_cts_qh, __wt_txn) commit_timestamph; + + /* List of transactions sorted by read timestamp. */ + WT_RWLOCK read_timestamp_rwlock; + TAILQ_HEAD(__wt_txn_rts_qh, __wt_txn) read_timestamph; + /* * Track information about the running checkpoint. The transaction * snapshot used when checkpointing are special. Checkpoints can run @@ -117,6 +122,7 @@ struct __wt_txn_global { volatile bool checkpoint_running; /* Checkpoint running */ volatile uint32_t checkpoint_id; /* Checkpoint's session ID */ WT_TXN_STATE checkpoint_state; /* Checkpoint's txn state */ + WT_TXN *checkpoint_txn; /* Checkpoint's txn structure */ volatile uint64_t metadata_pinned; /* Oldest ID for metadata */ @@ -194,8 +200,11 @@ struct __wt_txn { uint32_t snapshot_count; uint32_t txn_logsync; /* Log sync configuration */ - WT_DECL_TIMESTAMP(read_timestamp) WT_DECL_TIMESTAMP(commit_timestamp) + WT_DECL_TIMESTAMP(read_timestamp) + + TAILQ_ENTRY(__wt_txn) commit_timestampq; + TAILQ_ENTRY(__wt_txn) read_timestampq; /* Array of modifications by this transaction. */ WT_TXN_OP *mod; diff --git a/src/third_party/wiredtiger/src/include/txn.i b/src/third_party/wiredtiger/src/include/txn.i index 85d70fbde89..6de86eb0aaf 100644 --- a/src/third_party/wiredtiger/src/include/txn.i +++ b/src/third_party/wiredtiger/src/include/txn.i @@ -10,8 +10,6 @@ static inline int __wt_txn_id_check(WT_SESSION_IMPL *session); static inline void __wt_txn_read_last(WT_SESSION_IMPL *session); #ifdef HAVE_TIMESTAMPS -static const wt_timestamp_t zero_timestamp; - /* * __wt_timestamp_cmp -- * Compare two timestamps. @@ -39,8 +37,30 @@ __wt_timestamp_set(uint8_t *dest, const uint8_t *src) static inline bool __wt_timestamp_iszero(const uint8_t *ts) { + static const wt_timestamp_t zero_timestamp; + return (memcmp(ts, zero_timestamp, WT_TIMESTAMP_SIZE) == 0); } + +/* + * __wt_timestamp_set_inf -- + * Set a timestamp to the maximum value. + */ +static inline void +__wt_timestamp_set_inf(uint8_t *ts) +{ + memset(ts, 0xff, WT_TIMESTAMP_SIZE); +} + +/* + * __wt_timestamp_set_zero -- + * Zero out a timestamp. + */ +static inline void +__wt_timestamp_set_zero(uint8_t *ts) +{ + memset(ts, 0x00, WT_TIMESTAMP_SIZE); +} #endif /* diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index 60bacbcf51d..d8d8b864766 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -921,6 +921,12 @@ struct __wt_session { * cache. Not compatible with LSM tables; see @ref * tuning_cache_resident for more information., a boolean flag; default * \c false.} + * @config{log = (, the transaction log configuration for this object. + * Only valid if log is enabled in ::wiredtiger_open., a set of related + * configuration options defined below.} + * @config{ enabled, if false\, this object has + * checkpoint-level durability., a boolean flag; default \c true.} + * @config{ ),,} * @configend * @errors */ @@ -1971,6 +1977,11 @@ struct __wt_connection { * checkpoint; setting this value above 0 configures periodic * checkpoints., an integer between 0 and 100000; default \c 0.} * @config{ ),,} + * @config{compatibility = (, set compatibility version of database., a + * set of related configuration options defined below.} + * @config{ release, compatibility release + * version string., a string; default empty.} + * @config{ ),,} * @config{error_prefix, prefix string for error messages., a string; * default empty.} * @config{eviction = (, eviction configuration options., a set of @@ -2454,6 +2465,11 @@ struct __wt_connection { * @config{ ),,} * @config{checkpoint_sync, flush files to stable storage when closing or * writing checkpoints., a boolean flag; default \c true.} + * @config{compatibility = (, set compatibility version of database., a set of + * related configuration options defined below.} + * @config{ release, compatibility release version + * string., a string; default empty.} + * @config{ ),,} * @config{config_base, write the base configuration file if creating the * database. If \c false in the config passed directly to ::wiredtiger_open\, * will ignore any existing base configuration file in addition to not creating @@ -4471,6 +4487,8 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_LOGREC_FILE_SYNC 2 /*! message */ #define WT_LOGREC_MESSAGE 3 +/*! system/internal record */ +#define WT_LOGREC_SYSTEM 4 /*! invalid operation */ #define WT_LOGOP_INVALID 0 /*! column put */ @@ -4596,475 +4614,505 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_CONN_CACHE_EVICTION_AGGRESSIVE_SET 1044 /*! cache: eviction empty score */ #define WT_STAT_CONN_CACHE_EVICTION_EMPTY_SCORE 1045 +/*! cache: eviction passes of a file */ +#define WT_STAT_CONN_CACHE_EVICTION_WALK_PASSES 1046 /*! cache: eviction server candidate queue empty when topping up */ -#define WT_STAT_CONN_CACHE_EVICTION_QUEUE_EMPTY 1046 +#define WT_STAT_CONN_CACHE_EVICTION_QUEUE_EMPTY 1047 /*! cache: eviction server candidate queue not empty when topping up */ -#define WT_STAT_CONN_CACHE_EVICTION_QUEUE_NOT_EMPTY 1047 +#define WT_STAT_CONN_CACHE_EVICTION_QUEUE_NOT_EMPTY 1048 /*! cache: eviction server evicting pages */ -#define WT_STAT_CONN_CACHE_EVICTION_SERVER_EVICTING 1048 +#define WT_STAT_CONN_CACHE_EVICTION_SERVER_EVICTING 1049 /*! * cache: eviction server slept, because we did not make progress with * eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_SERVER_SLEPT 1049 +#define WT_STAT_CONN_CACHE_EVICTION_SERVER_SLEPT 1050 /*! cache: eviction server unable to reach eviction goal */ -#define WT_STAT_CONN_CACHE_EVICTION_SLOW 1050 +#define WT_STAT_CONN_CACHE_EVICTION_SLOW 1051 /*! cache: eviction state */ -#define WT_STAT_CONN_CACHE_EVICTION_STATE 1051 +#define WT_STAT_CONN_CACHE_EVICTION_STATE 1052 +/*! cache: eviction walk target pages histogram - 0-9 */ +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT10 1053 +/*! cache: eviction walk target pages histogram - 10-31 */ +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT32 1054 +/*! cache: eviction walk target pages histogram - 128 and higher */ +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_GE128 1055 +/*! cache: eviction walk target pages histogram - 32-63 */ +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT64 1056 +/*! cache: eviction walk target pages histogram - 64-128 */ +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT128 1057 /*! cache: eviction walks abandoned */ -#define WT_STAT_CONN_CACHE_EVICTION_WALKS_ABANDONED 1052 +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_ABANDONED 1058 +/*! cache: eviction walks gave up because they restarted their walk twice */ +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_STOPPED 1059 +/*! + * cache: eviction walks gave up because they saw too many pages and + * found no candidates + */ +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_GAVE_UP_NO_TARGETS 1060 +/*! + * cache: eviction walks gave up because they saw too many pages and + * found too few candidates + */ +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_GAVE_UP_RATIO 1061 +/*! cache: eviction walks reached end of tree */ +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_ENDED 1062 +/*! cache: eviction walks started from root of tree */ +#define WT_STAT_CONN_CACHE_EVICTION_WALK_FROM_ROOT 1063 +/*! cache: eviction walks started from saved location in tree */ +#define WT_STAT_CONN_CACHE_EVICTION_WALK_SAVED_POS 1064 /*! cache: eviction worker thread active */ -#define WT_STAT_CONN_CACHE_EVICTION_ACTIVE_WORKERS 1053 +#define WT_STAT_CONN_CACHE_EVICTION_ACTIVE_WORKERS 1065 /*! cache: eviction worker thread created */ -#define WT_STAT_CONN_CACHE_EVICTION_WORKER_CREATED 1054 +#define WT_STAT_CONN_CACHE_EVICTION_WORKER_CREATED 1066 /*! cache: eviction worker thread evicting pages */ -#define WT_STAT_CONN_CACHE_EVICTION_WORKER_EVICTING 1055 +#define WT_STAT_CONN_CACHE_EVICTION_WORKER_EVICTING 1067 /*! cache: eviction worker thread removed */ -#define WT_STAT_CONN_CACHE_EVICTION_WORKER_REMOVED 1056 +#define WT_STAT_CONN_CACHE_EVICTION_WORKER_REMOVED 1068 /*! cache: eviction worker thread stable number */ -#define WT_STAT_CONN_CACHE_EVICTION_STABLE_STATE_WORKERS 1057 +#define WT_STAT_CONN_CACHE_EVICTION_STABLE_STATE_WORKERS 1069 /*! * cache: failed eviction of pages that exceeded the in-memory maximum * count */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_FAIL 1058 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_FAIL 1070 /*! * cache: failed eviction of pages that exceeded the in-memory maximum * time (usecs) */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_FAIL_TIME 1059 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_FAIL_TIME 1071 /*! cache: files with active eviction walks */ -#define WT_STAT_CONN_CACHE_EVICTION_WALKS_ACTIVE 1060 +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_ACTIVE 1072 /*! cache: files with new eviction walks started */ -#define WT_STAT_CONN_CACHE_EVICTION_WALKS_STARTED 1061 +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_STARTED 1073 /*! cache: force re-tuning of eviction workers once in a while */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_RETUNE 1062 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_RETUNE 1074 /*! cache: hazard pointer blocked page eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_HAZARD 1063 +#define WT_STAT_CONN_CACHE_EVICTION_HAZARD 1075 /*! cache: hazard pointer check calls */ -#define WT_STAT_CONN_CACHE_HAZARD_CHECKS 1064 +#define WT_STAT_CONN_CACHE_HAZARD_CHECKS 1076 /*! cache: hazard pointer check entries walked */ -#define WT_STAT_CONN_CACHE_HAZARD_WALKS 1065 +#define WT_STAT_CONN_CACHE_HAZARD_WALKS 1077 /*! cache: hazard pointer maximum array length */ -#define WT_STAT_CONN_CACHE_HAZARD_MAX 1066 +#define WT_STAT_CONN_CACHE_HAZARD_MAX 1078 /*! cache: in-memory page passed criteria to be split */ -#define WT_STAT_CONN_CACHE_INMEM_SPLITTABLE 1067 +#define WT_STAT_CONN_CACHE_INMEM_SPLITTABLE 1079 /*! cache: in-memory page splits */ -#define WT_STAT_CONN_CACHE_INMEM_SPLIT 1068 +#define WT_STAT_CONN_CACHE_INMEM_SPLIT 1080 /*! cache: internal pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL 1069 +#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL 1081 /*! cache: internal pages split during eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_INTERNAL 1070 +#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_INTERNAL 1082 /*! cache: leaf pages split during eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_LEAF 1071 +#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_LEAF 1083 /*! cache: lookaside table insert calls */ -#define WT_STAT_CONN_CACHE_LOOKASIDE_INSERT 1072 +#define WT_STAT_CONN_CACHE_LOOKASIDE_INSERT 1084 /*! cache: lookaside table remove calls */ -#define WT_STAT_CONN_CACHE_LOOKASIDE_REMOVE 1073 +#define WT_STAT_CONN_CACHE_LOOKASIDE_REMOVE 1085 /*! cache: maximum bytes configured */ -#define WT_STAT_CONN_CACHE_BYTES_MAX 1074 +#define WT_STAT_CONN_CACHE_BYTES_MAX 1086 /*! cache: maximum page size at eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_MAXIMUM_PAGE_SIZE 1075 +#define WT_STAT_CONN_CACHE_EVICTION_MAXIMUM_PAGE_SIZE 1087 /*! cache: modified pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_DIRTY 1076 +#define WT_STAT_CONN_CACHE_EVICTION_DIRTY 1088 /*! cache: modified pages evicted by application threads */ -#define WT_STAT_CONN_CACHE_EVICTION_APP_DIRTY 1077 +#define WT_STAT_CONN_CACHE_EVICTION_APP_DIRTY 1089 /*! cache: overflow pages read into cache */ -#define WT_STAT_CONN_CACHE_READ_OVERFLOW 1078 -/*! cache: overflow values cached in memory */ -#define WT_STAT_CONN_CACHE_OVERFLOW_VALUE 1079 +#define WT_STAT_CONN_CACHE_READ_OVERFLOW 1090 /*! cache: page split during eviction deepened the tree */ -#define WT_STAT_CONN_CACHE_EVICTION_DEEPEN 1080 +#define WT_STAT_CONN_CACHE_EVICTION_DEEPEN 1091 /*! cache: page written requiring lookaside records */ -#define WT_STAT_CONN_CACHE_WRITE_LOOKASIDE 1081 +#define WT_STAT_CONN_CACHE_WRITE_LOOKASIDE 1092 /*! cache: pages currently held in the cache */ -#define WT_STAT_CONN_CACHE_PAGES_INUSE 1082 +#define WT_STAT_CONN_CACHE_PAGES_INUSE 1093 /*! cache: pages evicted because they exceeded the in-memory maximum count */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE 1083 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE 1094 /*! * cache: pages evicted because they exceeded the in-memory maximum time * (usecs) */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_TIME 1084 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_TIME 1095 /*! cache: pages evicted because they had chains of deleted items count */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_DELETE 1085 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_DELETE 1096 /*! * cache: pages evicted because they had chains of deleted items time * (usecs) */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_DELETE_TIME 1086 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_DELETE_TIME 1097 /*! cache: pages evicted by application threads */ -#define WT_STAT_CONN_CACHE_EVICTION_APP 1087 +#define WT_STAT_CONN_CACHE_EVICTION_APP 1098 /*! cache: pages queued for eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED 1088 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED 1099 /*! cache: pages queued for urgent eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_URGENT 1089 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_URGENT 1100 /*! cache: pages queued for urgent eviction during walk */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_OLDEST 1090 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_OLDEST 1101 /*! cache: pages read into cache */ -#define WT_STAT_CONN_CACHE_READ 1091 +#define WT_STAT_CONN_CACHE_READ 1102 /*! cache: pages read into cache requiring lookaside entries */ -#define WT_STAT_CONN_CACHE_READ_LOOKASIDE 1092 +#define WT_STAT_CONN_CACHE_READ_LOOKASIDE 1103 /*! cache: pages requested from the cache */ -#define WT_STAT_CONN_CACHE_PAGES_REQUESTED 1093 +#define WT_STAT_CONN_CACHE_PAGES_REQUESTED 1104 /*! cache: pages seen by eviction walk */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_SEEN 1094 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_SEEN 1105 /*! cache: pages selected for eviction unable to be evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_FAIL 1095 +#define WT_STAT_CONN_CACHE_EVICTION_FAIL 1106 /*! cache: pages walked for eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_WALK 1096 +#define WT_STAT_CONN_CACHE_EVICTION_WALK 1107 /*! cache: pages written from cache */ -#define WT_STAT_CONN_CACHE_WRITE 1097 +#define WT_STAT_CONN_CACHE_WRITE 1108 /*! cache: pages written requiring in-memory restoration */ -#define WT_STAT_CONN_CACHE_WRITE_RESTORE 1098 +#define WT_STAT_CONN_CACHE_WRITE_RESTORE 1109 /*! cache: percentage overhead */ -#define WT_STAT_CONN_CACHE_OVERHEAD 1099 +#define WT_STAT_CONN_CACHE_OVERHEAD 1110 /*! cache: tracked bytes belonging to internal pages in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_INTERNAL 1100 +#define WT_STAT_CONN_CACHE_BYTES_INTERNAL 1111 /*! cache: tracked bytes belonging to leaf pages in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_LEAF 1101 +#define WT_STAT_CONN_CACHE_BYTES_LEAF 1112 /*! cache: tracked dirty bytes in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_DIRTY 1102 +#define WT_STAT_CONN_CACHE_BYTES_DIRTY 1113 /*! cache: tracked dirty pages in the cache */ -#define WT_STAT_CONN_CACHE_PAGES_DIRTY 1103 +#define WT_STAT_CONN_CACHE_PAGES_DIRTY 1114 /*! cache: unmodified pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_CLEAN 1104 +#define WT_STAT_CONN_CACHE_EVICTION_CLEAN 1115 /*! connection: auto adjusting condition resets */ -#define WT_STAT_CONN_COND_AUTO_WAIT_RESET 1105 +#define WT_STAT_CONN_COND_AUTO_WAIT_RESET 1116 /*! connection: auto adjusting condition wait calls */ -#define WT_STAT_CONN_COND_AUTO_WAIT 1106 +#define WT_STAT_CONN_COND_AUTO_WAIT 1117 /*! connection: detected system time went backwards */ -#define WT_STAT_CONN_TIME_TRAVEL 1107 +#define WT_STAT_CONN_TIME_TRAVEL 1118 /*! connection: files currently open */ -#define WT_STAT_CONN_FILE_OPEN 1108 +#define WT_STAT_CONN_FILE_OPEN 1119 /*! connection: memory allocations */ -#define WT_STAT_CONN_MEMORY_ALLOCATION 1109 +#define WT_STAT_CONN_MEMORY_ALLOCATION 1120 /*! connection: memory frees */ -#define WT_STAT_CONN_MEMORY_FREE 1110 +#define WT_STAT_CONN_MEMORY_FREE 1121 /*! connection: memory re-allocations */ -#define WT_STAT_CONN_MEMORY_GROW 1111 +#define WT_STAT_CONN_MEMORY_GROW 1122 /*! connection: pthread mutex condition wait calls */ -#define WT_STAT_CONN_COND_WAIT 1112 +#define WT_STAT_CONN_COND_WAIT 1123 /*! connection: pthread mutex shared lock read-lock calls */ -#define WT_STAT_CONN_RWLOCK_READ 1113 +#define WT_STAT_CONN_RWLOCK_READ 1124 /*! connection: pthread mutex shared lock write-lock calls */ -#define WT_STAT_CONN_RWLOCK_WRITE 1114 +#define WT_STAT_CONN_RWLOCK_WRITE 1125 /*! connection: total fsync I/Os */ -#define WT_STAT_CONN_FSYNC_IO 1115 +#define WT_STAT_CONN_FSYNC_IO 1126 /*! connection: total read I/Os */ -#define WT_STAT_CONN_READ_IO 1116 +#define WT_STAT_CONN_READ_IO 1127 /*! connection: total write I/Os */ -#define WT_STAT_CONN_WRITE_IO 1117 +#define WT_STAT_CONN_WRITE_IO 1128 /*! cursor: cursor create calls */ -#define WT_STAT_CONN_CURSOR_CREATE 1118 +#define WT_STAT_CONN_CURSOR_CREATE 1129 /*! cursor: cursor insert calls */ -#define WT_STAT_CONN_CURSOR_INSERT 1119 +#define WT_STAT_CONN_CURSOR_INSERT 1130 /*! cursor: cursor modify calls */ -#define WT_STAT_CONN_CURSOR_MODIFY 1120 +#define WT_STAT_CONN_CURSOR_MODIFY 1131 /*! cursor: cursor next calls */ -#define WT_STAT_CONN_CURSOR_NEXT 1121 +#define WT_STAT_CONN_CURSOR_NEXT 1132 /*! cursor: cursor prev calls */ -#define WT_STAT_CONN_CURSOR_PREV 1122 +#define WT_STAT_CONN_CURSOR_PREV 1133 /*! cursor: cursor remove calls */ -#define WT_STAT_CONN_CURSOR_REMOVE 1123 +#define WT_STAT_CONN_CURSOR_REMOVE 1134 /*! cursor: cursor reserve calls */ -#define WT_STAT_CONN_CURSOR_RESERVE 1124 +#define WT_STAT_CONN_CURSOR_RESERVE 1135 /*! cursor: cursor reset calls */ -#define WT_STAT_CONN_CURSOR_RESET 1125 +#define WT_STAT_CONN_CURSOR_RESET 1136 /*! cursor: cursor restarted searches */ -#define WT_STAT_CONN_CURSOR_RESTART 1126 +#define WT_STAT_CONN_CURSOR_RESTART 1137 /*! cursor: cursor search calls */ -#define WT_STAT_CONN_CURSOR_SEARCH 1127 +#define WT_STAT_CONN_CURSOR_SEARCH 1138 /*! cursor: cursor search near calls */ -#define WT_STAT_CONN_CURSOR_SEARCH_NEAR 1128 +#define WT_STAT_CONN_CURSOR_SEARCH_NEAR 1139 /*! cursor: cursor update calls */ -#define WT_STAT_CONN_CURSOR_UPDATE 1129 +#define WT_STAT_CONN_CURSOR_UPDATE 1140 /*! cursor: truncate calls */ -#define WT_STAT_CONN_CURSOR_TRUNCATE 1130 +#define WT_STAT_CONN_CURSOR_TRUNCATE 1141 /*! data-handle: connection data handles currently active */ -#define WT_STAT_CONN_DH_CONN_HANDLE_COUNT 1131 +#define WT_STAT_CONN_DH_CONN_HANDLE_COUNT 1142 /*! data-handle: connection sweep candidate became referenced */ -#define WT_STAT_CONN_DH_SWEEP_REF 1132 +#define WT_STAT_CONN_DH_SWEEP_REF 1143 /*! data-handle: connection sweep dhandles closed */ -#define WT_STAT_CONN_DH_SWEEP_CLOSE 1133 +#define WT_STAT_CONN_DH_SWEEP_CLOSE 1144 /*! data-handle: connection sweep dhandles removed from hash list */ -#define WT_STAT_CONN_DH_SWEEP_REMOVE 1134 +#define WT_STAT_CONN_DH_SWEEP_REMOVE 1145 /*! data-handle: connection sweep time-of-death sets */ -#define WT_STAT_CONN_DH_SWEEP_TOD 1135 +#define WT_STAT_CONN_DH_SWEEP_TOD 1146 /*! data-handle: connection sweeps */ -#define WT_STAT_CONN_DH_SWEEPS 1136 +#define WT_STAT_CONN_DH_SWEEPS 1147 /*! data-handle: session dhandles swept */ -#define WT_STAT_CONN_DH_SESSION_HANDLES 1137 +#define WT_STAT_CONN_DH_SESSION_HANDLES 1148 /*! data-handle: session sweep attempts */ -#define WT_STAT_CONN_DH_SESSION_SWEEPS 1138 +#define WT_STAT_CONN_DH_SESSION_SWEEPS 1149 /*! lock: checkpoint lock acquisitions */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_COUNT 1139 +#define WT_STAT_CONN_LOCK_CHECKPOINT_COUNT 1150 /*! lock: checkpoint lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_APPLICATION 1140 +#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_APPLICATION 1151 /*! lock: checkpoint lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_INTERNAL 1141 +#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_INTERNAL 1152 /*! * lock: dhandle lock application thread time waiting for the dhandle * lock (usecs) */ -#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_APPLICATION 1142 +#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_APPLICATION 1153 /*! * lock: dhandle lock internal thread time waiting for the dhandle lock * (usecs) */ -#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_INTERNAL 1143 +#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_INTERNAL 1154 /*! lock: dhandle read lock acquisitions */ -#define WT_STAT_CONN_LOCK_DHANDLE_READ_COUNT 1144 +#define WT_STAT_CONN_LOCK_DHANDLE_READ_COUNT 1155 /*! lock: dhandle write lock acquisitions */ -#define WT_STAT_CONN_LOCK_DHANDLE_WRITE_COUNT 1145 +#define WT_STAT_CONN_LOCK_DHANDLE_WRITE_COUNT 1156 /*! lock: metadata lock acquisitions */ -#define WT_STAT_CONN_LOCK_METADATA_COUNT 1146 +#define WT_STAT_CONN_LOCK_METADATA_COUNT 1157 /*! lock: metadata lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_METADATA_WAIT_APPLICATION 1147 +#define WT_STAT_CONN_LOCK_METADATA_WAIT_APPLICATION 1158 /*! lock: metadata lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_METADATA_WAIT_INTERNAL 1148 +#define WT_STAT_CONN_LOCK_METADATA_WAIT_INTERNAL 1159 /*! lock: schema lock acquisitions */ -#define WT_STAT_CONN_LOCK_SCHEMA_COUNT 1149 +#define WT_STAT_CONN_LOCK_SCHEMA_COUNT 1160 /*! lock: schema lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_APPLICATION 1150 +#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_APPLICATION 1161 /*! lock: schema lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_INTERNAL 1151 +#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_INTERNAL 1162 /*! * lock: table lock application thread time waiting for the table lock * (usecs) */ -#define WT_STAT_CONN_LOCK_TABLE_WAIT_APPLICATION 1152 +#define WT_STAT_CONN_LOCK_TABLE_WAIT_APPLICATION 1163 /*! * lock: table lock internal thread time waiting for the table lock * (usecs) */ -#define WT_STAT_CONN_LOCK_TABLE_WAIT_INTERNAL 1153 +#define WT_STAT_CONN_LOCK_TABLE_WAIT_INTERNAL 1164 /*! lock: table read lock acquisitions */ -#define WT_STAT_CONN_LOCK_TABLE_READ_COUNT 1154 +#define WT_STAT_CONN_LOCK_TABLE_READ_COUNT 1165 /*! lock: table write lock acquisitions */ -#define WT_STAT_CONN_LOCK_TABLE_WRITE_COUNT 1155 +#define WT_STAT_CONN_LOCK_TABLE_WRITE_COUNT 1166 /*! log: busy returns attempting to switch slots */ -#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1156 +#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1167 +/*! log: force checkpoint calls slept */ +#define WT_STAT_CONN_LOG_FORCE_CKPT_SLEEP 1168 /*! log: log bytes of payload data */ -#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1157 +#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1169 /*! log: log bytes written */ -#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1158 +#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1170 /*! log: log files manually zero-filled */ -#define WT_STAT_CONN_LOG_ZERO_FILLS 1159 +#define WT_STAT_CONN_LOG_ZERO_FILLS 1171 /*! log: log flush operations */ -#define WT_STAT_CONN_LOG_FLUSH 1160 +#define WT_STAT_CONN_LOG_FLUSH 1172 /*! log: log force write operations */ -#define WT_STAT_CONN_LOG_FORCE_WRITE 1161 +#define WT_STAT_CONN_LOG_FORCE_WRITE 1173 /*! log: log force write operations skipped */ -#define WT_STAT_CONN_LOG_FORCE_WRITE_SKIP 1162 +#define WT_STAT_CONN_LOG_FORCE_WRITE_SKIP 1174 /*! log: log records compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1163 +#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1175 /*! log: log records not compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1164 +#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1176 /*! log: log records too small to compress */ -#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1165 +#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1177 /*! log: log release advances write LSN */ -#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1166 +#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1178 /*! log: log scan operations */ -#define WT_STAT_CONN_LOG_SCANS 1167 +#define WT_STAT_CONN_LOG_SCANS 1179 /*! log: log scan records requiring two reads */ -#define WT_STAT_CONN_LOG_SCAN_REREADS 1168 +#define WT_STAT_CONN_LOG_SCAN_REREADS 1180 /*! log: log server thread advances write LSN */ -#define WT_STAT_CONN_LOG_WRITE_LSN 1169 +#define WT_STAT_CONN_LOG_WRITE_LSN 1181 /*! log: log server thread write LSN walk skipped */ -#define WT_STAT_CONN_LOG_WRITE_LSN_SKIP 1170 +#define WT_STAT_CONN_LOG_WRITE_LSN_SKIP 1182 /*! log: log sync operations */ -#define WT_STAT_CONN_LOG_SYNC 1171 +#define WT_STAT_CONN_LOG_SYNC 1183 /*! log: log sync time duration (usecs) */ -#define WT_STAT_CONN_LOG_SYNC_DURATION 1172 +#define WT_STAT_CONN_LOG_SYNC_DURATION 1184 /*! log: log sync_dir operations */ -#define WT_STAT_CONN_LOG_SYNC_DIR 1173 +#define WT_STAT_CONN_LOG_SYNC_DIR 1185 /*! log: log sync_dir time duration (usecs) */ -#define WT_STAT_CONN_LOG_SYNC_DIR_DURATION 1174 +#define WT_STAT_CONN_LOG_SYNC_DIR_DURATION 1186 /*! log: log write operations */ -#define WT_STAT_CONN_LOG_WRITES 1175 +#define WT_STAT_CONN_LOG_WRITES 1187 /*! log: logging bytes consolidated */ -#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1176 +#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1188 /*! log: maximum log file size */ -#define WT_STAT_CONN_LOG_MAX_FILESIZE 1177 +#define WT_STAT_CONN_LOG_MAX_FILESIZE 1189 /*! log: number of pre-allocated log files to create */ -#define WT_STAT_CONN_LOG_PREALLOC_MAX 1178 +#define WT_STAT_CONN_LOG_PREALLOC_MAX 1190 /*! log: pre-allocated log files not ready and missed */ -#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1179 +#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1191 /*! log: pre-allocated log files prepared */ -#define WT_STAT_CONN_LOG_PREALLOC_FILES 1180 +#define WT_STAT_CONN_LOG_PREALLOC_FILES 1192 /*! log: pre-allocated log files used */ -#define WT_STAT_CONN_LOG_PREALLOC_USED 1181 +#define WT_STAT_CONN_LOG_PREALLOC_USED 1193 /*! log: records processed by log scan */ -#define WT_STAT_CONN_LOG_SCAN_RECORDS 1182 +#define WT_STAT_CONN_LOG_SCAN_RECORDS 1194 /*! log: slot close lost race */ -#define WT_STAT_CONN_LOG_SLOT_CLOSE_RACE 1183 +#define WT_STAT_CONN_LOG_SLOT_CLOSE_RACE 1195 /*! log: slot close unbuffered waits */ -#define WT_STAT_CONN_LOG_SLOT_CLOSE_UNBUF 1184 +#define WT_STAT_CONN_LOG_SLOT_CLOSE_UNBUF 1196 /*! log: slot closures */ -#define WT_STAT_CONN_LOG_SLOT_CLOSES 1185 +#define WT_STAT_CONN_LOG_SLOT_CLOSES 1197 /*! log: slot join atomic update races */ -#define WT_STAT_CONN_LOG_SLOT_RACES 1186 +#define WT_STAT_CONN_LOG_SLOT_RACES 1198 /*! log: slot join calls atomic updates raced */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_RACE 1187 +#define WT_STAT_CONN_LOG_SLOT_YIELD_RACE 1199 /*! log: slot join calls did not yield */ -#define WT_STAT_CONN_LOG_SLOT_IMMEDIATE 1188 +#define WT_STAT_CONN_LOG_SLOT_IMMEDIATE 1200 /*! log: slot join calls found active slot closed */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_CLOSE 1189 +#define WT_STAT_CONN_LOG_SLOT_YIELD_CLOSE 1201 /*! log: slot join calls slept */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_SLEEP 1190 +#define WT_STAT_CONN_LOG_SLOT_YIELD_SLEEP 1202 /*! log: slot join calls yielded */ -#define WT_STAT_CONN_LOG_SLOT_YIELD 1191 +#define WT_STAT_CONN_LOG_SLOT_YIELD 1203 /*! log: slot join found active slot closed */ -#define WT_STAT_CONN_LOG_SLOT_ACTIVE_CLOSED 1192 +#define WT_STAT_CONN_LOG_SLOT_ACTIVE_CLOSED 1204 /*! log: slot joins yield time (usecs) */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_DURATION 1193 +#define WT_STAT_CONN_LOG_SLOT_YIELD_DURATION 1205 /*! log: slot transitions unable to find free slot */ -#define WT_STAT_CONN_LOG_SLOT_NO_FREE_SLOTS 1194 +#define WT_STAT_CONN_LOG_SLOT_NO_FREE_SLOTS 1206 /*! log: slot unbuffered writes */ -#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1195 +#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1207 /*! log: total in-memory size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_MEM 1196 +#define WT_STAT_CONN_LOG_COMPRESS_MEM 1208 /*! log: total log buffer size */ -#define WT_STAT_CONN_LOG_BUFFER_SIZE 1197 +#define WT_STAT_CONN_LOG_BUFFER_SIZE 1209 /*! log: total size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_LEN 1198 +#define WT_STAT_CONN_LOG_COMPRESS_LEN 1210 /*! log: written slots coalesced */ -#define WT_STAT_CONN_LOG_SLOT_COALESCED 1199 +#define WT_STAT_CONN_LOG_SLOT_COALESCED 1211 /*! log: yields waiting for previous log file close */ -#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1200 +#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1212 /*! reconciliation: fast-path pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1201 +#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1213 /*! reconciliation: page reconciliation calls */ -#define WT_STAT_CONN_REC_PAGES 1202 +#define WT_STAT_CONN_REC_PAGES 1214 /*! reconciliation: page reconciliation calls for eviction */ -#define WT_STAT_CONN_REC_PAGES_EVICTION 1203 +#define WT_STAT_CONN_REC_PAGES_EVICTION 1215 /*! reconciliation: pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE 1204 +#define WT_STAT_CONN_REC_PAGE_DELETE 1216 /*! reconciliation: split bytes currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1205 +#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1217 /*! reconciliation: split objects currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1206 +#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1218 /*! session: open cursor count */ -#define WT_STAT_CONN_SESSION_CURSOR_OPEN 1207 +#define WT_STAT_CONN_SESSION_CURSOR_OPEN 1219 /*! session: open session count */ -#define WT_STAT_CONN_SESSION_OPEN 1208 +#define WT_STAT_CONN_SESSION_OPEN 1220 /*! session: table alter failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_FAIL 1209 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_FAIL 1221 /*! session: table alter successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_SUCCESS 1210 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_SUCCESS 1222 /*! session: table alter unchanged and skipped */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_SKIP 1211 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_SKIP 1223 /*! session: table compact failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_FAIL 1212 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_FAIL 1224 /*! session: table compact successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SUCCESS 1213 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SUCCESS 1225 /*! session: table create failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_CREATE_FAIL 1214 +#define WT_STAT_CONN_SESSION_TABLE_CREATE_FAIL 1226 /*! session: table create successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_CREATE_SUCCESS 1215 +#define WT_STAT_CONN_SESSION_TABLE_CREATE_SUCCESS 1227 /*! session: table drop failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_DROP_FAIL 1216 +#define WT_STAT_CONN_SESSION_TABLE_DROP_FAIL 1228 /*! session: table drop successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_DROP_SUCCESS 1217 +#define WT_STAT_CONN_SESSION_TABLE_DROP_SUCCESS 1229 /*! session: table rebalance failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_FAIL 1218 +#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_FAIL 1230 /*! session: table rebalance successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_SUCCESS 1219 +#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_SUCCESS 1231 /*! session: table rename failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1220 +#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1232 /*! session: table rename successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1221 +#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1233 /*! session: table salvage failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1222 +#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1234 /*! session: table salvage successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1223 +#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1235 /*! session: table truncate failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1224 +#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1236 /*! session: table truncate successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1225 +#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1237 /*! session: table verify failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1226 +#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1238 /*! session: table verify successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1227 +#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1239 /*! thread-state: active filesystem fsync calls */ -#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1228 +#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1240 /*! thread-state: active filesystem read calls */ -#define WT_STAT_CONN_THREAD_READ_ACTIVE 1229 +#define WT_STAT_CONN_THREAD_READ_ACTIVE 1241 /*! thread-state: active filesystem write calls */ -#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1230 +#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1242 /*! thread-yield: application thread time evicting (usecs) */ -#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1231 +#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1243 /*! thread-yield: application thread time waiting for cache (usecs) */ -#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1232 +#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1244 /*! thread-yield: page acquire busy blocked */ -#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1233 +#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1245 /*! thread-yield: page acquire eviction blocked */ -#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1234 +#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1246 /*! thread-yield: page acquire locked blocked */ -#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1235 +#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1247 /*! thread-yield: page acquire read blocked */ -#define WT_STAT_CONN_PAGE_READ_BLOCKED 1236 +#define WT_STAT_CONN_PAGE_READ_BLOCKED 1248 /*! thread-yield: page acquire time sleeping (usecs) */ -#define WT_STAT_CONN_PAGE_SLEEP 1237 +#define WT_STAT_CONN_PAGE_SLEEP 1249 /*! transaction: number of named snapshots created */ -#define WT_STAT_CONN_TXN_SNAPSHOTS_CREATED 1238 +#define WT_STAT_CONN_TXN_SNAPSHOTS_CREATED 1250 /*! transaction: number of named snapshots dropped */ -#define WT_STAT_CONN_TXN_SNAPSHOTS_DROPPED 1239 +#define WT_STAT_CONN_TXN_SNAPSHOTS_DROPPED 1251 /*! transaction: transaction begins */ -#define WT_STAT_CONN_TXN_BEGIN 1240 +#define WT_STAT_CONN_TXN_BEGIN 1252 /*! transaction: transaction checkpoint currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1241 +#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1253 /*! transaction: transaction checkpoint generation */ -#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1242 +#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1254 /*! transaction: transaction checkpoint max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1243 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1255 /*! transaction: transaction checkpoint min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1244 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1256 /*! transaction: transaction checkpoint most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1245 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1257 /*! transaction: transaction checkpoint scrub dirty target */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1246 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1258 /*! transaction: transaction checkpoint scrub time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1247 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1259 /*! transaction: transaction checkpoint total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1248 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1260 /*! transaction: transaction checkpoints */ -#define WT_STAT_CONN_TXN_CHECKPOINT 1249 +#define WT_STAT_CONN_TXN_CHECKPOINT 1261 /*! * transaction: transaction checkpoints skipped because database was * clean */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1250 +#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1262 /*! transaction: transaction failures due to cache overflow */ -#define WT_STAT_CONN_TXN_FAIL_CACHE 1251 +#define WT_STAT_CONN_TXN_FAIL_CACHE 1263 /*! * transaction: transaction fsync calls for checkpoint after allocating * the transaction ID */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1252 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1264 /*! * transaction: transaction fsync duration for checkpoint after * allocating the transaction ID (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1253 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1265 /*! transaction: transaction range of IDs currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_RANGE 1254 +#define WT_STAT_CONN_TXN_PINNED_RANGE 1266 /*! transaction: transaction range of IDs currently pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1255 +#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1267 /*! * transaction: transaction range of IDs currently pinned by named * snapshots */ -#define WT_STAT_CONN_TXN_PINNED_SNAPSHOT_RANGE 1256 +#define WT_STAT_CONN_TXN_PINNED_SNAPSHOT_RANGE 1268 /*! transaction: transaction sync calls */ -#define WT_STAT_CONN_TXN_SYNC 1257 +#define WT_STAT_CONN_TXN_SYNC 1269 /*! transaction: transactions committed */ -#define WT_STAT_CONN_TXN_COMMIT 1258 +#define WT_STAT_CONN_TXN_COMMIT 1270 /*! transaction: transactions rolled back */ -#define WT_STAT_CONN_TXN_ROLLBACK 1259 +#define WT_STAT_CONN_TXN_ROLLBACK 1271 /*! transaction: update conflicts */ -#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1260 +#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1272 /*! * @} @@ -5192,219 +5240,268 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_DSRC_CACHE_EVICTION_CHECKPOINT 2043 /*! cache: data source pages selected for eviction unable to be evicted */ #define WT_STAT_DSRC_CACHE_EVICTION_FAIL 2044 +/*! cache: eviction walk passes of a file */ +#define WT_STAT_DSRC_CACHE_EVICTION_WALK_PASSES 2045 +/*! cache: eviction walk target pages histogram - 0-9 */ +#define WT_STAT_DSRC_CACHE_EVICTION_TARGET_PAGE_LT10 2046 +/*! cache: eviction walk target pages histogram - 10-31 */ +#define WT_STAT_DSRC_CACHE_EVICTION_TARGET_PAGE_LT32 2047 +/*! cache: eviction walk target pages histogram - 128 and higher */ +#define WT_STAT_DSRC_CACHE_EVICTION_TARGET_PAGE_GE128 2048 +/*! cache: eviction walk target pages histogram - 32-63 */ +#define WT_STAT_DSRC_CACHE_EVICTION_TARGET_PAGE_LT64 2049 +/*! cache: eviction walk target pages histogram - 64-128 */ +#define WT_STAT_DSRC_CACHE_EVICTION_TARGET_PAGE_LT128 2050 +/*! cache: eviction walks abandoned */ +#define WT_STAT_DSRC_CACHE_EVICTION_WALKS_ABANDONED 2051 +/*! cache: eviction walks gave up because they restarted their walk twice */ +#define WT_STAT_DSRC_CACHE_EVICTION_WALKS_STOPPED 2052 +/*! + * cache: eviction walks gave up because they saw too many pages and + * found no candidates + */ +#define WT_STAT_DSRC_CACHE_EVICTION_WALKS_GAVE_UP_NO_TARGETS 2053 +/*! + * cache: eviction walks gave up because they saw too many pages and + * found too few candidates + */ +#define WT_STAT_DSRC_CACHE_EVICTION_WALKS_GAVE_UP_RATIO 2054 +/*! cache: eviction walks reached end of tree */ +#define WT_STAT_DSRC_CACHE_EVICTION_WALKS_ENDED 2055 +/*! cache: eviction walks started from root of tree */ +#define WT_STAT_DSRC_CACHE_EVICTION_WALK_FROM_ROOT 2056 +/*! cache: eviction walks started from saved location in tree */ +#define WT_STAT_DSRC_CACHE_EVICTION_WALK_SAVED_POS 2057 /*! cache: hazard pointer blocked page eviction */ -#define WT_STAT_DSRC_CACHE_EVICTION_HAZARD 2045 +#define WT_STAT_DSRC_CACHE_EVICTION_HAZARD 2058 /*! cache: in-memory page passed criteria to be split */ -#define WT_STAT_DSRC_CACHE_INMEM_SPLITTABLE 2046 +#define WT_STAT_DSRC_CACHE_INMEM_SPLITTABLE 2059 /*! cache: in-memory page splits */ -#define WT_STAT_DSRC_CACHE_INMEM_SPLIT 2047 +#define WT_STAT_DSRC_CACHE_INMEM_SPLIT 2060 /*! cache: internal pages evicted */ -#define WT_STAT_DSRC_CACHE_EVICTION_INTERNAL 2048 +#define WT_STAT_DSRC_CACHE_EVICTION_INTERNAL 2061 /*! cache: internal pages split during eviction */ -#define WT_STAT_DSRC_CACHE_EVICTION_SPLIT_INTERNAL 2049 +#define WT_STAT_DSRC_CACHE_EVICTION_SPLIT_INTERNAL 2062 /*! cache: leaf pages split during eviction */ -#define WT_STAT_DSRC_CACHE_EVICTION_SPLIT_LEAF 2050 +#define WT_STAT_DSRC_CACHE_EVICTION_SPLIT_LEAF 2063 /*! cache: modified pages evicted */ -#define WT_STAT_DSRC_CACHE_EVICTION_DIRTY 2051 +#define WT_STAT_DSRC_CACHE_EVICTION_DIRTY 2064 /*! cache: overflow pages read into cache */ -#define WT_STAT_DSRC_CACHE_READ_OVERFLOW 2052 -/*! cache: overflow values cached in memory */ -#define WT_STAT_DSRC_CACHE_OVERFLOW_VALUE 2053 +#define WT_STAT_DSRC_CACHE_READ_OVERFLOW 2065 /*! cache: page split during eviction deepened the tree */ -#define WT_STAT_DSRC_CACHE_EVICTION_DEEPEN 2054 +#define WT_STAT_DSRC_CACHE_EVICTION_DEEPEN 2066 /*! cache: page written requiring lookaside records */ -#define WT_STAT_DSRC_CACHE_WRITE_LOOKASIDE 2055 +#define WT_STAT_DSRC_CACHE_WRITE_LOOKASIDE 2067 /*! cache: pages read into cache */ -#define WT_STAT_DSRC_CACHE_READ 2056 +#define WT_STAT_DSRC_CACHE_READ 2068 /*! cache: pages read into cache requiring lookaside entries */ -#define WT_STAT_DSRC_CACHE_READ_LOOKASIDE 2057 +#define WT_STAT_DSRC_CACHE_READ_LOOKASIDE 2069 /*! cache: pages requested from the cache */ -#define WT_STAT_DSRC_CACHE_PAGES_REQUESTED 2058 +#define WT_STAT_DSRC_CACHE_PAGES_REQUESTED 2070 +/*! cache: pages seen by eviction walk */ +#define WT_STAT_DSRC_CACHE_EVICTION_PAGES_SEEN 2071 /*! cache: pages written from cache */ -#define WT_STAT_DSRC_CACHE_WRITE 2059 +#define WT_STAT_DSRC_CACHE_WRITE 2072 /*! cache: pages written requiring in-memory restoration */ -#define WT_STAT_DSRC_CACHE_WRITE_RESTORE 2060 +#define WT_STAT_DSRC_CACHE_WRITE_RESTORE 2073 /*! cache: tracked dirty bytes in the cache */ -#define WT_STAT_DSRC_CACHE_BYTES_DIRTY 2061 +#define WT_STAT_DSRC_CACHE_BYTES_DIRTY 2074 /*! cache: unmodified pages evicted */ -#define WT_STAT_DSRC_CACHE_EVICTION_CLEAN 2062 +#define WT_STAT_DSRC_CACHE_EVICTION_CLEAN 2075 /*! * cache_walk: Average difference between current eviction generation * when the page was last considered, only reported if cache_walk or all * statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_GEN_AVG_GAP 2063 +#define WT_STAT_DSRC_CACHE_STATE_GEN_AVG_GAP 2076 /*! * cache_walk: Average on-disk page image size seen, only reported if * cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_AVG_WRITTEN_SIZE 2064 +#define WT_STAT_DSRC_CACHE_STATE_AVG_WRITTEN_SIZE 2077 +/*! + * cache_walk: Average time in cache for pages that have been visited by + * the eviction server, only reported if cache_walk or all statistics are + * enabled + */ +#define WT_STAT_DSRC_CACHE_STATE_AVG_VISITED_AGE 2078 +/*! + * cache_walk: Average time in cache for pages that have not been visited + * by the eviction server, only reported if cache_walk or all statistics + * are enabled + */ +#define WT_STAT_DSRC_CACHE_STATE_AVG_UNVISITED_AGE 2079 /*! * cache_walk: Clean pages currently in cache, only reported if * cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_PAGES_CLEAN 2065 +#define WT_STAT_DSRC_CACHE_STATE_PAGES_CLEAN 2080 /*! * cache_walk: Current eviction generation, only reported if cache_walk * or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_GEN_CURRENT 2066 +#define WT_STAT_DSRC_CACHE_STATE_GEN_CURRENT 2081 /*! * cache_walk: Dirty pages currently in cache, only reported if * cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_PAGES_DIRTY 2067 +#define WT_STAT_DSRC_CACHE_STATE_PAGES_DIRTY 2082 /*! * cache_walk: Entries in the root page, only reported if cache_walk or * all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_ROOT_ENTRIES 2068 +#define WT_STAT_DSRC_CACHE_STATE_ROOT_ENTRIES 2083 /*! * cache_walk: Internal pages currently in cache, only reported if * cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_PAGES_INTERNAL 2069 +#define WT_STAT_DSRC_CACHE_STATE_PAGES_INTERNAL 2084 /*! * cache_walk: Leaf pages currently in cache, only reported if cache_walk * or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_PAGES_LEAF 2070 +#define WT_STAT_DSRC_CACHE_STATE_PAGES_LEAF 2085 /*! * cache_walk: Maximum difference between current eviction generation * when the page was last considered, only reported if cache_walk or all * statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_GEN_MAX_GAP 2071 +#define WT_STAT_DSRC_CACHE_STATE_GEN_MAX_GAP 2086 /*! * cache_walk: Maximum page size seen, only reported if cache_walk or all * statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_MAX_PAGESIZE 2072 +#define WT_STAT_DSRC_CACHE_STATE_MAX_PAGESIZE 2087 /*! * cache_walk: Minimum on-disk page image size seen, only reported if * cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_MIN_WRITTEN_SIZE 2073 +#define WT_STAT_DSRC_CACHE_STATE_MIN_WRITTEN_SIZE 2088 +/*! + * cache_walk: Number of pages never visited by eviction server, only + * reported if cache_walk or all statistics are enabled + */ +#define WT_STAT_DSRC_CACHE_STATE_UNVISITED_COUNT 2089 /*! * cache_walk: On-disk page image sizes smaller than a single allocation * unit, only reported if cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_SMALLER_ALLOC_SIZE 2074 +#define WT_STAT_DSRC_CACHE_STATE_SMALLER_ALLOC_SIZE 2090 /*! * cache_walk: Pages created in memory and never written, only reported * if cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_MEMORY 2075 +#define WT_STAT_DSRC_CACHE_STATE_MEMORY 2091 /*! * cache_walk: Pages currently queued for eviction, only reported if * cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_QUEUED 2076 +#define WT_STAT_DSRC_CACHE_STATE_QUEUED 2092 /*! * cache_walk: Pages that could not be queued for eviction, only reported * if cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_NOT_QUEUEABLE 2077 +#define WT_STAT_DSRC_CACHE_STATE_NOT_QUEUEABLE 2093 /*! * cache_walk: Refs skipped during cache traversal, only reported if * cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_REFS_SKIPPED 2078 +#define WT_STAT_DSRC_CACHE_STATE_REFS_SKIPPED 2094 /*! * cache_walk: Size of the root page, only reported if cache_walk or all * statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_ROOT_SIZE 2079 +#define WT_STAT_DSRC_CACHE_STATE_ROOT_SIZE 2095 /*! * cache_walk: Total number of pages currently in cache, only reported if * cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_PAGES 2080 +#define WT_STAT_DSRC_CACHE_STATE_PAGES 2096 /*! compression: compressed pages read */ -#define WT_STAT_DSRC_COMPRESS_READ 2081 +#define WT_STAT_DSRC_COMPRESS_READ 2097 /*! compression: compressed pages written */ -#define WT_STAT_DSRC_COMPRESS_WRITE 2082 +#define WT_STAT_DSRC_COMPRESS_WRITE 2098 /*! compression: page written failed to compress */ -#define WT_STAT_DSRC_COMPRESS_WRITE_FAIL 2083 +#define WT_STAT_DSRC_COMPRESS_WRITE_FAIL 2099 /*! compression: page written was too small to compress */ -#define WT_STAT_DSRC_COMPRESS_WRITE_TOO_SMALL 2084 +#define WT_STAT_DSRC_COMPRESS_WRITE_TOO_SMALL 2100 /*! compression: raw compression call failed, additional data available */ -#define WT_STAT_DSRC_COMPRESS_RAW_FAIL_TEMPORARY 2085 +#define WT_STAT_DSRC_COMPRESS_RAW_FAIL_TEMPORARY 2101 /*! compression: raw compression call failed, no additional data available */ -#define WT_STAT_DSRC_COMPRESS_RAW_FAIL 2086 +#define WT_STAT_DSRC_COMPRESS_RAW_FAIL 2102 /*! compression: raw compression call succeeded */ -#define WT_STAT_DSRC_COMPRESS_RAW_OK 2087 +#define WT_STAT_DSRC_COMPRESS_RAW_OK 2103 /*! cursor: bulk-loaded cursor-insert calls */ -#define WT_STAT_DSRC_CURSOR_INSERT_BULK 2088 +#define WT_STAT_DSRC_CURSOR_INSERT_BULK 2104 /*! cursor: create calls */ -#define WT_STAT_DSRC_CURSOR_CREATE 2089 +#define WT_STAT_DSRC_CURSOR_CREATE 2105 /*! cursor: cursor-insert key and value bytes inserted */ -#define WT_STAT_DSRC_CURSOR_INSERT_BYTES 2090 +#define WT_STAT_DSRC_CURSOR_INSERT_BYTES 2106 /*! cursor: cursor-remove key bytes removed */ -#define WT_STAT_DSRC_CURSOR_REMOVE_BYTES 2091 +#define WT_STAT_DSRC_CURSOR_REMOVE_BYTES 2107 /*! cursor: cursor-update value bytes updated */ -#define WT_STAT_DSRC_CURSOR_UPDATE_BYTES 2092 +#define WT_STAT_DSRC_CURSOR_UPDATE_BYTES 2108 /*! cursor: insert calls */ -#define WT_STAT_DSRC_CURSOR_INSERT 2093 +#define WT_STAT_DSRC_CURSOR_INSERT 2109 /*! cursor: modify calls */ -#define WT_STAT_DSRC_CURSOR_MODIFY 2094 +#define WT_STAT_DSRC_CURSOR_MODIFY 2110 /*! cursor: next calls */ -#define WT_STAT_DSRC_CURSOR_NEXT 2095 +#define WT_STAT_DSRC_CURSOR_NEXT 2111 /*! cursor: prev calls */ -#define WT_STAT_DSRC_CURSOR_PREV 2096 +#define WT_STAT_DSRC_CURSOR_PREV 2112 /*! cursor: remove calls */ -#define WT_STAT_DSRC_CURSOR_REMOVE 2097 +#define WT_STAT_DSRC_CURSOR_REMOVE 2113 /*! cursor: reserve calls */ -#define WT_STAT_DSRC_CURSOR_RESERVE 2098 +#define WT_STAT_DSRC_CURSOR_RESERVE 2114 /*! cursor: reset calls */ -#define WT_STAT_DSRC_CURSOR_RESET 2099 +#define WT_STAT_DSRC_CURSOR_RESET 2115 /*! cursor: restarted searches */ -#define WT_STAT_DSRC_CURSOR_RESTART 2100 +#define WT_STAT_DSRC_CURSOR_RESTART 2116 /*! cursor: search calls */ -#define WT_STAT_DSRC_CURSOR_SEARCH 2101 +#define WT_STAT_DSRC_CURSOR_SEARCH 2117 /*! cursor: search near calls */ -#define WT_STAT_DSRC_CURSOR_SEARCH_NEAR 2102 +#define WT_STAT_DSRC_CURSOR_SEARCH_NEAR 2118 /*! cursor: truncate calls */ -#define WT_STAT_DSRC_CURSOR_TRUNCATE 2103 +#define WT_STAT_DSRC_CURSOR_TRUNCATE 2119 /*! cursor: update calls */ -#define WT_STAT_DSRC_CURSOR_UPDATE 2104 +#define WT_STAT_DSRC_CURSOR_UPDATE 2120 /*! reconciliation: dictionary matches */ -#define WT_STAT_DSRC_REC_DICTIONARY 2105 +#define WT_STAT_DSRC_REC_DICTIONARY 2121 /*! reconciliation: fast-path pages deleted */ -#define WT_STAT_DSRC_REC_PAGE_DELETE_FAST 2106 +#define WT_STAT_DSRC_REC_PAGE_DELETE_FAST 2122 /*! * reconciliation: internal page key bytes discarded using suffix * compression */ -#define WT_STAT_DSRC_REC_SUFFIX_COMPRESSION 2107 +#define WT_STAT_DSRC_REC_SUFFIX_COMPRESSION 2123 /*! reconciliation: internal page multi-block writes */ -#define WT_STAT_DSRC_REC_MULTIBLOCK_INTERNAL 2108 +#define WT_STAT_DSRC_REC_MULTIBLOCK_INTERNAL 2124 /*! reconciliation: internal-page overflow keys */ -#define WT_STAT_DSRC_REC_OVERFLOW_KEY_INTERNAL 2109 +#define WT_STAT_DSRC_REC_OVERFLOW_KEY_INTERNAL 2125 /*! reconciliation: leaf page key bytes discarded using prefix compression */ -#define WT_STAT_DSRC_REC_PREFIX_COMPRESSION 2110 +#define WT_STAT_DSRC_REC_PREFIX_COMPRESSION 2126 /*! reconciliation: leaf page multi-block writes */ -#define WT_STAT_DSRC_REC_MULTIBLOCK_LEAF 2111 +#define WT_STAT_DSRC_REC_MULTIBLOCK_LEAF 2127 /*! reconciliation: leaf-page overflow keys */ -#define WT_STAT_DSRC_REC_OVERFLOW_KEY_LEAF 2112 +#define WT_STAT_DSRC_REC_OVERFLOW_KEY_LEAF 2128 /*! reconciliation: maximum blocks required for a page */ -#define WT_STAT_DSRC_REC_MULTIBLOCK_MAX 2113 +#define WT_STAT_DSRC_REC_MULTIBLOCK_MAX 2129 /*! reconciliation: overflow values written */ -#define WT_STAT_DSRC_REC_OVERFLOW_VALUE 2114 +#define WT_STAT_DSRC_REC_OVERFLOW_VALUE 2130 /*! reconciliation: page checksum matches */ -#define WT_STAT_DSRC_REC_PAGE_MATCH 2115 +#define WT_STAT_DSRC_REC_PAGE_MATCH 2131 /*! reconciliation: page reconciliation calls */ -#define WT_STAT_DSRC_REC_PAGES 2116 +#define WT_STAT_DSRC_REC_PAGES 2132 /*! reconciliation: page reconciliation calls for eviction */ -#define WT_STAT_DSRC_REC_PAGES_EVICTION 2117 +#define WT_STAT_DSRC_REC_PAGES_EVICTION 2133 /*! reconciliation: pages deleted */ -#define WT_STAT_DSRC_REC_PAGE_DELETE 2118 +#define WT_STAT_DSRC_REC_PAGE_DELETE 2134 /*! session: object compaction */ -#define WT_STAT_DSRC_SESSION_COMPACT 2119 +#define WT_STAT_DSRC_SESSION_COMPACT 2135 /*! session: open cursor count */ -#define WT_STAT_DSRC_SESSION_CURSOR_OPEN 2120 +#define WT_STAT_DSRC_SESSION_CURSOR_OPEN 2136 /*! transaction: update conflicts */ -#define WT_STAT_DSRC_TXN_UPDATE_CONFLICT 2121 +#define WT_STAT_DSRC_TXN_UPDATE_CONFLICT 2137 /*! * @} diff --git a/src/third_party/wiredtiger/src/include/wt_internal.h b/src/third_party/wiredtiger/src/include/wt_internal.h index 1c9600dd27f..74fdc4c3925 100644 --- a/src/third_party/wiredtiger/src/include/wt_internal.h +++ b/src/third_party/wiredtiger/src/include/wt_internal.h @@ -250,8 +250,6 @@ struct __wt_ovfl_reuse; typedef struct __wt_ovfl_reuse WT_OVFL_REUSE; struct __wt_ovfl_track; typedef struct __wt_ovfl_track WT_OVFL_TRACK; -struct __wt_ovfl_txnc; - typedef struct __wt_ovfl_txnc WT_OVFL_TXNC; struct __wt_page; typedef struct __wt_page WT_PAGE; struct __wt_page_deleted; diff --git a/src/third_party/wiredtiger/src/log/log.c b/src/third_party/wiredtiger/src/log/log.c index 868f5d0eaf4..2a912e6568f 100644 --- a/src/third_party/wiredtiger/src/log/log.c +++ b/src/third_party/wiredtiger/src/log/log.c @@ -9,8 +9,7 @@ #include "wt_internal.h" static int __log_newfile(WT_SESSION_IMPL *, bool, bool *); -static int __log_openfile( - WT_SESSION_IMPL *, WT_FH **, const char *, uint32_t, uint32_t); +static int __log_openfile(WT_SESSION_IMPL *, uint32_t, uint32_t, WT_FH **); static int __log_write_internal( WT_SESSION_IMPL *, WT_ITEM *, WT_LSN *, uint32_t); @@ -19,7 +18,109 @@ static int __log_write_internal( /* Flags to __log_openfile */ #define WT_LOG_OPEN_CREATE_OK 0x01 -#define WT_LOG_OPEN_VERIFY 0x02 + +/* + * __log_printf_internal -- + * Internal call to write a log message. + */ +static int +__log_printf_internal(WT_SESSION_IMPL *session, const char *fmt, ...) +{ + WT_DECL_RET; + va_list ap; + + va_start(ap, fmt); + ret = __wt_log_vprintf(session, fmt, ap); + va_end(ap); + return (ret); +} + +/* + * __log_checksum_match -- + * Given a log record, return whether the checksum matches. + */ +static bool +__log_checksum_match(WT_SESSION_IMPL *session, WT_ITEM *buf, uint32_t reclen) +{ + WT_LOG_RECORD *logrec; + uint32_t checksum_calculate, checksum_tmp; + + WT_UNUSED(session); + logrec = (WT_LOG_RECORD *)buf->mem; + checksum_tmp = logrec->checksum; + logrec->checksum = 0; + checksum_calculate = __wt_checksum(logrec, reclen); +#ifdef WORDS_BIGENDIAN + checksum_calculate = __wt_bswap32(checksum_calculate); +#endif + logrec->checksum = checksum_tmp; + if (logrec->checksum != checksum_calculate) + return (false); + else + return (true); +} + +/* + * __log_get_files -- + * Retrieve the list of all log-related files of the given prefix type. + */ +static int +__log_get_files(WT_SESSION_IMPL *session, + const char *file_prefix, char ***filesp, u_int *countp) +{ + WT_CONNECTION_IMPL *conn; + const char *log_path; + + *countp = 0; + *filesp = NULL; + + conn = S2C(session); + log_path = conn->log_path; + if (log_path == NULL) + log_path = ""; + return (__wt_fs_directory_list( + session, log_path, file_prefix, filesp, countp)); +} + +/* + * __log_prealloc_remove -- + * Remove all previously created pre-allocated files. + */ +static int +__log_prealloc_remove(WT_SESSION_IMPL *session) +{ + WT_DECL_RET; + WT_LOG *log; + uint32_t lognum; + u_int i, logcount; + char **logfiles; + + logfiles = NULL; + logcount = 0; + log = S2C(session)->log; + __wt_spin_lock(session, &log->log_fs_lock); + /* + * Clean up any old interim pre-allocated files. We clean + * up these files because settings may have changed upon reboot + * and we want those settings to take effect right away. + */ + WT_ERR(__log_get_files(session, + WT_LOG_TMPNAME, &logfiles, &logcount)); + for (i = 0; i < logcount; i++) { + WT_ERR(__wt_log_extract_lognum(session, logfiles[i], &lognum)); + WT_ERR(__wt_log_remove(session, WT_LOG_TMPNAME, lognum)); + } + WT_ERR(__wt_fs_directory_list_free(session, &logfiles, logcount)); + WT_ERR(__log_get_files(session, + WT_LOG_PREPNAME, &logfiles, &logcount)); + for (i = 0; i < logcount; i++) { + WT_ERR(__wt_log_extract_lognum(session, logfiles[i], &lognum)); + WT_ERR(__wt_log_remove(session, WT_LOG_PREPNAME, lognum)); + } +err: WT_TRET(__wt_fs_directory_list_free(session, &logfiles, logcount)); + __wt_spin_unlock(session, &log->log_fs_lock); + return (ret); +} /* * __log_wait_for_earlier_slot -- @@ -66,11 +167,13 @@ __log_fs_write(WT_SESSION_IMPL *session, WT_DECL_RET; /* - * If we're writing into a new log file, we have to wait for all + * If we're writing into a new log file and we're running in + * compatibility mode to an older release, we have to wait for all * writes to the previous log file to complete otherwise there could * be a hole at the end of the previous log file that we cannot detect. */ - if (slot->slot_release_lsn.l.file < slot->slot_start_lsn.l.file) { + if (S2C(session)->log->log_version != WT_LOG_VERSION && + slot->slot_release_lsn.l.file < slot->slot_start_lsn.l.file) { __log_wait_for_earlier_slot(session, slot); WT_RET(__wt_log_force_sync(session, &slot->slot_release_lsn)); } @@ -208,8 +311,7 @@ __wt_log_force_sync(WT_SESSION_IMPL *session, WT_LSN *min_lsn) * from under us and either be NULL or point to a different * file than we want. */ - WT_ERR(__log_openfile(session, - &log_fh, WT_LOG_FILENAME, min_lsn->l.file, 0)); + WT_ERR(__log_openfile(session, min_lsn->l.file, 0, &log_fh)); __wt_verbose(session, WT_VERB_LOG, "log_force_sync: sync %s to LSN %" PRIu32 "/%" PRIu32, log_fh->name, min_lsn->l.file, min_lsn->l.offset); @@ -314,28 +416,6 @@ __wt_log_written_reset(WT_SESSION_IMPL *session) } /* - * __log_get_files -- - * Retrieve the list of all log-related files of the given prefix type. - */ -static int -__log_get_files(WT_SESSION_IMPL *session, - const char *file_prefix, char ***filesp, u_int *countp) -{ - WT_CONNECTION_IMPL *conn; - const char *log_path; - - *countp = 0; - *filesp = NULL; - - conn = S2C(session); - log_path = conn->log_path; - if (log_path == NULL) - log_path = ""; - return (__wt_fs_directory_list( - session, log_path, file_prefix, filesp, countp)); -} - -/* * __wt_log_get_all_files -- * Retrieve the list of log files, either all of them or only the active * ones (those that are not candidates for archiving). The caller is @@ -571,7 +651,7 @@ __log_prealloc(WT_SESSION_IMPL *session, WT_FH *fh) */ if (FLD_ISSET(conn->log_flags, WT_CONN_LOG_ZERO_FILL)) return (__log_zero(session, fh, - WT_LOG_FIRST_RECORD, conn->log_file_max)); + WT_LOG_END_HEADER, conn->log_file_max)); /* * We have exclusive access to the log file and there are no other @@ -593,7 +673,7 @@ __log_size_fit(WT_SESSION_IMPL *session, WT_LSN *lsn, uint64_t recsize) conn = S2C(session); log = conn->log; - return (lsn->l.offset == WT_LOG_FIRST_RECORD || + return (lsn->l.offset == log->first_record || lsn->l.offset + (wt_off_t)recsize < conn->log_file_max); } @@ -662,17 +742,17 @@ __log_decrypt(WT_SESSION_IMPL *session, WT_ITEM *in, WT_ITEM *out) } /* - * __log_fill -- + * __wt_log_fill -- * Copy a thread's log records into the assigned slot. */ -static int -__log_fill(WT_SESSION_IMPL *session, +int +__wt_log_fill(WT_SESSION_IMPL *session, WT_MYSLOT *myslot, bool force, WT_ITEM *record, WT_LSN *lsnp) { WT_DECL_RET; /* - * Call __wt_write or copy into the buffer. For now the offset is the + * Call write or copy into the buffer. For now the offset is the * real byte offset. If the offset becomes a unit of WT_LOG_ALIGN this * is where we would multiply by WT_LOG_ALIGN to get the real file byte * offset for write(). @@ -733,8 +813,7 @@ __log_file_header( logrec = (WT_LOG_RECORD *)buf->mem; desc = (WT_LOG_DESC *)logrec->record; desc->log_magic = WT_LOG_MAGIC; - desc->majorv = WT_LOG_MAJOR_VERSION; - desc->minorv = WT_LOG_MINOR_VERSION; + desc->version = log->log_version; desc->log_size = (uint64_t)conn->log_file_max; __wt_log_desc_byteswap(desc); @@ -759,7 +838,7 @@ __log_file_header( /* * We may recursively call __wt_log_acquire to allocate log space for - * the log descriptor record. Call __log_fill to write it, but we + * the log descriptor record. Call __wt_log_fill to write it, but we * do not need to call __wt_log_release because we're not waiting for * any earlier operations to complete. */ @@ -770,7 +849,7 @@ __log_file_header( WT_ASSERT(session, fh == NULL); WT_ERR(__wt_log_acquire(session, log->allocsize, &tmp)); } - WT_ERR(__log_fill(session, &myslot, true, buf, NULL)); + WT_ERR(__wt_log_fill(session, &myslot, true, buf, NULL)); /* * Make sure the header gets to disk. */ @@ -787,63 +866,143 @@ err: __wt_scr_free(session, &buf); * Open a log file with the given log file number and return the WT_FH. */ static int -__log_openfile(WT_SESSION_IMPL *session, - WT_FH **fhp, const char *file_prefix, uint32_t id, uint32_t flags) +__log_openfile( + WT_SESSION_IMPL *session, uint32_t id, uint32_t flags, WT_FH **fhp) { WT_CONNECTION_IMPL *conn; WT_DECL_ITEM(buf); WT_DECL_RET; - WT_LOG *log; - WT_LOG_DESC *desc; - WT_LOG_RECORD *logrec; - uint32_t allocsize; u_int wtopen_flags; conn = S2C(session); - log = conn->log; - if (log == NULL) - allocsize = WT_LOG_ALIGN; - else - allocsize = log->allocsize; WT_RET(__wt_scr_alloc(session, 0, &buf)); - WT_ERR(__log_filename(session, id, file_prefix, buf)); + /* + * If we are creating the file then we use a temporary file name. + * Otherwise it is a log file name. + */ + if (LF_ISSET(WT_LOG_OPEN_CREATE_OK)) { + wtopen_flags = WT_FS_OPEN_CREATE; + WT_ERR(__log_filename(session, id, WT_LOG_TMPNAME, buf)); + } else { + wtopen_flags = 0; + WT_ERR(__log_filename(session, id, WT_LOG_FILENAME, buf)); + } __wt_verbose(session, WT_VERB_LOG, "opening log %s", (const char *)buf->data); - wtopen_flags = 0; - if (LF_ISSET(WT_LOG_OPEN_CREATE_OK)) - FLD_SET(wtopen_flags, WT_FS_OPEN_CREATE); if (FLD_ISSET(conn->direct_io, WT_DIRECT_IO_LOG)) FLD_SET(wtopen_flags, WT_FS_OPEN_DIRECTIO); WT_ERR(__wt_open( session, buf->data, WT_FS_OPEN_FILE_TYPE_LOG, wtopen_flags, fhp)); +err: __wt_scr_free(session, &buf); + return (ret); +} + +/* + * __log_open_verify -- + * Open a log file with the given log file number, verify its + * header and return various pieces of system information about + * this log file. + */ +static int +__log_open_verify(WT_SESSION_IMPL *session, uint32_t id, WT_FH **fhp, + WT_LSN *lsnp, uint16_t *versionp) +{ + WT_CONNECTION_IMPL *conn; + WT_DECL_ITEM(buf); + WT_DECL_RET; + WT_FH *fh; + WT_LOG *log; + WT_LOG_DESC *desc; + WT_LOG_RECORD *logrec; + uint32_t allocsize, rectype; + const uint8_t *end, *p; + + conn = S2C(session); + log = conn->log; + WT_RET(__wt_scr_alloc(session, 0, &buf)); + WT_ERR(__log_openfile(session, id, 0, &fh)); + + if (log == NULL) + allocsize = WT_LOG_ALIGN; + else + allocsize = log->allocsize; + if (lsnp != NULL) + WT_ZERO_LSN(lsnp); + WT_ERR(__wt_buf_grow(session, buf, allocsize)); + memset(buf->mem, 0, allocsize); /* - * If we are not creating the log file but opening it for reading, - * check that the magic number and versions are correct. + * Read in the log file header and verify it. */ - if (LF_ISSET(WT_LOG_OPEN_VERIFY)) { - WT_ERR(__wt_buf_grow(session, buf, allocsize)); - memset(buf->mem, 0, allocsize); - WT_ERR(__wt_read(session, *fhp, 0, allocsize, buf->mem)); - logrec = (WT_LOG_RECORD *)buf->mem; - __wt_log_record_byteswap(logrec); - desc = (WT_LOG_DESC *)logrec->record; - __wt_log_desc_byteswap(desc); - if (desc->log_magic != WT_LOG_MAGIC) - WT_PANIC_RET(session, WT_ERROR, - "log file %s corrupted: Bad magic number %" PRIu32, - (*fhp)->name, desc->log_magic); - if (desc->majorv > WT_LOG_MAJOR_VERSION || - (desc->majorv == WT_LOG_MAJOR_VERSION && - desc->minorv > WT_LOG_MINOR_VERSION)) - WT_ERR_MSG(session, WT_ERROR, - "unsupported WiredTiger file version: this build " - " only supports major/minor versions up to %d/%d, " - " and the file is version %" PRIu16 "/%" PRIu16, - WT_LOG_MAJOR_VERSION, WT_LOG_MINOR_VERSION, - desc->majorv, desc->minorv); + WT_ERR(__wt_read(session, fh, 0, allocsize, buf->mem)); + logrec = (WT_LOG_RECORD *)buf->mem; + __wt_log_record_byteswap(logrec); + desc = (WT_LOG_DESC *)logrec->record; + __wt_log_desc_byteswap(desc); + if (desc->log_magic != WT_LOG_MAGIC) + WT_PANIC_RET(session, WT_ERROR, + "log file %s corrupted: Bad magic number %" PRIu32, + fh->name, desc->log_magic); + /* + * We cannot read future log file formats. + */ + if (desc->version > WT_LOG_VERSION) + WT_ERR_MSG(session, WT_ERROR, + "unsupported WiredTiger file version: this build " + " only supports versions up to %d," + " and the file is version %" PRIu16, + WT_LOG_VERSION, desc->version); + + /* + * Set up the return values if the magic number is valid. + */ + if (versionp != NULL) + *versionp = desc->version; + + /* + * Skip reading in the previous LSN if log file is an old version + * or if the caller doesn't care about the LSN. Otherwise read that + * record in and set up the LSN. We already have a buffer that is + * the correct size. Reuse it. + */ + if (lsnp == NULL || + (desc->version < WT_LOG_VERSION_SYSTEM)) + goto err; + + memset(buf->mem, 0, allocsize); + WT_ERR(__wt_read(session, fh, allocsize, allocsize, buf->mem)); + logrec = (WT_LOG_RECORD *)buf->mem; + /* + * We have a valid header but the system record is not there. + * The log ends here. Return without setting the LSN. + */ + if (logrec->len == 0) { + __wt_verbose(session, WT_VERB_LOG, + "Log %s found empty log after header", fh->name); + goto err; } + + if (!__log_checksum_match(session, buf, allocsize)) + WT_ERR_MSG(session, WT_ERROR, + "%s: System log record checksum mismatch", fh->name); + __wt_log_record_byteswap(logrec); + p = WT_LOG_SKIP_HEADER(buf->data); + end = (const uint8_t *)buf->data + allocsize; + WT_ERR(__wt_logrec_read(session, &p, end, &rectype)); + if (rectype != WT_LOGREC_SYSTEM) + WT_ERR_MSG(session, WT_ERROR, "System log record missing"); + WT_ERR(__wt_log_recover_system(session, &p, end, lsnp)); + err: __wt_scr_free(session, &buf); + + /* + * Return the file handle if needed, otherwise close it. + */ + if (fhp != NULL && ret == 0) + *fhp = fh; + else + WT_TRET(__wt_close(session, &fh)); + return (ret); } @@ -858,6 +1017,7 @@ __log_alloc_prealloc(WT_SESSION_IMPL *session, uint32_t to_num) WT_DECL_ITEM(from_path); WT_DECL_ITEM(to_path); WT_DECL_RET; + WT_LOG *log; uint32_t from_num; u_int logcount; char **logfiles; @@ -865,6 +1025,7 @@ __log_alloc_prealloc(WT_SESSION_IMPL *session, uint32_t to_num) /* * If there are no pre-allocated files, return WT_NOTFOUND. */ + log = S2C(session)->log; logfiles = NULL; WT_ERR(__log_get_files(session, WT_LOG_PREPNAME, &logfiles, &logcount)); if (logcount == 0) @@ -879,6 +1040,7 @@ __log_alloc_prealloc(WT_SESSION_IMPL *session, uint32_t to_num) WT_ERR(__wt_scr_alloc(session, 0, &to_path)); WT_ERR(__log_filename(session, from_num, WT_LOG_PREPNAME, from_path)); WT_ERR(__log_filename(session, to_num, WT_LOG_FILENAME, to_path)); + __wt_spin_lock(session, &log->log_fs_lock); __wt_verbose(session, WT_VERB_LOG, "log_alloc_prealloc: rename log %s to %s", (const char *)from_path->data, (const char *)to_path->data); @@ -891,6 +1053,7 @@ __log_alloc_prealloc(WT_SESSION_IMPL *session, uint32_t to_num) err: __wt_scr_free(session, &from_path); __wt_scr_free(session, &to_path); + __wt_spin_unlock(session, &log->log_fs_lock); WT_TRET(__wt_fs_directory_list_free(session, &logfiles, logcount)); return (ret); } @@ -906,7 +1069,7 @@ __log_newfile(WT_SESSION_IMPL *session, bool conn_open, bool *created) WT_DECL_RET; WT_FH *log_fh; WT_LOG *log; - WT_LSN end_lsn; + WT_LSN end_lsn, logrec_lsn; u_int yield_cnt; bool create_log; @@ -941,7 +1104,7 @@ __log_newfile(WT_SESSION_IMPL *session, bool conn_open, bool *created) /* * If pre-allocating log files look for one; otherwise, or if we don't - * find one create a log file. We can't use pre-allocated log files in + * find one, create a log file. We can't use pre-allocated log files * while a hot backup is in progress: applications can copy the files * in any way they choose, and a log file rename might confuse things. */ @@ -986,15 +1149,31 @@ __log_newfile(WT_SESSION_IMPL *session, bool conn_open, bool *created) * we must pass in a local file handle. Otherwise there is a wide * window where another thread could see a NULL log file handle. */ - WT_RET(__log_openfile(session, - &log_fh, WT_LOG_FILENAME, log->fileid, 0)); - WT_PUBLISH(log->log_fh, log_fh); + WT_RET(__log_open_verify(session, log->fileid, &log_fh, NULL, NULL)); + /* + * Write the LSN at the end of the last record in the previous log file + * as the first record in this log file. + */ + if (log->fileid == 1) + WT_INIT_LSN(&logrec_lsn); + else + logrec_lsn = log->alloc_lsn; /* * We need to setup the LSNs. Set the end LSN and alloc LSN to * the end of the header. */ - WT_SET_LSN(&log->alloc_lsn, log->fileid, WT_LOG_FIRST_RECORD); + WT_SET_LSN(&log->alloc_lsn, log->fileid, WT_LOG_END_HEADER); + /* + * If we're running the version where we write a system record + * do so now and update the alloc_lsn. + */ + if (log->log_version == WT_LOG_VERSION) { + WT_RET(__wt_log_system_record(session, + log_fh, &logrec_lsn)); + WT_SET_LSN(&log->alloc_lsn, log->fileid, log->first_record); + } end_lsn = log->alloc_lsn; + WT_PUBLISH(log->log_fh, log_fh); /* * If we're called from connection creation code, we need to update @@ -1012,6 +1191,83 @@ __log_newfile(WT_SESSION_IMPL *session, bool conn_open, bool *created) } /* + * __log_set_version -- + * Set version related information under lock. + */ +static int +__log_set_version(WT_SESSION_IMPL *session, uint16_t version, + uint32_t first_rec, bool live_chg, bool downgrade) +{ + WT_CONNECTION_IMPL *conn; + WT_LOG *log; + + conn = S2C(session); + log = conn->log; + + log->log_version = version; + log->first_record = first_rec; + if (downgrade) + FLD_SET(conn->log_flags, WT_CONN_LOG_DOWNGRADED); + else + FLD_CLR(conn->log_flags, WT_CONN_LOG_DOWNGRADED); + if (live_chg) + F_SET(log, WT_LOG_FORCE_NEWFILE); + if (!F_ISSET(conn, WT_CONN_READONLY)) + return (__log_prealloc_remove(session)); + else + return (0); +} + +/* + * __wt_log_set_version -- + * Change the version number in logging. Will be done with locking. + * We need to force the log file to advance and remove all old + * pre-allocated files. + */ +int +__wt_log_set_version(WT_SESSION_IMPL *session, uint16_t version, + uint32_t first_rec, bool downgrade, bool live_chg, uint32_t *lognump) +{ + WT_CONNECTION_IMPL *conn; + WT_DECL_RET; + WT_LOG *log; + + conn = S2C(session); + log = conn->log; + + /* + * The steps are: + * - Set up versions and remove files under lock. + * - Set a flag so that the next slot change forces a file change. + * - Force out the slot that is currently active in the current log. + * - Write a log record to force a record into the new log file. + */ + WT_WITH_SLOT_LOCK(session, log, + ret = __log_set_version(session, + version, first_rec, live_chg, downgrade)); + if (!live_chg) + return (ret); + WT_ERR(ret); + /* + * A new log file will be used when we force out the earlier slot. + */ + WT_ERR(__wt_log_force_write(session, 1, NULL)); + + /* + * We need to write a record to the new version log file so that + * a potential checkpoint finds LSNs in that new log file and + * an archive correctly removes all earlier logs. + * Write an internal printf record. + */ + WT_ERR(__log_printf_internal(session, + "COMPATIBILITY: Version now %" PRIu16, log->log_version)); + if (lognump != NULL) + *lognump = log->alloc_lsn.l.file; +err: + return (ret); +} + +/* * __wt_log_acquire -- * Called serially when switching slots. Can be called recursively * from __log_newfile when we change log files. @@ -1043,8 +1299,10 @@ __wt_log_acquire(WT_SESSION_IMPL *session, uint64_t recsize, WT_LOGSLOT *slot) * that exceed the maximum file size. We want to minimize the risk * of an error due to no space. */ - if (!__log_size_fit(session, &log->alloc_lsn, recsize)) { + if (F_ISSET(log, WT_LOG_FORCE_NEWFILE) || + !__log_size_fit(session, &log->alloc_lsn, recsize)) { WT_RET(__log_newfile(session, false, &created_log)); + F_CLR(log, WT_LOG_FORCE_NEWFILE); if (log->log_close_fh != NULL) F_SET(slot, WT_SLOT_CLOSEFH); } @@ -1053,7 +1311,7 @@ __wt_log_acquire(WT_SESSION_IMPL *session, uint64_t recsize, WT_LOGSLOT *slot) * Pre-allocate on the first real write into the log file, if it * was just created (i.e. not pre-allocated). */ - if (log->alloc_lsn.l.offset == WT_LOG_FIRST_RECORD && created_log) + if (log->alloc_lsn.l.offset == log->first_record && created_log) WT_RET(__log_prealloc(session, log->log_fh)); /* * Initialize the slot for activation. @@ -1107,8 +1365,7 @@ __log_truncate_file(WT_SESSION_IMPL *session, WT_FH *log_fh, wt_off_t offset) * we are in recovery or other dedicated time and not during live running. */ static int -__log_truncate(WT_SESSION_IMPL *session, - WT_LSN *lsn, const char *file_prefix, uint32_t this_log) +__log_truncate(WT_SESSION_IMPL *session, WT_LSN *lsn, bool this_log) { WT_CONNECTION_IMPL *conn; WT_DECL_RET; @@ -1133,7 +1390,7 @@ __log_truncate(WT_SESSION_IMPL *session, * before doing work, and if there's a not-supported error, turn off * future truncates. */ - WT_ERR(__log_openfile(session, &log_fh, file_prefix, lsn->l.file, 0)); + WT_ERR(__log_openfile(session, lsn->l.file, 0, &log_fh)); WT_ERR(__log_truncate_file(session, log_fh, lsn->l.offset)); WT_ERR(__wt_fsync(session, log_fh, true)); WT_ERR(__wt_close(session, &log_fh)); @@ -1148,14 +1405,13 @@ __log_truncate(WT_SESSION_IMPL *session, for (i = 0; i < logcount; i++) { WT_ERR(__wt_log_extract_lognum(session, logfiles[i], &lognum)); if (lognum > lsn->l.file && lognum < log->trunc_lsn.l.file) { - WT_ERR(__log_openfile(session, - &log_fh, file_prefix, lognum, 0)); + WT_ERR(__log_openfile(session, lognum, 0, &log_fh)); /* * If there are intervening files pre-allocated, * truncate them to the end of the log file header. */ WT_ERR(__log_truncate_file( - session, log_fh, WT_LOG_FIRST_RECORD)); + session, log_fh, log->first_record)); WT_ERR(__wt_fsync(session, log_fh, true)); WT_ERR(__wt_close(session, &log_fh)); } @@ -1198,17 +1454,17 @@ __wt_log_allocfile( tmp_id = __wt_atomic_add32(&log->tmp_fileid, 1); WT_ERR(__log_filename(session, tmp_id, WT_LOG_TMPNAME, from_path)); WT_ERR(__log_filename(session, lognum, dest, to_path)); + __wt_spin_lock(session, &log->log_fs_lock); /* * Set up the temporary file. */ - WT_ERR(__log_openfile(session, - &log_fh, WT_LOG_TMPNAME, tmp_id, WT_LOG_OPEN_CREATE_OK)); + WT_ERR(__log_openfile(session, tmp_id, WT_LOG_OPEN_CREATE_OK, &log_fh)); WT_ERR(__log_file_header(session, log_fh, NULL, true)); WT_ERR(__log_prealloc(session, log_fh)); WT_ERR(__wt_fsync(session, log_fh, true)); WT_ERR(__wt_close(session, &log_fh)); __wt_verbose(session, WT_VERB_LOG, - "log_prealloc: rename %s to %s", + "log_allocfile: rename %s to %s", (const char *)from_path->data, (const char *)to_path->data); /* * Rename it into place and make it available. @@ -1217,6 +1473,7 @@ __wt_log_allocfile( err: __wt_scr_free(session, &from_path); __wt_scr_free(session, &to_path); + __wt_spin_unlock(session, &log->log_fs_lock); WT_TRET(__wt_close(session, &log_fh)); return (ret); } @@ -1255,6 +1512,7 @@ __wt_log_open(WT_SESSION_IMPL *session) WT_DECL_RET; WT_LOG *log; uint32_t firstlog, lastlog, lognum; + uint16_t version; u_int i, logcount; char **logfiles; @@ -1275,33 +1533,8 @@ __wt_log_open(WT_SESSION_IMPL *session) WT_FS_OPEN_FILE_TYPE_DIRECTORY, 0, &log->log_dir_fh)); } - if (!F_ISSET(conn, WT_CONN_READONLY)) { - /* - * Clean up any old interim pre-allocated files. We clean - * up these files because settings have changed upon reboot - * and we want those settings to take effect right away. - */ - WT_ERR(__log_get_files(session, - WT_LOG_TMPNAME, &logfiles, &logcount)); - for (i = 0; i < logcount; i++) { - WT_ERR(__wt_log_extract_lognum( - session, logfiles[i], &lognum)); - WT_ERR(__wt_log_remove( - session, WT_LOG_TMPNAME, lognum)); - } - WT_ERR( - __wt_fs_directory_list_free(session, &logfiles, logcount)); - WT_ERR(__log_get_files(session, - WT_LOG_PREPNAME, &logfiles, &logcount)); - for (i = 0; i < logcount; i++) { - WT_ERR(__wt_log_extract_lognum( - session, logfiles[i], &lognum)); - WT_ERR(__wt_log_remove( - session, WT_LOG_PREPNAME, lognum)); - } - WT_ERR( - __wt_fs_directory_list_free(session, &logfiles, logcount)); - } + if (!F_ISSET(conn, WT_CONN_READONLY)) + WT_ERR(__log_prealloc_remove(session)); /* * Now look at the log files and set our LSNs. @@ -1334,6 +1567,33 @@ __wt_log_open(WT_SESSION_IMPL *session) /* If we found log files, save the new state. */ if (logcount > 0) { + /* + * If we're running in a downgraded mode and there are earlier + * logs detect if they're at a higher version. If so, we need + * to force recovery (to write a full checkpoint) and force + * archiving to remove all higher version logs. + */ + if (FLD_ISSET(conn->log_flags, WT_CONN_LOG_DOWNGRADED)) { + for (i = 0; i < logcount; ++i) { + WT_ERR(__wt_log_extract_lognum( + session, logfiles[i], &lognum)); + /* + * By sending in a NULL file handle, we don't + * have to close the file. + */ + WT_ERR(__log_open_verify(session, + lognum, NULL, NULL, &version)); + /* + * If we find any log file at the wrong version + * set the flag and we're done. + */ + if (log->log_version != version) { + FLD_SET(conn->log_flags, + WT_CONN_LOG_FORCE_DOWNGRADE); + break; + } + } + } log->trunc_lsn = log->alloc_lsn; FLD_SET(conn->log_flags, WT_CONN_LOG_EXISTED); } @@ -1631,12 +1891,12 @@ __wt_log_scan(WT_SESSION_IMPL *session, WT_LSN *lsnp, uint32_t flags, WT_ITEM *cbbuf; WT_LOG *log; WT_LOG_RECORD *logrec; - WT_LSN end_lsn, next_lsn, rd_lsn, start_lsn; + WT_LSN end_lsn, next_lsn, prev_eof, prev_lsn, rd_lsn, start_lsn; wt_off_t log_size; uint32_t allocsize, firstlog, lastlog, lognum, rdup_len, reclen; - uint32_t checksum_calculate, checksum_tmp; u_int i, logcount; int firstrecord; + uint16_t version; bool eol, partial_record; char **logfiles; @@ -1660,66 +1920,26 @@ __wt_log_scan(WT_SESSION_IMPL *session, WT_LSN *lsnp, uint32_t flags, "__wt_log_scan truncating to %" PRIu32 "/%" PRIu32, log->trunc_lsn.l.file, log->trunc_lsn.l.offset); + if (lsnp != NULL && + LF_ISSET(WT_LOGSCAN_FIRST|WT_LOGSCAN_FROM_CKP)) + WT_RET_MSG(session, WT_ERROR, + "choose either a start LSN or a start flag"); + /* + * Set up the allocation size, starting and ending LSNs. The values + * for those depend on whether logging is currently enabled or not. + */ + lastlog = 0; if (log != NULL) { allocsize = log->allocsize; - + end_lsn = log->alloc_lsn; + start_lsn = log->first_lsn; if (lsnp == NULL) { - if (LF_ISSET(WT_LOGSCAN_FIRST)) - start_lsn = log->first_lsn; - else if (LF_ISSET(WT_LOGSCAN_FROM_CKP)) + if (LF_ISSET(WT_LOGSCAN_FROM_CKP)) start_lsn = log->ckpt_lsn; - else + else if (!LF_ISSET(WT_LOGSCAN_FIRST)) return (WT_ERROR); /* Illegal usage */ - } else { - if (LF_ISSET(WT_LOGSCAN_FIRST|WT_LOGSCAN_FROM_CKP)) - WT_RET_MSG(session, WT_ERROR, - "choose either a start LSN or a start flag"); - - /* - * Offsets must be on allocation boundaries. - * An invalid LSN from a user should just return - * WT_NOTFOUND. It is not an error. But if it is - * from recovery, we expect valid LSNs so give more - * information about that. - */ - if (lsnp->l.offset % allocsize != 0) { - if (LF_ISSET(WT_LOGSCAN_RECOVER)) - WT_RET_MSG(session, WT_NOTFOUND, - "__wt_log_scan unaligned LSN %" - PRIu32 "/%" PRIu32, - lsnp->l.file, lsnp->l.offset); - else - return (WT_NOTFOUND); - } - /* - * If the file is in the future it doesn't exist. - * An invalid LSN from a user should just return - * WT_NOTFOUND. It is not an error. But if it is - * from recovery, we expect valid LSNs so give more - * information about that. - */ - if (lsnp->l.file > log->fileid) { - if (LF_ISSET(WT_LOGSCAN_RECOVER)) - WT_RET_MSG(session, WT_NOTFOUND, - "__wt_log_scan LSN %" PRIu32 "/%" - PRIu32 - " larger than biggest log file %" - PRIu32, lsnp->l.file, - lsnp->l.offset, log->fileid); - else - return (WT_NOTFOUND); - } - - /* - * Log cursors may not know the starting LSN. If an - * LSN is passed in that it is equal to the smallest - * LSN, start from the beginning of the log. - */ - start_lsn = *lsnp; - if (WT_IS_INIT_LSN(&start_lsn)) - start_lsn = log->first_lsn; } - end_lsn = log->alloc_lsn; + lastlog = log->fileid; } else { /* * If logging is not configured, we can still print out the log @@ -1732,7 +1952,6 @@ __wt_log_scan(WT_SESSION_IMPL *session, WT_LSN *lsnp, uint32_t flags, * a multiple of this. */ allocsize = WT_LOG_ALIGN; - lastlog = 0; firstlog = UINT32_MAX; WT_RET(__log_get_files(session, WT_LOG_FILENAME, &logfiles, &logcount)); @@ -1749,8 +1968,49 @@ __wt_log_scan(WT_SESSION_IMPL *session, WT_LSN *lsnp, uint32_t flags, WT_ERR( __wt_fs_directory_list_free(session, &logfiles, logcount)); } - WT_ERR(__log_openfile(session, - &log_fh, WT_LOG_FILENAME, start_lsn.l.file, WT_LOG_OPEN_VERIFY)); + if (lsnp != NULL) { + /* + * Offsets must be on allocation boundaries. + * An invalid LSN from a user should just return + * WT_NOTFOUND. It is not an error. But if it is + * from recovery, we expect valid LSNs so give more + * information about that. + */ + if (lsnp->l.offset % allocsize != 0) { + if (LF_ISSET(WT_LOGSCAN_RECOVER)) + WT_ERR_MSG(session, WT_NOTFOUND, + "__wt_log_scan unaligned LSN %" + PRIu32 "/%" PRIu32, + lsnp->l.file, lsnp->l.offset); + else + return (WT_NOTFOUND); + } + /* + * If the file is in the future it doesn't exist. + * An invalid LSN from a user should just return + * WT_NOTFOUND. It is not an error. But if it is + * from recovery, we expect valid LSNs so give more + * information about that. + */ + if (lsnp->l.file > lastlog) { + if (LF_ISSET(WT_LOGSCAN_RECOVER)) + WT_ERR_MSG(session, WT_NOTFOUND, + "__wt_log_scan LSN %" PRIu32 "/%" PRIu32 + " larger than biggest log file %" PRIu32, + lsnp->l.file, lsnp->l.offset, lastlog); + else + return (WT_NOTFOUND); + } + /* + * Log cursors may not know the starting LSN. If an + * LSN is passed in that it is equal to the smallest + * LSN, start from the beginning of the log. + */ + if (!WT_IS_INIT_LSN(lsnp)) + start_lsn = *lsnp; + } + WT_ERR(__log_open_verify(session, + start_lsn.l.file, &log_fh, &prev_lsn, NULL)); WT_ERR(__wt_filesize(session, log_fh, &log_size)); rd_lsn = start_lsn; if (LF_ISSET(WT_LOGSCAN_RECOVER)) @@ -1784,8 +2044,7 @@ advance: * Truncate this log file before we move to the next. */ if (LF_ISSET(WT_LOGSCAN_RECOVER)) - WT_ERR(__log_truncate(session, - &rd_lsn, WT_LOG_FILENAME, 1)); + WT_ERR(__log_truncate(session, &rd_lsn, true)); /* * If we had a partial record, we'll want to break * now after closing and truncating. Although for now @@ -1795,21 +2054,47 @@ advance: */ if (partial_record) break; - WT_SET_LSN(&rd_lsn, rd_lsn.l.file + 1, 0); /* * Avoid an error message when we reach end of log * by checking here. */ + prev_eof = rd_lsn; + WT_SET_LSN(&rd_lsn, rd_lsn.l.file + 1, 0); if (rd_lsn.l.file > end_lsn.l.file) break; - WT_ERR(__log_openfile(session, - &log_fh, WT_LOG_FILENAME, - rd_lsn.l.file, WT_LOG_OPEN_VERIFY)); if (LF_ISSET(WT_LOGSCAN_RECOVER)) __wt_verbose(session, WT_VERB_RECOVERY_PROGRESS, "Recovering log %" PRIu32 " through %" PRIu32, rd_lsn.l.file, end_lsn.l.file); + WT_ERR(__log_open_verify(session, + rd_lsn.l.file, &log_fh, &prev_lsn, &version)); + /* + * Opening the log file reads with verify sets up the + * previous LSN from the first record. This detects + * a "hole" at the end of the previous log file. + */ + if (LF_ISSET(WT_LOGSCAN_RECOVER) && + !WT_IS_INIT_LSN(&prev_lsn) && + !WT_IS_ZERO_LSN(&prev_lsn) && + prev_lsn.l.offset != prev_eof.l.offset) { + WT_ASSERT(session, + prev_eof.l.file == prev_lsn.l.file); + break; + } + /* + * If we read a current version log file without a + * previous LSN record the log ended after writing + * that header. We're done. + */ + if (LF_ISSET(WT_LOGSCAN_RECOVER) && + version == WT_LOG_VERSION_SYSTEM && + WT_IS_ZERO_LSN(&prev_lsn)) { + __wt_verbose(session, WT_VERB_LOG, + "log_scan: Stopping, no system " + "record detected in %s.", log_fh->name); + break; + } WT_ERR(__wt_filesize(session, log_fh, &log_size)); eol = false; continue; @@ -1881,12 +2166,7 @@ advance: */ buf->size = reclen; logrec = (WT_LOG_RECORD *)buf->mem; - checksum_tmp = logrec->checksum; - logrec->checksum = 0; - checksum_calculate = __wt_checksum(logrec, reclen); - logrec->checksum = checksum_tmp; - __wt_log_record_byteswap(logrec); - if (logrec->checksum != checksum_calculate) { + if (!__log_checksum_match(session, buf, reclen)) { /* * A checksum mismatch means we have reached the end of * the useful part of the log. This should be found on @@ -1904,6 +2184,7 @@ advance: ret = WT_NOTFOUND; break; } + __wt_log_record_byteswap(logrec); /* * We have a valid log record. If it is not the log file @@ -1946,8 +2227,12 @@ advance: /* Truncate if we're in recovery. */ if (LF_ISSET(WT_LOGSCAN_RECOVER) && - __wt_log_cmp(&rd_lsn, &log->trunc_lsn) < 0) - WT_ERR(__log_truncate(session, &rd_lsn, WT_LOG_FILENAME, 0)); + __wt_log_cmp(&rd_lsn, &log->trunc_lsn) < 0) { + __wt_verbose(session, WT_VERB_LOG, + "__wt_log_scan truncating to %" PRIu32 "/%" PRIu32, + rd_lsn.l.file, rd_lsn.l.offset); + WT_ERR(__log_truncate(session, &rd_lsn, false)); + } err: WT_STAT_CONN_INCR(session, log_scans); /* @@ -2212,7 +2497,7 @@ __log_write_internal(WT_SESSION_IMPL *session, WT_ITEM *record, WT_LSN *lsnp, F_ISSET(&myslot, WT_MYSLOT_UNBUFFERED) || force) ret = __wt_log_slot_switch(session, &myslot, true, false, NULL); if (ret == 0) - ret = __log_fill(session, &myslot, false, record, &lsn); + ret = __wt_log_fill(session, &myslot, false, record, &lsn); release_size = __wt_log_slot_release(&myslot, (int64_t)rdup_len); /* * If we get an error we still need to do proper accounting in @@ -2365,7 +2650,7 @@ __wt_log_flush(WT_SESSION_IMPL *session, uint32_t flags) * is single-threaded we could wait here forever because the write LSN * doesn't switch into the new file until it contains a record. */ - if (last_lsn.l.offset == WT_LOG_FIRST_RECORD) + if (last_lsn.l.offset == log->first_record) last_lsn = log->log_close_lsn; /* diff --git a/src/third_party/wiredtiger/src/log/log_slot.c b/src/third_party/wiredtiger/src/log/log_slot.c index 5bd3d53a973..e0b301398a3 100644 --- a/src/third_party/wiredtiger/src/log/log_slot.c +++ b/src/third_party/wiredtiger/src/log/log_slot.c @@ -31,7 +31,7 @@ __log_slot_dump(WT_SESSION_IMPL *session) earliest = i; __wt_errx(session, "Slot %d:", i); __wt_errx(session, " State: %" PRIx64 " Flags: %" PRIx32, - slot->slot_state, slot->flags); + (uint64_t)slot->slot_state, slot->flags); __wt_errx(session, " Start LSN: %" PRIu32 "/%" PRIu32, slot->slot_start_lsn.l.file, slot->slot_start_lsn.l.offset); __wt_errx(session, " End LSN: %" PRIu32 "/%" PRIu32, @@ -173,9 +173,9 @@ retry: if (WT_TIMEDIFF_SEC(now, begin) > 10) { __wt_errx(session, "SLOT_CLOSE: Slot %" PRIu32 " Timeout unbuffered, state 0x%" - PRIx64 " unbuffered %" PRIu64, + PRIx64 " unbuffered %" PRId64, (uint32_t)(slot - &log->slot_pool[0]), - slot->slot_state, + (uint64_t)slot->slot_state, slot->slot_unbuffered); __log_slot_dump(session); __wt_abort(session); @@ -238,6 +238,7 @@ __log_slot_new(WT_SESSION_IMPL *session) /* * Rotate among the slots to lessen collisions. */ + WT_RET(WT_SESSION_CHECK_PANIC(session)); for (i = 0, pool_i = log->pool_index; i < WT_SLOT_POOL; i++, pool_i++) { if (pool_i >= WT_SLOT_POOL) @@ -314,7 +315,7 @@ __log_slot_switch_internal( * because the slot could be part of an unbuffered operation. */ joined = WT_LOG_SLOT_JOINED(slot->slot_state); - if (joined == 0 && forced) { + if (joined == 0 && forced && !F_ISSET(log, WT_LOG_FORCE_NEWFILE)) { WT_STAT_CONN_INCR(session, log_force_write_skip); if (did_work != NULL) *did_work = false; @@ -393,6 +394,9 @@ __wt_log_slot_switch(WT_SESSION_IMPL *session, WT_STAT_CONN_INCR(session, log_slot_switch_busy); __wt_yield(); } + WT_RET(WT_SESSION_CHECK_PANIC(session)); + if (F_ISSET(S2C(session), WT_CONN_CLOSING)) + break; } while (F_ISSET(myslot, WT_MYSLOT_CLOSE) || (retry && ret == EBUSY)); return (ret); } diff --git a/src/third_party/wiredtiger/src/log/log_sys.c b/src/third_party/wiredtiger/src/log/log_sys.c new file mode 100644 index 00000000000..0f3cfdbc14b --- /dev/null +++ b/src/third_party/wiredtiger/src/log/log_sys.c @@ -0,0 +1,97 @@ +/*- + * Copyright (c) 2014-2017 MongoDB, Inc. + * Copyright (c) 2008-2014 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_log_system_record -- + * Write a system log record for the previous LSN. + */ +int +__wt_log_system_record( + WT_SESSION_IMPL *session, WT_FH *log_fh, WT_LSN *lsn) +{ + WT_DECL_ITEM(logrec_buf); + WT_DECL_RET; + WT_ITEM *dummy, empty; + WT_LOG *log; + WT_LOG_RECORD *logrec; + WT_LOGSLOT tmp; + WT_MYSLOT myslot; + const char *fmt = WT_UNCHECKED_STRING(IIIU); + uint32_t rectype = WT_LOGREC_SYSTEM; + size_t recsize; + + log = S2C(session)->log; + WT_RET(__wt_logrec_alloc(session, log->allocsize, &logrec_buf)); + memset((uint8_t *)logrec_buf->mem, 0, log->allocsize); + WT_CLEAR(empty); + dummy = ∅ + /* + * There is currently an unused portion of the system record for + * future use. Send in a NULL entry. + */ + WT_ERR(__wt_struct_size(session, &recsize, fmt, rectype, + lsn->l.file, lsn->l.offset, dummy)); + WT_ASSERT(session, recsize <= log->allocsize); + WT_ERR(__wt_struct_pack(session, + (uint8_t *)logrec_buf->data + logrec_buf->size, recsize, fmt, + rectype, lsn->l.file, lsn->l.offset, dummy)); + logrec = (WT_LOG_RECORD *)logrec_buf->mem; + /* + * We know system records are this size. And we have to adjust + * the size now because we're not going through the normal log + * write path and the packing functions needed the correct offset + * earlier. + */ + logrec_buf->size = logrec->len = log->allocsize; + /* + * We do not compress nor encrypt this record. + */ + logrec->checksum = 0; + logrec->flags = 0; + __wt_log_record_byteswap(logrec); + logrec->checksum = __wt_checksum(logrec, log->allocsize); +#ifdef WORDS_BIGENDIAN + logrec->checksum = __wt_bswap32(logrec->checksum); +#endif + WT_CLEAR(tmp); + memset(&myslot, 0, sizeof(myslot)); + myslot.slot = &tmp; + __wt_log_slot_activate(session, &tmp); + /* + * Override the file handle to the one we're using. + */ + tmp.slot_fh = log_fh; + WT_ERR(__wt_log_fill(session, &myslot, true, logrec_buf, NULL)); +err: __wt_logrec_free(session, &logrec_buf); + return (ret); +} + +/* + * __wt_log_recover_system -- + * Process a system log record for the previous LSN in recovery. + */ +int +__wt_log_recover_system(WT_SESSION_IMPL *session, + const uint8_t **pp, const uint8_t *end, WT_LSN *lsnp) +{ + WT_DECL_RET; + WT_ITEM unused; + uint32_t prev_file, prev_offset; + const char *fmt = WT_UNCHECKED_STRING(IIU); + + if ((ret = __wt_struct_unpack(session, *pp, WT_PTRDIFF(end, *pp), fmt, + &prev_file, &prev_offset, &unused)) != 0) + WT_RET_MSG(session, ret, + "log_recover_prevlsn: unpack failure"); + if (lsnp != NULL) + WT_SET_LSN(lsnp, prev_file, prev_offset); + *pp = end; + return (0); +} diff --git a/src/third_party/wiredtiger/src/lsm/lsm_cursor.c b/src/third_party/wiredtiger/src/lsm/lsm_cursor.c index 06dfec725bb..6a1709b03f2 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_cursor.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_cursor.c @@ -1775,9 +1775,7 @@ __wt_clsm_open(WT_SESSION_IMPL *session, if (!WT_PREFIX_MATCH(uri, "lsm:")) return (__wt_unexpected_object_type(session, uri, "lsm:")); - if (F_ISSET(S2C(session), WT_CONN_IN_MEMORY)) - WT_RET_MSG(session, EINVAL, - "LSM trees not supported by in-memory configurations"); + WT_RET(__wt_inmem_unsupported_op(session, "LSM trees")); WT_RET(__wt_config_gets_def(session, cfg, "checkpoint", 0, &cval)); if (cval.len != 0) diff --git a/src/third_party/wiredtiger/src/lsm/lsm_merge.c b/src/third_party/wiredtiger/src/lsm/lsm_merge.c index 28e954c6f6a..f3990a72d95 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_merge.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_merge.c @@ -621,7 +621,7 @@ err: if (locked) if (ret == EINTR) __wt_verbose(session, WT_VERB_LSM, - "Merge aborted due to close"); + "%s", "Merge aborted due to close"); else __wt_verbose(session, WT_VERB_LSM, "Merge failed with %s", diff --git a/src/third_party/wiredtiger/src/lsm/lsm_tree.c b/src/third_party/wiredtiger/src/lsm/lsm_tree.c index ebdd1e492cc..18e1f6d3115 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_tree.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_tree.c @@ -15,6 +15,42 @@ static int __lsm_tree_open( static int __lsm_tree_set_name(WT_SESSION_IMPL *, WT_LSM_TREE *, const char *); /* + * __lsm_tree_discard_state -- + * Free the metadata configuration state related LSM tree pointers. + */ +static void +__lsm_tree_discard_state(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) +{ + WT_LSM_CHUNK *chunk; + u_int i; + + __wt_free(session, lsm_tree->config); + __wt_free(session, lsm_tree->key_format); + __wt_free(session, lsm_tree->value_format); + __wt_free(session, lsm_tree->collator_name); + __wt_free(session, lsm_tree->bloom_config); + __wt_free(session, lsm_tree->file_config); + + for (i = 0; i < lsm_tree->nchunks; i++) { + if ((chunk = lsm_tree->chunk[i]) == NULL) + continue; + + __wt_free(session, chunk->bloom_uri); + __wt_free(session, chunk->uri); + __wt_free(session, chunk); + } + + for (i = 0; i < lsm_tree->nold_chunks; i++) { + chunk = lsm_tree->old_chunks[i]; + WT_ASSERT(session, chunk != NULL); + + __wt_free(session, chunk->bloom_uri); + __wt_free(session, chunk->uri); + __wt_free(session, chunk); + } +} + +/* * __lsm_tree_discard -- * Free an LSM tree structure. */ @@ -22,8 +58,6 @@ static int __lsm_tree_discard(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, bool final) { WT_DECL_RET; - WT_LSM_CHUNK *chunk; - u_int i; WT_UNUSED(final); /* Only used in diagnostic builds */ @@ -48,34 +82,12 @@ __lsm_tree_discard(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, bool final) lsm_tree->collator, &session->iface)); __wt_free(session, lsm_tree->name); - __wt_free(session, lsm_tree->config); - __wt_free(session, lsm_tree->key_format); - __wt_free(session, lsm_tree->value_format); - __wt_free(session, lsm_tree->collator_name); - __wt_free(session, lsm_tree->bloom_config); - __wt_free(session, lsm_tree->file_config); - - __wt_rwlock_destroy(session, &lsm_tree->rwlock); - - for (i = 0; i < lsm_tree->nchunks; i++) { - if ((chunk = lsm_tree->chunk[i]) == NULL) - continue; - - __wt_free(session, chunk->bloom_uri); - __wt_free(session, chunk->uri); - __wt_free(session, chunk); - } + __lsm_tree_discard_state(session, lsm_tree); __wt_free(session, lsm_tree->chunk); + __wt_free(session, lsm_tree->old_chunks); - for (i = 0; i < lsm_tree->nold_chunks; i++) { - chunk = lsm_tree->old_chunks[i]; - WT_ASSERT(session, chunk != NULL); + __wt_rwlock_destroy(session, &lsm_tree->rwlock); - __wt_free(session, chunk->bloom_uri); - __wt_free(session, chunk->uri); - __wt_free(session, chunk); - } - __wt_free(session, lsm_tree->old_chunks); __wt_free(session, lsm_tree); return (ret); @@ -851,45 +863,6 @@ __wt_lsm_tree_retire_chunks(WT_SESSION_IMPL *session, } /* - * __wt_lsm_tree_alter -- - * Alter an LSM tree. - */ -int -__wt_lsm_tree_alter( - WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) -{ - WT_DECL_RET; - WT_LSM_CHUNK *chunk; - WT_LSM_TREE *lsm_tree; - u_int i; - bool locked; - - locked = false; - - /* Get the LSM tree. */ - WT_RET(__wt_lsm_tree_get(session, uri, false, &lsm_tree)); - - /* Prevent any new opens. */ - __wt_lsm_tree_writelock(session, lsm_tree); - locked = true; - - /* Alter the chunks. */ - for (i = 0; i < lsm_tree->nchunks; i++) { - chunk = lsm_tree->chunk[i]; - WT_ERR(__wt_schema_alter(session, chunk->uri, cfg)); - if (F_ISSET(chunk, WT_LSM_CHUNK_BLOOM)) - WT_ERR( - __wt_schema_alter(session, chunk->bloom_uri, cfg)); - } - WT_ERR(__wt_lsm_meta_write(session, lsm_tree, cfg[0])); - -err: if (locked) - __wt_lsm_tree_writeunlock(session, lsm_tree); - __wt_lsm_tree_release(session, lsm_tree); - return (ret); -} - -/* * __wt_lsm_tree_drop -- * Drop an LSM tree. */ @@ -1385,6 +1358,24 @@ __wt_lsm_tree_worker(WT_SESSION_IMPL *session, WT_ERR(__wt_schema_worker(session, chunk->bloom_uri, file_func, name_func, cfg, open_flags)); } + /* + * If this was an alter operation, we need to alter the configuration + * for the overall tree and then reread it so it isn't out of date. + * Reread it here so that we update the configuration of the + * current tree's structure to any new, altered values. + */ + if (FLD_ISSET(open_flags, WT_BTREE_ALTER)) { + WT_ERR(__wt_lsm_meta_write(session, lsm_tree, cfg[0])); + /* + * We're about to read in the new configuration that + * we just wrote. Free the old ones. + */ + __lsm_tree_discard_state(session, lsm_tree); + if ((ret = __wt_lsm_meta_read(session, lsm_tree)) != 0) + WT_PANIC_ERR(session, ret, + "Failed to read updated LSM configuration"); + } + err: if (locked) { if (exclusive) __wt_lsm_tree_writeunlock(session, lsm_tree); diff --git a/src/third_party/wiredtiger/src/meta/meta_ckpt.c b/src/third_party/wiredtiger/src/meta/meta_ckpt.c index 0e96c4ee6ca..24f991cf549 100644 --- a/src/third_party/wiredtiger/src/meta/meta_ckpt.c +++ b/src/third_party/wiredtiger/src/meta/meta_ckpt.c @@ -429,7 +429,7 @@ __wt_meta_ckptlist_set(WT_SESSION_IMPL *session, } if (strcmp(ckpt->name, WT_CHECKPOINT) == 0) WT_ERR(__wt_buf_catfmt(session, buf, - "%s%s.%" PRId64 "=(addr=\"%.*s\",order=%" PRIu64 + "%s%s.%" PRId64 "=(addr=\"%.*s\",order=%" PRId64 ",time=%" PRIuMAX ",size=%" PRIu64 ",write_gen=%" PRIu64 ")", sep, ckpt->name, ckpt->order, @@ -438,7 +438,7 @@ __wt_meta_ckptlist_set(WT_SESSION_IMPL *session, ckpt->write_gen)); else WT_ERR(__wt_buf_catfmt(session, buf, - "%s%s=(addr=\"%.*s\",order=%" PRIu64 + "%s%s=(addr=\"%.*s\",order=%" PRId64 ",time=%" PRIuMAX ",size=%" PRIu64 ",write_gen=%" PRIu64 ")", sep, ckpt->name, diff --git a/src/third_party/wiredtiger/src/os_win/os_map.c b/src/third_party/wiredtiger/src/os_win/os_map.c index c0aa6dac28f..25835d2615d 100644 --- a/src/third_party/wiredtiger/src/os_win/os_map.c +++ b/src/third_party/wiredtiger/src/os_win/os_map.c @@ -82,6 +82,8 @@ __wt_win_unmap(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session, WT_FILE_HANDLE_WIN *win_fh; WT_SESSION_IMPL *session; + WT_UNUSED(length); /* !HAVE_VERBOSE */ + win_fh = (WT_FILE_HANDLE_WIN *)file_handle; session = (WT_SESSION_IMPL *)wt_session; diff --git a/src/third_party/wiredtiger/src/reconcile/rec_track.c b/src/third_party/wiredtiger/src/reconcile/rec_track.c index a431465661f..ee97d684529 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_track.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_track.c @@ -16,11 +16,11 @@ (sizeof(s) + 2 * sizeof(void *) + (p)->addr_size + (p)->value_size) /* - * __ovfl_track_init -- + * __wt_ovfl_track_init -- * Initialize the overflow tracking structure. */ -static int -__ovfl_track_init(WT_SESSION_IMPL *session, WT_PAGE *page) +int +__wt_ovfl_track_init(WT_SESSION_IMPL *session, WT_PAGE *page) { return (__wt_calloc_one(session, &page->modify->ovfl_track)); } @@ -36,6 +36,9 @@ __ovfl_discard_verbose( WT_CELL_UNPACK *unpack, _unpack; WT_DECL_ITEM(tmp); + WT_UNUSED(page); /* !HAVE_VERBOSE */ + WT_UNUSED(tag); /* !HAVE_VERBOSE */ + WT_RET(__wt_scr_alloc(session, 512, &tmp)); unpack = &_unpack; @@ -128,7 +131,7 @@ __wt_ovfl_discard_add(WT_SESSION_IMPL *session, WT_PAGE *page, WT_CELL *cell) WT_OVFL_TRACK *track; if (page->modify->ovfl_track == NULL) - WT_RET(__ovfl_track_init(session, page)); + WT_RET(__wt_ovfl_track_init(session, page)); track = page->modify->ovfl_track; WT_RET(__wt_realloc_def(session, &track->discard_allocated, @@ -169,6 +172,10 @@ __ovfl_reuse_verbose(WT_SESSION_IMPL *session, { WT_DECL_ITEM(tmp); + WT_UNUSED(page); /* !HAVE_VERBOSE */ + WT_UNUSED(reuse); /* !HAVE_VERBOSE */ + WT_UNUSED(tag); /* !HAVE_VERBOSE */ + WT_RET(__wt_scr_alloc(session, 64, &tmp)); __wt_verbose(session, WT_VERB_OVERFLOW, @@ -182,7 +189,8 @@ __ovfl_reuse_verbose(WT_SESSION_IMPL *session, F_ISSET(reuse, WT_OVFL_REUSE_INUSE) && F_ISSET(reuse, WT_OVFL_REUSE_JUST_ADDED) ? ", " : "", F_ISSET(reuse, WT_OVFL_REUSE_JUST_ADDED) ? "just-added" : "", - WT_MIN(reuse->value_size, 40), (char *)WT_OVFL_REUSE_VALUE(reuse)); + (int)WT_MIN(reuse->value_size, 40), + (char *)WT_OVFL_REUSE_VALUE(reuse)); __wt_scr_free(session, &tmp); return (0); @@ -487,7 +495,7 @@ __wt_ovfl_reuse_add(WT_SESSION_IMPL *session, WT_PAGE *page, uint8_t *p; if (page->modify->ovfl_track == NULL) - WT_RET(__ovfl_track_init(session, page)); + WT_RET(__wt_ovfl_track_init(session, page)); head = page->modify->ovfl_track->ovfl_reuse; @@ -557,311 +565,12 @@ __wt_ovfl_reuse_free(WT_SESSION_IMPL *session, WT_PAGE *page) } /* - * __ovfl_txnc_verbose -- - * Dump information about a transaction-cached overflow record. - */ -static int -__ovfl_txnc_verbose(WT_SESSION_IMPL *session, - WT_PAGE *page, WT_OVFL_TXNC *txnc, const char *tag) -{ - WT_DECL_ITEM(tmp); - - WT_RET(__wt_scr_alloc(session, 64, &tmp)); - - __wt_verbose(session, WT_VERB_OVERFLOW, - "txn-cache: %s%s%p %s %" PRIu64 " {%.*s}", - tag == NULL ? "" : tag, - tag == NULL ? "" : ": ", - (void *)page, - __wt_addr_string( - session, WT_OVFL_TXNC_ADDR(txnc), txnc->addr_size, tmp), - txnc->current, - WT_MIN(txnc->value_size, 40), (char *)WT_OVFL_TXNC_VALUE(txnc)); - - __wt_scr_free(session, &tmp); - return (0); -} - -#if 0 -/* - * __ovfl_txnc_dump -- - * Debugging information. - */ -static void -__ovfl_txnc_dump(WT_SESSION_IMPL *session, WT_PAGE *page) -{ - WT_OVFL_TXNC **head, *txnc; - - if (page->modify == NULL || page->modify->ovfl_track == NULL) - return; - head = page->modify->ovfl_track->ovfl_txnc; - - for (txnc = head[0]; txnc != NULL; txnc = txnc->next[0]) - (void)__ovfl_txnc_verbose(session, page, txnc, "dump"); -} -#endif - -/* - * __ovfl_txnc_skip_search -- - * Return the first matching addr in the overflow transaction-cache list. - */ -static WT_OVFL_TXNC * -__ovfl_txnc_skip_search(WT_OVFL_TXNC **head, const void *addr, size_t addr_size) -{ - WT_OVFL_TXNC **e; - size_t len; - int cmp, i; - - /* - * Start at the highest skip level, then go as far as possible at each - * level before stepping down to the next. - */ - for (i = WT_SKIP_MAXDEPTH - 1, e = &head[i]; i >= 0;) { - if (*e == NULL) { /* Empty levels */ - --i; - --e; - continue; - } - - /* - * Return any exact matches: we don't care in what search level - * we found a match. - */ - len = WT_MIN((*e)->addr_size, addr_size); - cmp = memcmp(WT_OVFL_TXNC_ADDR(*e), addr, len); - if (cmp == 0 && (*e)->addr_size == addr_size) - return (*e); - - /* - * If the skiplist address is larger than the search address, or - * they compare equally and the skiplist address is longer than - * the search address, drop down a level, otherwise continue on - * this level. - */ - if (cmp > 0 || (cmp == 0 && (*e)->addr_size > addr_size)) { - --i; /* Drop down a level */ - --e; - } else /* Keep going at this level */ - e = &(*e)->next[i]; - } - return (NULL); -} - -/* - * __ovfl_txnc_skip_search_stack -- - * Search an overflow transaction-cache skiplist, returning an - * insert/remove stack. - */ -static void -__ovfl_txnc_skip_search_stack(WT_OVFL_TXNC **head, - WT_OVFL_TXNC ***stack, const void *addr, size_t addr_size) -{ - WT_OVFL_TXNC **e; - size_t len; - int cmp, i; - - /* - * Start at the highest skip level, then go as far as possible at each - * level before stepping down to the next. - */ - for (i = WT_SKIP_MAXDEPTH - 1, e = &head[i]; i >= 0;) { - if (*e == NULL) { /* Empty levels */ - stack[i--] = e--; - continue; - } - - /* - * If the skiplist addr is larger than the search addr, or - * they compare equally and the skiplist addr is longer than - * the search addr, drop down a level, otherwise continue on - * this level. - */ - len = WT_MIN((*e)->addr_size, addr_size); - cmp = memcmp(WT_OVFL_TXNC_ADDR(*e), addr, len); - if (cmp > 0 || (cmp == 0 && (*e)->addr_size > addr_size)) - stack[i--] = e--; /* Drop down a level */ - else - e = &(*e)->next[i]; /* Keep going at this level */ - } -} - -/* - * __ovfl_txnc_wrapup -- - * Resolve the page's transaction-cache list. - */ -static int -__ovfl_txnc_wrapup(WT_SESSION_IMPL *session, WT_PAGE *page) -{ - WT_OVFL_TXNC **e, **head, *txnc; - uint64_t oldest_txn; - size_t decr; - int i; - - head = page->modify->ovfl_track->ovfl_txnc; - - /* - * Take a snapshot of the oldest transaction ID we need to keep alive. - * Since we do two passes through entries in the structure, the normal - * visibility check could give different results as the global ID moves - * forward. - */ - oldest_txn = __wt_txn_oldest_id(session); - - /* - * Discard any transaction-cache records with transaction IDs earlier - * than any in the system. - * - * First, walk the overflow transaction-cache skip lists (except for - * the lowest level), fixing up links. - */ - for (i = WT_SKIP_MAXDEPTH - 1; i > 0; --i) - for (e = &head[i]; (txnc = *e) != NULL;) { - if (WT_TXNID_LE(oldest_txn, txnc->current)) { - e = &txnc->next[i]; - continue; - } - *e = txnc->next[i]; - } - - /* Second, discard any no longer needed transaction-cache records. */ - decr = 0; - for (e = &head[0]; (txnc = *e) != NULL;) { - if (WT_TXNID_LE(oldest_txn, txnc->current)) { - e = &txnc->next[0]; - continue; - } - *e = txnc->next[0]; - - if (WT_VERBOSE_ISSET(session, WT_VERB_OVERFLOW)) - WT_RET( - __ovfl_txnc_verbose(session, page, txnc, "free")); - - decr += WT_OVFL_SIZE(txnc, WT_OVFL_TXNC); - __wt_free(session, txnc); - } - - if (decr != 0) - __wt_cache_page_inmem_decr(session, page, decr); - return (0); -} - -/* - * __wt_ovfl_txnc_search -- - * Search the page's list of transaction-cache overflow records for a - * match. - */ -int -__wt_ovfl_txnc_search( - WT_PAGE *page, const uint8_t *addr, size_t addr_size, WT_ITEM *store) -{ - WT_OVFL_TXNC **head, *txnc; - - if (page->modify->ovfl_track == NULL) - return (WT_NOTFOUND); - - head = page->modify->ovfl_track->ovfl_txnc; - - if ((txnc = __ovfl_txnc_skip_search(head, addr, addr_size)) == NULL) - return (WT_NOTFOUND); - - store->data = WT_OVFL_TXNC_VALUE(txnc); - store->size = txnc->value_size; - return (0); -} - -/* - * __wt_ovfl_txnc_add -- - * Add a new entry to the page's list of transaction-cached overflow - * records. - */ -int -__wt_ovfl_txnc_add(WT_SESSION_IMPL *session, WT_PAGE *page, - const uint8_t *addr, size_t addr_size, - const void *value, size_t value_size) -{ - WT_OVFL_TXNC **head, **stack[WT_SKIP_MAXDEPTH], *txnc; - size_t size; - u_int i, skipdepth; - uint8_t *p; - - if (page->modify->ovfl_track == NULL) - WT_RET(__ovfl_track_init(session, page)); - - head = page->modify->ovfl_track->ovfl_txnc; - - /* Choose a skiplist depth for this insert. */ - skipdepth = __wt_skip_choose_depth(session); - - /* - * Allocate the WT_OVFL_TXNC structure, next pointers for the skip - * list, room for the address and value, then copy everything into - * place. - * - * To minimize the WT_OVFL_TXNC structure size, the address offset - * and size are single bytes: that's safe because the address follows - * the structure (which can't be more than about 100B), and address - * cookies are limited to 255B. - */ - size = sizeof(WT_OVFL_TXNC) + - skipdepth * sizeof(WT_OVFL_TXNC *) + addr_size + value_size; - WT_RET(__wt_calloc(session, 1, size, &txnc)); - p = (uint8_t *)txnc + - sizeof(WT_OVFL_TXNC) + skipdepth * sizeof(WT_OVFL_TXNC *); - txnc->addr_offset = (uint8_t)WT_PTRDIFF(p, txnc); - txnc->addr_size = (uint8_t)addr_size; - memcpy(p, addr, addr_size); - p += addr_size; - txnc->value_offset = WT_PTRDIFF32(p, txnc); - txnc->value_size = WT_STORE_SIZE(value_size); - memcpy(p, value, value_size); - txnc->current = __wt_txn_id_alloc(session, false); - - __wt_cache_page_inmem_incr( - session, page, WT_OVFL_SIZE(txnc, WT_OVFL_TXNC)); - - /* Insert the new entry into the skiplist. */ - __ovfl_txnc_skip_search_stack(head, stack, addr, addr_size); - for (i = 0; i < skipdepth; ++i) { - txnc->next[i] = *stack[i]; - *stack[i] = txnc; - } - - if (WT_VERBOSE_ISSET(session, WT_VERB_OVERFLOW)) - WT_RET(__ovfl_txnc_verbose(session, page, txnc, "add")); - - return (0); -} - -/* - * __wt_ovfl_txnc_free -- - * Free the page's list of transaction-cached overflow records. - */ -void -__wt_ovfl_txnc_free(WT_SESSION_IMPL *session, WT_PAGE *page) -{ - WT_OVFL_TXNC *txnc; - WT_PAGE_MODIFY *mod; - void *next; - - mod = page->modify; - if (mod == NULL || mod->ovfl_track == NULL) - return; - - for (txnc = mod->ovfl_track->ovfl_txnc[0]; - txnc != NULL; txnc = next) { - next = txnc->next[0]; - __wt_free(session, txnc); - } -} - -/* * __wt_ovfl_track_wrapup -- * Resolve the page's overflow tracking on reconciliation success. */ int __wt_ovfl_track_wrapup(WT_SESSION_IMPL *session, WT_PAGE *page) { - WT_DECL_RET; WT_OVFL_TRACK *track; if (page->modify == NULL || page->modify->ovfl_track == NULL) @@ -874,12 +583,7 @@ __wt_ovfl_track_wrapup(WT_SESSION_IMPL *session, WT_PAGE *page) if (track->ovfl_reuse[0] != NULL) WT_RET(__ovfl_reuse_wrapup(session, page)); - if (track->ovfl_txnc[0] != NULL) { - __wt_writelock(session, &S2BT(session)->ovfl_lock); - ret = __ovfl_txnc_wrapup(session, page); - __wt_writeunlock(session, &S2BT(session)->ovfl_lock); - } - return (ret); + return (0); } /* @@ -889,7 +593,6 @@ __wt_ovfl_track_wrapup(WT_SESSION_IMPL *session, WT_PAGE *page) int __wt_ovfl_track_wrapup_err(WT_SESSION_IMPL *session, WT_PAGE *page) { - WT_DECL_RET; WT_OVFL_TRACK *track; if (page->modify == NULL || page->modify->ovfl_track == NULL) @@ -902,10 +605,5 @@ __wt_ovfl_track_wrapup_err(WT_SESSION_IMPL *session, WT_PAGE *page) if (track->ovfl_reuse[0] != NULL) WT_RET(__ovfl_reuse_wrapup_err(session, page)); - if (track->ovfl_txnc[0] != NULL) { - __wt_writelock(session, &S2BT(session)->ovfl_lock); - ret = __ovfl_txnc_wrapup(session, page); - __wt_writeunlock(session, &S2BT(session)->ovfl_lock); - } - return (ret); + return (0); } diff --git a/src/third_party/wiredtiger/src/reconcile/rec_write.c b/src/third_party/wiredtiger/src/reconcile/rec_write.c index a304dc54340..970f760f4ca 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_write.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_write.c @@ -1179,7 +1179,7 @@ __rec_txn_read(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_DECL_TIMESTAMP(max_timestamp) WT_PAGE *page; WT_UPDATE *append, *upd, *upd_list; - size_t notused, update_mem; + size_t size, update_mem; uint64_t max_txn, min_txn, txnid; bool append_origv, skipped; @@ -1204,8 +1204,8 @@ __rec_txn_read(WT_SESSION_IMPL *session, WT_RECONCILE *r, update_mem = 0; max_txn = WT_TXN_NONE; #ifdef HAVE_TIMESTAMPS - __wt_timestamp_set(max_timestamp, zero_timestamp); - memset(min_timestamp, 0xff, WT_TIMESTAMP_SIZE); + __wt_timestamp_set_zero(max_timestamp); + __wt_timestamp_set_inf(min_timestamp); #endif min_txn = UINT64_MAX; @@ -1419,45 +1419,6 @@ __rec_txn_read(WT_SESSION_IMPL *session, WT_RECONCILE *r, /* The page can't be marked clean. */ r->leave_dirty = true; - - /* - * A special-case for overflow values, where we can't write the - * original on-page value item to disk because it's been updated - * or removed. - * - * What happens is that an overflow value is updated or removed - * and its backing blocks freed. If any reader in the system - * might still want the value, a copy was cached in the page - * reconciliation tracking memory, and the page cell set to - * WT_CELL_VALUE_OVFL_RM. Eviction then chose the page and - * we're splitting it up in order to push parts of it out of - * memory. - * - * We could write the original on-page value item to disk... if - * we had a copy. The cache may not have a copy (a globally - * visible update would have kept a value from being cached), or - * an update that subsequently became globally visible could - * cause a cached value to be discarded. Either way, once there - * is a globally visible update, we may not have the original - * value. - * - * Fortunately, if there's a globally visible update we don't - * care about the original version, so we simply ignore it, no - * transaction can ever try and read it. If there isn't a - * globally visible update, there had better be a cached value. - * - * In the latter case, we could write the value out to disk, but - * (1) we are planning on re-instantiating this page in memory, - * it isn't going to disk, and (2) the value item is eventually - * going to be discarded, that seems like a waste of a write. - * Instead, find the cached value and append it to the update - * list we're saving for later restoration. - */ - if (vpack != NULL && - vpack->raw == WT_CELL_VALUE_OVFL_RM && - !__wt_txn_visible_all( - session, min_txn, WT_TIMESTAMP(min_timestamp))) - append_origv = true; } else { /* * The lookaside table eviction path. @@ -1482,22 +1443,23 @@ __rec_txn_read(WT_SESSION_IMPL *session, WT_RECONCILE *r, * key/value pair which simply doesn't exist for some reader; * place a deleted record at the end of the update list. */ + size = 0; /* -Wconditional-uninitialized */ if (vpack == NULL || vpack->type == WT_CELL_DEL) WT_RET(__wt_update_alloc(session, - NULL, &append, ¬used, WT_UPDATE_DELETED)); + NULL, &append, &size, WT_UPDATE_DELETED)); else { WT_RET(__wt_scr_alloc(session, 0, &tmp)); if ((ret = __wt_page_cell_data_ref( session, page, vpack, tmp)) == 0) ret = __wt_update_alloc(session, - tmp, &append, ¬used, WT_UPDATE_STANDARD); + tmp, &append, &size, WT_UPDATE_STANDARD); __wt_scr_free(session, &tmp); WT_RET(ret); } /* - * Give the entry an impossibly low transaction ID to ensure its - * global visibility, append it to the update list. + * Give the entry no transaction ID to ensure global visibility, + * append it to the update list. * * Note the change to the actual reader-accessible update list: * from now on, the original on-page value appears at the end @@ -1508,8 +1470,7 @@ __rec_txn_read(WT_SESSION_IMPL *session, WT_RECONCILE *r, for (upd = upd_list; upd->next != NULL; upd = upd->next) ; WT_PUBLISH(upd->next, append); - __wt_cache_page_inmem_incr( - session, page, WT_UPDATE_MEMSIZE(append)); + __wt_cache_page_inmem_incr(session, page, size); } /* @@ -4752,29 +4713,25 @@ record_loop: /* deleted = false; /* - * If doing update save and restore, there's an - * update that's not globally visible, and the + * If doing an update save and restore, and the * underlying value is a removed overflow value, * we end up here. * - * When the update save/restore code noticed the - * removed overflow value, it appended a copy of - * the cached, original overflow value to the - * update list being saved (ensuring the on-page - * item will never be accessed after the page is - * re-instantiated), then returned a NULL update - * to us. + * If necessary, when the overflow value was + * originally removed, reconciliation appended + * a globally visible copy of the value to the + * key's update list, meaning the on-page item + * isn't accessed after page re-instantiation. * - * Assert the case: if we remove an underlying - * overflow object, checkpoint reconciliation - * should never see it again, there should be a - * visible update in the way. - * - * Write a placeholder. + * Assert the case. */ WT_ASSERT(session, F_ISSET(r, WT_EVICT_UPDATE_RESTORE)); + /* + * The on-page value will never be accessed, + * write a placeholder record. + */ data = "@"; size = 1; } else { @@ -4905,16 +4862,12 @@ compare: /* } /* - * If we had a reference to an overflow record we never used, + * The first time we find an overflow record we never used, * discard the underlying blocks, they're no longer useful. - * - * One complication: we must cache a copy before discarding the - * on-disk version if there's a transaction in the system that - * might read the original value. */ if (ovfl_state == OVFL_UNUSED && vpack->raw != WT_CELL_VALUE_OVFL_RM) - WT_ERR(__wt_ovfl_cache(session, page, upd, vpack)); + WT_ERR(__wt_ovfl_remove(session, page, upd, vpack)); } /* Walk any append list. */ @@ -5079,7 +5032,7 @@ __rec_row_int(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) cell = NULL; key_onpage_ovfl = false; - WT_RET(__rec_split_init(session, r, page, 0ULL, btree->maxintlpage)); + WT_RET(__rec_split_init(session, r, page, 0, btree->maxintlpage)); /* * Ideally, we'd never store the 0th key on row-store internal pages @@ -5361,7 +5314,7 @@ __rec_row_leaf(WT_SESSION_IMPL *session, key = &r->k; val = &r->v; - WT_RET(__rec_split_init(session, r, page, 0ULL, btree->maxleafpage)); + WT_RET(__rec_split_init(session, r, page, 0, btree->maxleafpage)); /* * Write any K/V pairs inserted into the page before the first from-disk @@ -5451,18 +5404,15 @@ __rec_row_leaf(WT_SESSION_IMPL *session, dictionary = true; } else if (vpack->raw == WT_CELL_VALUE_OVFL_RM) { /* - * If doing update save and restore in service - * of eviction, there's an update that's not - * globally visible, and the underlying value - * is a removed overflow value, we end up here. + * If doing an update save and restore, and the + * underlying value is a removed overflow value, + * we end up here. * - * When the update save/restore code noticed the - * removed overflow value, it appended a copy of - * the cached, original overflow value to the - * update list being saved (ensuring any on-page - * item will never be accessed after the page is - * re-instantiated), then returned a NULL update - * to us. + * If necessary, when the overflow value was + * originally removed, reconciliation appended + * a globally visible copy of the value to the + * key's update list, meaning the on-page item + * isn't accessed after page re-instantiation. * * Assert the case. */ @@ -5508,16 +5458,13 @@ __rec_row_leaf(WT_SESSION_IMPL *session, } } else { /* - * If the original value was an overflow and we've not - * already done so, discard it. One complication: we - * must cache a copy before discarding the on-disk - * version if there's a transaction in the system that - * might read the original value. + * The first time we find an overflow record we're not + * going to use, discard the underlying blocks. */ if (vpack != NULL && vpack->ovfl && vpack->raw != WT_CELL_VALUE_OVFL_RM) - WT_ERR( - __wt_ovfl_cache(session, page, rip, vpack)); + WT_ERR(__wt_ovfl_remove( + session, page, upd, vpack)); /* If this key/value pair was deleted, we're done. */ if (upd->type == WT_UPDATE_DELETED) { diff --git a/src/third_party/wiredtiger/src/schema/schema_alter.c b/src/third_party/wiredtiger/src/schema/schema_alter.c index 346f09a1a64..7af4b86bc85 100644 --- a/src/third_party/wiredtiger/src/schema/schema_alter.c +++ b/src/third_party/wiredtiger/src/schema/schema_alter.c @@ -9,16 +9,24 @@ #include "wt_internal.h" /* - * __alter_file -- + * __wt_alter -- * Alter a file. */ -static int -__alter_file(WT_SESSION_IMPL *session, const char *uri, const char *newcfg[]) +int +__wt_alter(WT_SESSION_IMPL *session, const char *newcfg[]) { WT_DECL_RET; - const char *cfg[4], *filename; + const char *cfg[4], *filename, *uri; char *config, *newconfig; + uri = session->dhandle->name; + WT_RET(__wt_meta_track_on(session)); + + /* + * We know that we have exclusive access to the file. So it will be + * closed after we're done with it and the next open will see the + * updated metadata. + */ filename = uri; newconfig = NULL; if (!WT_PREFIX_SKIP(filename, "file:")) @@ -48,122 +56,6 @@ __alter_file(WT_SESSION_IMPL *session, const char *uri, const char *newcfg[]) err: __wt_free(session, config); __wt_free(session, newconfig); - return (ret); -} - -/* - * __alter_colgroup -- - * WT_SESSION::alter for a colgroup. - */ -static int -__alter_colgroup( - WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) -{ - WT_COLGROUP *colgroup; - WT_DECL_RET; - - WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_TABLE)); - - /* If we can get the colgroup, perform any potential alterations. */ - if ((ret = __wt_schema_get_colgroup( - session, uri, false, NULL, &colgroup)) == 0) - WT_TRET(__wt_schema_alter(session, colgroup->source, cfg)); - - return (ret); -} - -/* - * __alter_index -- - * WT_SESSION::alter for an index. - */ -static int -__alter_index( - WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) -{ - WT_INDEX *idx; - WT_DECL_RET; - - /* If we can get the index, perform any potential alterations. */ - if ((ret = __wt_schema_get_index( - session, uri, false, NULL, &idx)) == 0) - WT_TRET(__wt_schema_alter(session, idx->source, cfg)); - - return (ret); -} - -/* - * __alter_table -- - * WT_SESSION::alter for a table. - */ -static int -__alter_table(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) -{ - WT_COLGROUP *colgroup; - WT_DECL_RET; - WT_TABLE *table; - const char *name; - u_int i; - - name = uri; - (void)WT_PREFIX_SKIP(name, "table:"); - - WT_RET(__wt_schema_get_table( - session, name, strlen(name), true, &table)); - - /* - * Alter the column groups only if we are using the default - * column group. Otherwise the user should alter each - * index or column group explicitly. - */ - if (table->ncolgroups == 0) - for (i = 0; i < WT_COLGROUPS(table); i++) { - if ((colgroup = table->cgroups[i]) == NULL) - continue; - /* - * Alter the column group before updating the metadata - * to avoid the metadata for the table becoming - * inconsistent if we can't get exclusive access. - */ - WT_ERR(__wt_schema_alter( - session, colgroup->source, cfg)); - } -err: __wt_schema_release_table(session, table); - return (ret); -} - -/* - * __wt_schema_alter -- - * Process a WT_SESSION::alter operation for all supported types. - */ -int -__wt_schema_alter(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) -{ - WT_DATA_SOURCE *dsrc; - WT_DECL_RET; - - WT_RET(__wt_meta_track_on(session)); - - /* Paranoia: clear any handle from our caller. */ - session->dhandle = NULL; - - if (WT_PREFIX_MATCH(uri, "colgroup:")) - ret = __alter_colgroup(session, uri, cfg); - else if (WT_PREFIX_MATCH(uri, "file:")) - ret = __alter_file(session, uri, cfg); - else if (WT_PREFIX_MATCH(uri, "index:")) - ret = __alter_index(session, uri, cfg); - else if (WT_PREFIX_MATCH(uri, "lsm:")) - ret = __wt_lsm_tree_alter(session, uri, cfg); - else if (WT_PREFIX_MATCH(uri, "table:")) - ret = __alter_table(session, uri, cfg); - else if ((dsrc = __wt_schema_get_source(session, uri)) != NULL) - ret = dsrc->alter == NULL ? - __wt_object_unsupported(session, uri) : - dsrc->alter(dsrc, - &session->iface, uri, (WT_CONFIG_ARG *)cfg); - else - ret = __wt_bad_object_type(session, uri); - /* * Map WT_NOTFOUND to ENOENT, based on the assumption WT_NOTFOUND means * there was no metadata entry. diff --git a/src/third_party/wiredtiger/src/session/session_api.c b/src/third_party/wiredtiger/src/session/session_api.c index 882de30f12b..2ad2380dc2c 100644 --- a/src/third_party/wiredtiger/src/session/session_api.c +++ b/src/third_party/wiredtiger/src/session/session_api.c @@ -149,6 +149,10 @@ __session_alter(WT_SESSION *wt_session, const char *uri, const char *config) SESSION_API_CALL(session, alter, config, cfg); + /* In-memory ignores alter operations. */ + if (F_ISSET(S2C(session), WT_CONN_IN_MEMORY)) + goto err; + /* Disallow objects in the WiredTiger name space. */ WT_ERR(__wt_str_name_check(session, uri)); @@ -161,8 +165,9 @@ __session_alter(WT_SESSION *wt_session, const char *uri, const char *config) cfg[1] = NULL; WT_WITH_CHECKPOINT_LOCK(session, WT_WITH_SCHEMA_LOCK(session, - WT_WITH_TABLE_WRITE_LOCK(session, - ret = __wt_schema_alter(session, uri, cfg)))); + ret = __wt_schema_worker(session, uri, __wt_alter, NULL, cfg, + WT_BTREE_ALTER | + WT_DHANDLE_DISCARD_FORCE | WT_DHANDLE_EXCLUSIVE))); err: if (ret != 0) WT_STAT_CONN_INCR(session, session_table_alter_fail); @@ -700,6 +705,10 @@ __session_rebalance(WT_SESSION *wt_session, const char *uri, const char *config) SESSION_API_CALL(session, rebalance, config, cfg); + /* In-memory ignores rebalance operations. */ + if (F_ISSET(S2C(session), WT_CONN_IN_MEMORY)) + goto err; + /* Block out checkpoints to avoid spurious EBUSY errors. */ WT_WITH_CHECKPOINT_LOCK(session, WT_WITH_SCHEMA_LOCK(session, @@ -709,8 +718,7 @@ __session_rebalance(WT_SESSION *wt_session, const char *uri, const char *config) err: if (ret != 0) WT_STAT_CONN_INCR(session, session_table_rebalance_fail); else - WT_STAT_CONN_INCR(session, - session_table_rebalance_success); + WT_STAT_CONN_INCR(session, session_table_rebalance_success); API_END_RET_NOTFOUND_MAP(session, ret); } @@ -1046,10 +1054,7 @@ __session_salvage(WT_SESSION *wt_session, const char *uri, const char *config) SESSION_API_CALL(session, salvage, config, cfg); - if (F_ISSET(S2C(session), WT_CONN_IN_MEMORY)) - WT_ERR_MSG(session, ENOTSUP, - "WT_SESSION.salvage not supported for in-memory " - "configurations"); + WT_ERR(__wt_inmem_unsupported_op(session, NULL)); /* Block out checkpoints to avoid spurious EBUSY errors. */ WT_WITH_CHECKPOINT_LOCK(session, @@ -1258,7 +1263,7 @@ __session_truncate(WT_SESSION *wt_session, WT_ERR_MSG(session, EINVAL, "the truncate method should not specify any" "target after the log: URI prefix"); - WT_ERR(__wt_log_truncate_files(session, start, cfg)); + WT_ERR(__wt_log_truncate_files(session, start, false)); } else if (WT_PREFIX_MATCH(uri, "file:")) WT_ERR(__wt_session_range_truncate( session, uri, start, stop)); @@ -1319,6 +1324,9 @@ __session_upgrade(WT_SESSION *wt_session, const char *uri, const char *config) session = (WT_SESSION_IMPL *)wt_session; SESSION_API_CALL(session, upgrade, config, cfg); + + WT_ERR(__wt_inmem_unsupported_op(session, NULL)); + /* Block out checkpoints to avoid spurious EBUSY errors. */ WT_WITH_CHECKPOINT_LOCK(session, WT_WITH_SCHEMA_LOCK(session, @@ -1363,10 +1371,7 @@ __session_verify(WT_SESSION *wt_session, const char *uri, const char *config) SESSION_API_CALL(session, verify, config, cfg); - if (F_ISSET(S2C(session), WT_CONN_IN_MEMORY)) - WT_ERR_MSG(session, ENOTSUP, - "WT_SESSION.verify not supported for in-memory " - "configurations"); + WT_ERR(__wt_inmem_unsupported_op(session, NULL)); /* Block out checkpoints to avoid spurious EBUSY errors. */ WT_WITH_CHECKPOINT_LOCK(session, @@ -1632,10 +1637,7 @@ __session_checkpoint(WT_SESSION *wt_session, const char *config) WT_STAT_CONN_INCR(session, txn_checkpoint); SESSION_API_CALL(session, checkpoint, config, cfg); - if (F_ISSET(S2C(session), WT_CONN_IN_MEMORY)) - WT_ERR_MSG(session, ENOTSUP, - "WT_SESSION.checkpoint not supported for in-memory " - "configurations"); + WT_ERR(__wt_inmem_unsupported_op(session, NULL)); /* * Checkpoints require a snapshot to write a transactionally consistent @@ -1826,7 +1828,7 @@ __open_session(WT_CONNECTION_IMPL *conn, if (i == conn->session_size) WT_ERR_MSG(session, ENOMEM, "only configured to support %" PRIu32 " sessions" - " (including %" PRIu32 " additional internal sessions)", + " (including %d additional internal sessions)", conn->session_size, WT_EXTRA_INTERNAL_SESSIONS); /* diff --git a/src/third_party/wiredtiger/src/session/session_compact.c b/src/third_party/wiredtiger/src/session/session_compact.c index c4710dbb1a5..10328e0db5b 100644 --- a/src/third_party/wiredtiger/src/session/session_compact.c +++ b/src/third_party/wiredtiger/src/session/session_compact.c @@ -324,7 +324,7 @@ __wt_session_compact( session = (WT_SESSION_IMPL *)wt_session; SESSION_API_CALL(session, compact, config, cfg); - /* In-memory is already as compact as it's going to get. */ + /* In-memory ignores compaction operations. */ if (F_ISSET(S2C(session), WT_CONN_IN_MEMORY)) goto err; diff --git a/src/third_party/wiredtiger/src/support/err.c b/src/third_party/wiredtiger/src/support/err.c index 5ec995d8f65..94ae27628c2 100644 --- a/src/third_party/wiredtiger/src/support/err.c +++ b/src/third_party/wiredtiger/src/support/err.c @@ -361,6 +361,22 @@ __wt_ext_err_printf( } /* + * __wt_verbose_worker -- + * Verbose message. + */ +void +__wt_verbose_worker(WT_SESSION_IMPL *session, const char *fmt, ...) + WT_GCC_FUNC_ATTRIBUTE((format (printf, 2, 3))) + WT_GCC_FUNC_ATTRIBUTE((cold)) +{ + va_list ap; + + va_start(ap, fmt); + WT_IGNORE_RET(__wt_eventv(session, true, 0, NULL, 0, fmt, ap)); + va_end(ap); +} + +/* * info_msg -- * Informational message. */ @@ -533,6 +549,22 @@ __wt_illegal_value(WT_SESSION_IMPL *session, const char *name) } /* + * __wt_inmem_unsupported_op -- + * Print a standard error message for an operation that's not supported + * for in-memory configurations. + */ +int +__wt_inmem_unsupported_op(WT_SESSION_IMPL *session, const char *tag) + WT_GCC_FUNC_ATTRIBUTE((cold)) +{ + if (F_ISSET(S2C(session), WT_CONN_IN_MEMORY)) + WT_RET_MSG(session, ENOTSUP, + "%s%snot supported for in-memory configurations", + tag == NULL ? "" : tag, tag == NULL ? "" : ": "); + return (0); +} + +/* * __wt_object_unsupported -- * Print a standard error message for an object that doesn't support a * particular operation. diff --git a/src/third_party/wiredtiger/src/support/generation.c b/src/third_party/wiredtiger/src/support/generation.c index 6e16d7e57fe..182fb2de095 100644 --- a/src/third_party/wiredtiger/src/support/generation.c +++ b/src/third_party/wiredtiger/src/support/generation.c @@ -116,6 +116,12 @@ __wt_gen_drain(WT_SESSION_IMPL *session, int which, uint64_t generation) if (v == 0 || v >= generation) break; + /* If we're waiting on ourselves, we're deadlocked. */ + if (session == s) { + WT_ASSERT(session, session != s); + WT_IGNORE_RET(__wt_panic(session)); + } + /* * The pause count is cumulative, quit spinning if it's * not doing us any good, that can happen in generations diff --git a/src/third_party/wiredtiger/src/support/rand.c b/src/third_party/wiredtiger/src/support/rand.c index 8083b8801c1..64ed2341eeb 100644 --- a/src/third_party/wiredtiger/src/support/rand.c +++ b/src/third_party/wiredtiger/src/support/rand.c @@ -120,15 +120,3 @@ __wt_random(WT_RAND_STATE volatile * rnd_state) return ((z << 16) + (w & 65535)); } - -/* - * __wt_random64 -- - * Return a 64-bit pseudo-random number. - */ -uint64_t -__wt_random64(WT_RAND_STATE volatile * rnd_state) - WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) -{ - return (((uint64_t)__wt_random(rnd_state) << 32) + - __wt_random(rnd_state)); -} diff --git a/src/third_party/wiredtiger/src/support/stat.c b/src/third_party/wiredtiger/src/support/stat.c index 2dc006da827..14a1d2a3b86 100644 --- a/src/third_party/wiredtiger/src/support/stat.c +++ b/src/third_party/wiredtiger/src/support/stat.c @@ -48,6 +48,19 @@ static const char * const __stats_dsrc_desc[] = { "cache: bytes written from cache", "cache: checkpoint blocked page eviction", "cache: data source pages selected for eviction unable to be evicted", + "cache: eviction walk passes of a file", + "cache: eviction walk target pages histogram - 0-9", + "cache: eviction walk target pages histogram - 10-31", + "cache: eviction walk target pages histogram - 128 and higher", + "cache: eviction walk target pages histogram - 32-63", + "cache: eviction walk target pages histogram - 64-128", + "cache: eviction walks abandoned", + "cache: eviction walks gave up because they restarted their walk twice", + "cache: eviction walks gave up because they saw too many pages and found no candidates", + "cache: eviction walks gave up because they saw too many pages and found too few candidates", + "cache: eviction walks reached end of tree", + "cache: eviction walks started from root of tree", + "cache: eviction walks started from saved location in tree", "cache: hazard pointer blocked page eviction", "cache: in-memory page passed criteria to be split", "cache: in-memory page splits", @@ -56,18 +69,20 @@ static const char * const __stats_dsrc_desc[] = { "cache: leaf pages split during eviction", "cache: modified pages evicted", "cache: overflow pages read into cache", - "cache: overflow values cached in memory", "cache: page split during eviction deepened the tree", "cache: page written requiring lookaside records", "cache: pages read into cache", "cache: pages read into cache requiring lookaside entries", "cache: pages requested from the cache", + "cache: pages seen by eviction walk", "cache: pages written from cache", "cache: pages written requiring in-memory restoration", "cache: tracked dirty bytes in the cache", "cache: unmodified pages evicted", "cache_walk: Average difference between current eviction generation when the page was last considered", "cache_walk: Average on-disk page image size seen", + "cache_walk: Average time in cache for pages that have been visited by the eviction server", + "cache_walk: Average time in cache for pages that have not been visited by the eviction server", "cache_walk: Clean pages currently in cache", "cache_walk: Current eviction generation", "cache_walk: Dirty pages currently in cache", @@ -77,6 +92,7 @@ static const char * const __stats_dsrc_desc[] = { "cache_walk: Maximum difference between current eviction generation when the page was last considered", "cache_walk: Maximum page size seen", "cache_walk: Minimum on-disk page image size seen", + "cache_walk: Number of pages never visited by eviction server", "cache_walk: On-disk page image sizes smaller than a single allocation unit", "cache_walk: Pages created in memory and never written", "cache_walk: Pages currently queued for eviction", @@ -212,6 +228,19 @@ __wt_stat_dsrc_clear_single(WT_DSRC_STATS *stats) stats->cache_bytes_write = 0; stats->cache_eviction_checkpoint = 0; stats->cache_eviction_fail = 0; + stats->cache_eviction_walk_passes = 0; + stats->cache_eviction_target_page_lt10 = 0; + stats->cache_eviction_target_page_lt32 = 0; + stats->cache_eviction_target_page_ge128 = 0; + stats->cache_eviction_target_page_lt64 = 0; + stats->cache_eviction_target_page_lt128 = 0; + stats->cache_eviction_walks_abandoned = 0; + stats->cache_eviction_walks_stopped = 0; + stats->cache_eviction_walks_gave_up_no_targets = 0; + stats->cache_eviction_walks_gave_up_ratio = 0; + stats->cache_eviction_walks_ended = 0; + stats->cache_eviction_walk_from_root = 0; + stats->cache_eviction_walk_saved_pos = 0; stats->cache_eviction_hazard = 0; stats->cache_inmem_splittable = 0; stats->cache_inmem_split = 0; @@ -220,18 +249,20 @@ __wt_stat_dsrc_clear_single(WT_DSRC_STATS *stats) stats->cache_eviction_split_leaf = 0; stats->cache_eviction_dirty = 0; stats->cache_read_overflow = 0; - stats->cache_overflow_value = 0; stats->cache_eviction_deepen = 0; stats->cache_write_lookaside = 0; stats->cache_read = 0; stats->cache_read_lookaside = 0; stats->cache_pages_requested = 0; + stats->cache_eviction_pages_seen = 0; stats->cache_write = 0; stats->cache_write_restore = 0; /* not clearing cache_bytes_dirty */ stats->cache_eviction_clean = 0; /* not clearing cache_state_gen_avg_gap */ /* not clearing cache_state_avg_written_size */ + /* not clearing cache_state_avg_visited_age */ + /* not clearing cache_state_avg_unvisited_age */ /* not clearing cache_state_pages_clean */ /* not clearing cache_state_gen_current */ /* not clearing cache_state_pages_dirty */ @@ -241,6 +272,7 @@ __wt_stat_dsrc_clear_single(WT_DSRC_STATS *stats) /* not clearing cache_state_gen_max_gap */ /* not clearing cache_state_max_pagesize */ /* not clearing cache_state_min_written_size */ + /* not clearing cache_state_unvisited_count */ /* not clearing cache_state_smaller_alloc_size */ /* not clearing cache_state_memory */ /* not clearing cache_state_queued */ @@ -361,6 +393,30 @@ __wt_stat_dsrc_aggregate_single( to->cache_bytes_write += from->cache_bytes_write; to->cache_eviction_checkpoint += from->cache_eviction_checkpoint; to->cache_eviction_fail += from->cache_eviction_fail; + to->cache_eviction_walk_passes += from->cache_eviction_walk_passes; + to->cache_eviction_target_page_lt10 += + from->cache_eviction_target_page_lt10; + to->cache_eviction_target_page_lt32 += + from->cache_eviction_target_page_lt32; + to->cache_eviction_target_page_ge128 += + from->cache_eviction_target_page_ge128; + to->cache_eviction_target_page_lt64 += + from->cache_eviction_target_page_lt64; + to->cache_eviction_target_page_lt128 += + from->cache_eviction_target_page_lt128; + to->cache_eviction_walks_abandoned += + from->cache_eviction_walks_abandoned; + to->cache_eviction_walks_stopped += + from->cache_eviction_walks_stopped; + to->cache_eviction_walks_gave_up_no_targets += + from->cache_eviction_walks_gave_up_no_targets; + to->cache_eviction_walks_gave_up_ratio += + from->cache_eviction_walks_gave_up_ratio; + to->cache_eviction_walks_ended += from->cache_eviction_walks_ended; + to->cache_eviction_walk_from_root += + from->cache_eviction_walk_from_root; + to->cache_eviction_walk_saved_pos += + from->cache_eviction_walk_saved_pos; to->cache_eviction_hazard += from->cache_eviction_hazard; to->cache_inmem_splittable += from->cache_inmem_splittable; to->cache_inmem_split += from->cache_inmem_split; @@ -370,12 +426,12 @@ __wt_stat_dsrc_aggregate_single( to->cache_eviction_split_leaf += from->cache_eviction_split_leaf; to->cache_eviction_dirty += from->cache_eviction_dirty; to->cache_read_overflow += from->cache_read_overflow; - to->cache_overflow_value += from->cache_overflow_value; to->cache_eviction_deepen += from->cache_eviction_deepen; to->cache_write_lookaside += from->cache_write_lookaside; to->cache_read += from->cache_read; to->cache_read_lookaside += from->cache_read_lookaside; to->cache_pages_requested += from->cache_pages_requested; + to->cache_eviction_pages_seen += from->cache_eviction_pages_seen; to->cache_write += from->cache_write; to->cache_write_restore += from->cache_write_restore; to->cache_bytes_dirty += from->cache_bytes_dirty; @@ -383,6 +439,9 @@ __wt_stat_dsrc_aggregate_single( to->cache_state_gen_avg_gap += from->cache_state_gen_avg_gap; to->cache_state_avg_written_size += from->cache_state_avg_written_size; + to->cache_state_avg_visited_age += from->cache_state_avg_visited_age; + to->cache_state_avg_unvisited_age += + from->cache_state_avg_unvisited_age; to->cache_state_pages_clean += from->cache_state_pages_clean; to->cache_state_gen_current += from->cache_state_gen_current; to->cache_state_pages_dirty += from->cache_state_pages_dirty; @@ -393,6 +452,7 @@ __wt_stat_dsrc_aggregate_single( to->cache_state_max_pagesize += from->cache_state_max_pagesize; to->cache_state_min_written_size += from->cache_state_min_written_size; + to->cache_state_unvisited_count += from->cache_state_unvisited_count; to->cache_state_smaller_alloc_size += from->cache_state_smaller_alloc_size; to->cache_state_memory += from->cache_state_memory; @@ -520,6 +580,32 @@ __wt_stat_dsrc_aggregate( to->cache_eviction_checkpoint += WT_STAT_READ(from, cache_eviction_checkpoint); to->cache_eviction_fail += WT_STAT_READ(from, cache_eviction_fail); + to->cache_eviction_walk_passes += + WT_STAT_READ(from, cache_eviction_walk_passes); + to->cache_eviction_target_page_lt10 += + WT_STAT_READ(from, cache_eviction_target_page_lt10); + to->cache_eviction_target_page_lt32 += + WT_STAT_READ(from, cache_eviction_target_page_lt32); + to->cache_eviction_target_page_ge128 += + WT_STAT_READ(from, cache_eviction_target_page_ge128); + to->cache_eviction_target_page_lt64 += + WT_STAT_READ(from, cache_eviction_target_page_lt64); + to->cache_eviction_target_page_lt128 += + WT_STAT_READ(from, cache_eviction_target_page_lt128); + to->cache_eviction_walks_abandoned += + WT_STAT_READ(from, cache_eviction_walks_abandoned); + to->cache_eviction_walks_stopped += + WT_STAT_READ(from, cache_eviction_walks_stopped); + to->cache_eviction_walks_gave_up_no_targets += + WT_STAT_READ(from, cache_eviction_walks_gave_up_no_targets); + to->cache_eviction_walks_gave_up_ratio += + WT_STAT_READ(from, cache_eviction_walks_gave_up_ratio); + to->cache_eviction_walks_ended += + WT_STAT_READ(from, cache_eviction_walks_ended); + to->cache_eviction_walk_from_root += + WT_STAT_READ(from, cache_eviction_walk_from_root); + to->cache_eviction_walk_saved_pos += + WT_STAT_READ(from, cache_eviction_walk_saved_pos); to->cache_eviction_hazard += WT_STAT_READ(from, cache_eviction_hazard); to->cache_inmem_splittable += @@ -533,7 +619,6 @@ __wt_stat_dsrc_aggregate( WT_STAT_READ(from, cache_eviction_split_leaf); to->cache_eviction_dirty += WT_STAT_READ(from, cache_eviction_dirty); to->cache_read_overflow += WT_STAT_READ(from, cache_read_overflow); - to->cache_overflow_value += WT_STAT_READ(from, cache_overflow_value); to->cache_eviction_deepen += WT_STAT_READ(from, cache_eviction_deepen); to->cache_write_lookaside += @@ -542,6 +627,8 @@ __wt_stat_dsrc_aggregate( to->cache_read_lookaside += WT_STAT_READ(from, cache_read_lookaside); to->cache_pages_requested += WT_STAT_READ(from, cache_pages_requested); + to->cache_eviction_pages_seen += + WT_STAT_READ(from, cache_eviction_pages_seen); to->cache_write += WT_STAT_READ(from, cache_write); to->cache_write_restore += WT_STAT_READ(from, cache_write_restore); to->cache_bytes_dirty += WT_STAT_READ(from, cache_bytes_dirty); @@ -550,6 +637,10 @@ __wt_stat_dsrc_aggregate( WT_STAT_READ(from, cache_state_gen_avg_gap); to->cache_state_avg_written_size += WT_STAT_READ(from, cache_state_avg_written_size); + to->cache_state_avg_visited_age += + WT_STAT_READ(from, cache_state_avg_visited_age); + to->cache_state_avg_unvisited_age += + WT_STAT_READ(from, cache_state_avg_unvisited_age); to->cache_state_pages_clean += WT_STAT_READ(from, cache_state_pages_clean); to->cache_state_gen_current += @@ -568,6 +659,8 @@ __wt_stat_dsrc_aggregate( WT_STAT_READ(from, cache_state_max_pagesize); to->cache_state_min_written_size += WT_STAT_READ(from, cache_state_min_written_size); + to->cache_state_unvisited_count += + WT_STAT_READ(from, cache_state_unvisited_count); to->cache_state_smaller_alloc_size += WT_STAT_READ(from, cache_state_smaller_alloc_size); to->cache_state_memory += WT_STAT_READ(from, cache_state_memory); @@ -678,13 +771,25 @@ static const char * const __stats_connection_desc[] = { "cache: eviction calls to get a page found queue empty after locking", "cache: eviction currently operating in aggressive mode", "cache: eviction empty score", + "cache: eviction passes of a file", "cache: eviction server candidate queue empty when topping up", "cache: eviction server candidate queue not empty when topping up", "cache: eviction server evicting pages", "cache: eviction server slept, because we did not make progress with eviction", "cache: eviction server unable to reach eviction goal", "cache: eviction state", + "cache: eviction walk target pages histogram - 0-9", + "cache: eviction walk target pages histogram - 10-31", + "cache: eviction walk target pages histogram - 128 and higher", + "cache: eviction walk target pages histogram - 32-63", + "cache: eviction walk target pages histogram - 64-128", "cache: eviction walks abandoned", + "cache: eviction walks gave up because they restarted their walk twice", + "cache: eviction walks gave up because they saw too many pages and found no candidates", + "cache: eviction walks gave up because they saw too many pages and found too few candidates", + "cache: eviction walks reached end of tree", + "cache: eviction walks started from root of tree", + "cache: eviction walks started from saved location in tree", "cache: eviction worker thread active", "cache: eviction worker thread created", "cache: eviction worker thread evicting pages", @@ -711,7 +816,6 @@ static const char * const __stats_connection_desc[] = { "cache: modified pages evicted", "cache: modified pages evicted by application threads", "cache: overflow pages read into cache", - "cache: overflow values cached in memory", "cache: page split during eviction deepened the tree", "cache: page written requiring lookaside records", "cache: pages currently held in the cache", @@ -789,6 +893,7 @@ static const char * const __stats_connection_desc[] = { "lock: table read lock acquisitions", "lock: table write lock acquisitions", "log: busy returns attempting to switch slots", + "log: force checkpoint calls slept", "log: log bytes of payload data", "log: log bytes written", "log: log files manually zero-filled", @@ -981,13 +1086,25 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->cache_eviction_get_ref_empty2 = 0; /* not clearing cache_eviction_aggressive_set */ /* not clearing cache_eviction_empty_score */ + stats->cache_eviction_walk_passes = 0; stats->cache_eviction_queue_empty = 0; stats->cache_eviction_queue_not_empty = 0; stats->cache_eviction_server_evicting = 0; stats->cache_eviction_server_slept = 0; stats->cache_eviction_slow = 0; /* not clearing cache_eviction_state */ + stats->cache_eviction_target_page_lt10 = 0; + stats->cache_eviction_target_page_lt32 = 0; + stats->cache_eviction_target_page_ge128 = 0; + stats->cache_eviction_target_page_lt64 = 0; + stats->cache_eviction_target_page_lt128 = 0; stats->cache_eviction_walks_abandoned = 0; + stats->cache_eviction_walks_stopped = 0; + stats->cache_eviction_walks_gave_up_no_targets = 0; + stats->cache_eviction_walks_gave_up_ratio = 0; + stats->cache_eviction_walks_ended = 0; + stats->cache_eviction_walk_from_root = 0; + stats->cache_eviction_walk_saved_pos = 0; /* not clearing cache_eviction_active_workers */ stats->cache_eviction_worker_created = 0; stats->cache_eviction_worker_evicting = 0; @@ -1014,7 +1131,6 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->cache_eviction_dirty = 0; stats->cache_eviction_app_dirty = 0; stats->cache_read_overflow = 0; - stats->cache_overflow_value = 0; stats->cache_eviction_deepen = 0; stats->cache_write_lookaside = 0; /* not clearing cache_pages_inuse */ @@ -1092,6 +1208,7 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->lock_table_read_count = 0; stats->lock_table_write_count = 0; stats->log_slot_switch_busy = 0; + stats->log_force_ckpt_sleep = 0; stats->log_bytes_payload = 0; stats->log_bytes_written = 0; stats->log_zero_fills = 0; @@ -1272,6 +1389,8 @@ __wt_stat_connection_aggregate( WT_STAT_READ(from, cache_eviction_aggressive_set); to->cache_eviction_empty_score += WT_STAT_READ(from, cache_eviction_empty_score); + to->cache_eviction_walk_passes += + WT_STAT_READ(from, cache_eviction_walk_passes); to->cache_eviction_queue_empty += WT_STAT_READ(from, cache_eviction_queue_empty); to->cache_eviction_queue_not_empty += @@ -1282,8 +1401,30 @@ __wt_stat_connection_aggregate( WT_STAT_READ(from, cache_eviction_server_slept); to->cache_eviction_slow += WT_STAT_READ(from, cache_eviction_slow); to->cache_eviction_state += WT_STAT_READ(from, cache_eviction_state); + to->cache_eviction_target_page_lt10 += + WT_STAT_READ(from, cache_eviction_target_page_lt10); + to->cache_eviction_target_page_lt32 += + WT_STAT_READ(from, cache_eviction_target_page_lt32); + to->cache_eviction_target_page_ge128 += + WT_STAT_READ(from, cache_eviction_target_page_ge128); + to->cache_eviction_target_page_lt64 += + WT_STAT_READ(from, cache_eviction_target_page_lt64); + to->cache_eviction_target_page_lt128 += + WT_STAT_READ(from, cache_eviction_target_page_lt128); to->cache_eviction_walks_abandoned += WT_STAT_READ(from, cache_eviction_walks_abandoned); + to->cache_eviction_walks_stopped += + WT_STAT_READ(from, cache_eviction_walks_stopped); + to->cache_eviction_walks_gave_up_no_targets += + WT_STAT_READ(from, cache_eviction_walks_gave_up_no_targets); + to->cache_eviction_walks_gave_up_ratio += + WT_STAT_READ(from, cache_eviction_walks_gave_up_ratio); + to->cache_eviction_walks_ended += + WT_STAT_READ(from, cache_eviction_walks_ended); + to->cache_eviction_walk_from_root += + WT_STAT_READ(from, cache_eviction_walk_from_root); + to->cache_eviction_walk_saved_pos += + WT_STAT_READ(from, cache_eviction_walk_saved_pos); to->cache_eviction_active_workers += WT_STAT_READ(from, cache_eviction_active_workers); to->cache_eviction_worker_created += @@ -1330,7 +1471,6 @@ __wt_stat_connection_aggregate( to->cache_eviction_app_dirty += WT_STAT_READ(from, cache_eviction_app_dirty); to->cache_read_overflow += WT_STAT_READ(from, cache_read_overflow); - to->cache_overflow_value += WT_STAT_READ(from, cache_overflow_value); to->cache_eviction_deepen += WT_STAT_READ(from, cache_eviction_deepen); to->cache_write_lookaside += @@ -1433,6 +1573,7 @@ __wt_stat_connection_aggregate( to->lock_table_write_count += WT_STAT_READ(from, lock_table_write_count); to->log_slot_switch_busy += WT_STAT_READ(from, log_slot_switch_busy); + to->log_force_ckpt_sleep += WT_STAT_READ(from, log_force_ckpt_sleep); to->log_bytes_payload += WT_STAT_READ(from, log_bytes_payload); to->log_bytes_written += WT_STAT_READ(from, log_bytes_written); to->log_zero_fills += WT_STAT_READ(from, log_zero_fills); diff --git a/src/third_party/wiredtiger/src/txn/txn.c b/src/third_party/wiredtiger/src/txn/txn.c index 02adad65ba5..18a3c0021f0 100644 --- a/src/third_party/wiredtiger/src/txn/txn.c +++ b/src/third_party/wiredtiger/src/txn/txn.c @@ -441,7 +441,6 @@ __wt_txn_config(WT_SESSION_IMPL *session, const char *cfg[]) if (cval.len > 0) { #ifdef HAVE_TIMESTAMPS WT_TXN_GLOBAL *txn_global = &S2C(session)->txn_global; - WT_TXN_STATE *txn_state = WT_SESSION_TXN_STATE(session); wt_timestamp_t oldest_timestamp; WT_RET(__wt_txn_parse_timestamp( @@ -455,9 +454,8 @@ __wt_txn_config(WT_SESSION_IMPL *session, const char *cfg[]) WT_RET_MSG(session, EINVAL, "read timestamp %.*s older than oldest timestamp", (int)cval.len, cval.str); - __wt_timestamp_set( - txn_state->read_timestamp, txn->read_timestamp); - F_SET(txn, WT_TXN_HAS_TS_READ); + + __wt_txn_set_read_timestamp(session); txn->isolation = WT_ISO_SNAPSHOT; #else WT_RET_MSG(session, EINVAL, "read_timestamp requires a " @@ -531,22 +529,15 @@ __wt_txn_release(WT_SESSION_IMPL *session) WT_ASSERT(session, txn_state->id != WT_TXN_NONE && txn->id != WT_TXN_NONE); WT_PUBLISH(txn_state->id, WT_TXN_NONE); -#ifdef HAVE_TIMESTAMPS - if (F_ISSET(txn, WT_TXN_HAS_TS_COMMIT)) { - /* - * We rely on a non-zero ID to protect our published - * commit timestamp. Otherwise we would need a lock - * here. - */ - WT_WRITE_BARRIER(); - __wt_timestamp_set( - txn_state->commit_timestamp, zero_timestamp); - } -#endif txn->id = WT_TXN_NONE; } +#ifdef HAVE_TIMESTAMPS + __wt_txn_clear_commit_timestamp(session); + __wt_txn_clear_read_timestamp(session); +#endif + /* Free the scratch buffer allocated for logging. */ __wt_logrec_free(session, &txn->logrec); @@ -578,7 +569,6 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) WT_TXN_OP *op; #ifdef HAVE_TIMESTAMPS WT_TXN_GLOBAL *txn_global = &S2C(session)->txn_global; - WT_TXN_STATE *txn_state = WT_SESSION_TXN_STATE(session); wt_timestamp_t prev_commit_timestamp; bool update_timestamp; #endif @@ -601,13 +591,7 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) #ifdef HAVE_TIMESTAMPS WT_ERR(__wt_txn_parse_timestamp( session, "commit", txn->commit_timestamp, &cval)); - if (!F_ISSET(txn, WT_TXN_HAS_TS_COMMIT)) { - __wt_writelock(session, &txn_global->rwlock); - __wt_timestamp_set(txn_state->commit_timestamp, - txn->commit_timestamp); - __wt_writeunlock(session, &txn_global->rwlock); - F_SET(txn, WT_TXN_HAS_TS_COMMIT); - } + __wt_txn_set_commit_timestamp(session); #else WT_ERR_MSG(session, EINVAL, "commit_timestamp requires a " "version of WiredTiger built with timestamp support"); @@ -944,8 +928,14 @@ __wt_txn_global_init(WT_SESSION_IMPL *session, const char *cfg[]) WT_RET(__wt_spin_init( session, &txn_global->id_lock, "transaction id lock")); WT_RET(__wt_rwlock_init(session, &txn_global->rwlock)); - WT_RET(__wt_rwlock_init(session, &txn_global->nsnap_rwlock)); + WT_RET(__wt_rwlock_init(session, &txn_global->commit_timestamp_rwlock)); + TAILQ_INIT(&txn_global->commit_timestamph); + + WT_RET(__wt_rwlock_init(session, &txn_global->read_timestamp_rwlock)); + TAILQ_INIT(&txn_global->read_timestamph); + + WT_RET(__wt_rwlock_init(session, &txn_global->nsnap_rwlock)); txn_global->nsnap_oldest_id = WT_TXN_NONE; TAILQ_INIT(&txn_global->nsnaph); @@ -976,6 +966,8 @@ __wt_txn_global_destroy(WT_SESSION_IMPL *session) __wt_spin_destroy(session, &txn_global->id_lock); __wt_rwlock_destroy(session, &txn_global->rwlock); + __wt_rwlock_destroy(session, &txn_global->commit_timestamp_rwlock); + __wt_rwlock_destroy(session, &txn_global->read_timestamp_rwlock); __wt_rwlock_destroy(session, &txn_global->nsnap_rwlock); __wt_free(session, txn_global->states); } diff --git a/src/third_party/wiredtiger/src/txn/txn_ckpt.c b/src/third_party/wiredtiger/src/txn/txn_ckpt.c index 73ad96b5518..519d3469865 100644 --- a/src/third_party/wiredtiger/src/txn/txn_ckpt.c +++ b/src/third_party/wiredtiger/src/txn/txn_ckpt.c @@ -625,6 +625,7 @@ __checkpoint_prepare(WT_SESSION_IMPL *session, const char *cfg[]) */ __wt_writelock(session, &txn_global->rwlock); txn_global->checkpoint_state = *txn_state; + txn_global->checkpoint_txn = txn; txn_global->checkpoint_state.pinned_id = WT_MIN(txn->id, txn->snap_min); /* @@ -646,6 +647,15 @@ __checkpoint_prepare(WT_SESSION_IMPL *session, const char *cfg[]) txn_state->metadata_pinned = WT_TXN_NONE; __wt_writeunlock(session, &txn_global->rwlock); +#ifdef HAVE_TIMESTAMPS + /* + * Now that the checkpoint transaction is published, clear it from the + * regular lists. + */ + __wt_txn_clear_commit_timestamp(session); + __wt_txn_clear_read_timestamp(session); +#endif + /* * Get a list of handles we want to flush; for named checkpoints this * may pull closed objects into the session cache. diff --git a/src/third_party/wiredtiger/src/txn/txn_log.c b/src/third_party/wiredtiger/src/txn/txn_log.c index 3d218aea94f..d291139284a 100644 --- a/src/third_party/wiredtiger/src/txn/txn_log.c +++ b/src/third_party/wiredtiger/src/txn/txn_log.c @@ -425,7 +425,8 @@ __wt_txn_checkpoint_log( * metadata LSN and we do not want to archive in that case. */ if (!conn->hot_backup && - !FLD_ISSET(conn->log_flags, WT_CONN_LOG_RECOVER_DIRTY) && + (!FLD_ISSET(conn->log_flags, WT_CONN_LOG_RECOVER_DIRTY) || + FLD_ISSET(conn->log_flags, WT_CONN_LOG_FORCE_DOWNGRADE)) && txn->full_ckpt) __wt_log_ckpt(session, ckpt_lsn); @@ -589,6 +590,16 @@ __txn_printlog(WT_SESSION_IMPL *session, WT_RET(__wt_fprintf(session, WT_STDOUT(session), " \"message\" : \"%s\"\n", msg)); break; + + case WT_LOGREC_SYSTEM: + WT_RET(__wt_struct_unpack(session, p, WT_PTRDIFF(end, p), + WT_UNCHECKED_STRING(II), &lsnfile, &lsnoffset)); + WT_RET(__wt_fprintf(session, WT_STDOUT(session), + " \"type\" : \"system\",\n")); + WT_RET(__wt_fprintf(session, WT_STDOUT(session), + " \"prev_lsn\" : [%" PRIu32 ",%" PRIu32 "]\n", + lsnfile, lsnoffset)); + break; } WT_RET(__wt_fprintf(session, WT_STDOUT(session), " }")); diff --git a/src/third_party/wiredtiger/src/txn/txn_recover.c b/src/third_party/wiredtiger/src/txn/txn_recover.c index 58f4f0750d7..d08c2717f7a 100644 --- a/src/third_party/wiredtiger/src/txn/txn_recover.c +++ b/src/third_party/wiredtiger/src/txn/txn_recover.c @@ -235,8 +235,8 @@ __txn_op_apply( return (0); err: __wt_err(session, ret, - "operation apply failed during recovery: operation type %d " - "at LSN %" PRIu32 "/%" PRIu32, + "operation apply failed during recovery: operation type %" + PRIu32 " at LSN %" PRIu32 "/%" PRIu32, optype, lsnp->l.file, lsnp->l.offset); return (ret); } @@ -588,7 +588,6 @@ __wt_txn_recover(WT_SESSION_IMPL *session) * LSN and archiving. */ ckpt: WT_ERR(session->iface.checkpoint(&session->iface, "force=1")); - done: FLD_SET(conn->log_flags, WT_CONN_LOG_RECOVER_DONE); err: WT_TRET(__recovery_free(&r)); __wt_free(session, config); diff --git a/src/third_party/wiredtiger/src/txn/txn_timestamp.c b/src/third_party/wiredtiger/src/txn/txn_timestamp.c index ebef0320d4f..a975341c189 100644 --- a/src/third_party/wiredtiger/src/txn/txn_timestamp.c +++ b/src/third_party/wiredtiger/src/txn/txn_timestamp.c @@ -24,7 +24,7 @@ __wt_txn_parse_timestamp(WT_SESSION_IMPL *session, const char *hexts; char padbuf[2 * WT_TIMESTAMP_SIZE + 1]; - __wt_timestamp_set(timestamp, zero_timestamp); + __wt_timestamp_set_zero(timestamp); if (cval->len == 0) return (0); @@ -80,9 +80,8 @@ __txn_global_query_timestamp( { WT_CONNECTION_IMPL *conn; WT_CONFIG_ITEM cval; + WT_TXN *txn; WT_TXN_GLOBAL *txn_global; - WT_TXN_STATE *s; - uint32_t i, session_cnt; conn = S2C(session); txn_global = &conn->txn_global; @@ -93,35 +92,36 @@ __txn_global_query_timestamp( return (WT_NOTFOUND); __wt_readlock(session, &txn_global->rwlock); __wt_timestamp_set(ts, txn_global->commit_timestamp); - WT_ORDERED_READ(session_cnt, conn->session_cnt); - for (i = 0, s = txn_global->states; i < session_cnt; i++, s++) { - if (s->id == WT_TXN_NONE || - __wt_timestamp_iszero(s->commit_timestamp)) - continue; - if (__wt_timestamp_cmp(s->commit_timestamp, ts) < 0) - __wt_timestamp_set(ts, s->commit_timestamp); - } __wt_readunlock(session, &txn_global->rwlock); + + /* Compare with the oldest running transaction. */ + __wt_readlock(session, &txn_global->commit_timestamp_rwlock); + txn = TAILQ_FIRST(&txn_global->commit_timestamph); + if (txn != NULL && + __wt_timestamp_cmp(txn->commit_timestamp, ts) < 0) + __wt_timestamp_set(ts, txn->commit_timestamp); + __wt_readunlock(session, &txn_global->commit_timestamp_rwlock); } else if (WT_STRING_MATCH("oldest_reader", cval.str, cval.len)) { if (!txn_global->has_oldest_timestamp) return (WT_NOTFOUND); __wt_readlock(session, &txn_global->rwlock); __wt_timestamp_set(ts, txn_global->oldest_timestamp); - /* Look at running checkpoints. */ - s = &txn_global->checkpoint_state; - if (s->pinned_id != WT_TXN_NONE && - !__wt_timestamp_iszero(s->read_timestamp) && - __wt_timestamp_cmp(s->read_timestamp, ts) < 0) - __wt_timestamp_set(ts, s->read_timestamp); - WT_ORDERED_READ(session_cnt, conn->session_cnt); - for (i = 0, s = txn_global->states; i < session_cnt; i++, s++) { - if (s->pinned_id == WT_TXN_NONE || - __wt_timestamp_iszero(s->read_timestamp)) - continue; - if (__wt_timestamp_cmp(s->read_timestamp, ts) < 0) - __wt_timestamp_set(ts, s->read_timestamp); - } + + /* Check for a running checkpoint */ + txn = txn_global->checkpoint_txn; + if (txn_global->checkpoint_state.pinned_id != WT_TXN_NONE && + !__wt_timestamp_iszero(txn->read_timestamp) && + __wt_timestamp_cmp(txn->read_timestamp, ts) < 0) + __wt_timestamp_set(ts, txn->read_timestamp); __wt_readunlock(session, &txn_global->rwlock); + + /* Look for the oldest ordinary reader. */ + __wt_readlock(session, &txn_global->read_timestamp_rwlock); + txn = TAILQ_FIRST(&txn_global->read_timestamph); + if (txn != NULL && + __wt_timestamp_cmp(txn->read_timestamp, ts) < 0) + __wt_timestamp_set(ts, txn->read_timestamp); + __wt_readunlock(session, &txn_global->read_timestamp_rwlock); } else return (__wt_illegal_value(session, NULL)); @@ -143,6 +143,11 @@ __wt_txn_global_query_timestamp( size_t len; uint8_t *tsp; + /* + * Keep clang-analyzer happy: it can't tell that ts will be set + * whenever the call below succeeds. + */ + WT_CLEAR(ts); WT_RET(__txn_global_query_timestamp(session, ts, cfg)); /* Avoid memory allocation: set up an item guaranteed large enough. */ @@ -279,18 +284,10 @@ __wt_txn_set_timestamp(WT_SESSION_IMPL *session, const char *cfg[]) if (ret == 0 && cval.len != 0) { #ifdef HAVE_TIMESTAMPS WT_TXN *txn = &session->txn; - WT_TXN_GLOBAL *txn_global = &S2C(session)->txn_global; - WT_TXN_STATE *txn_state = WT_SESSION_TXN_STATE(session); WT_RET(__wt_txn_parse_timestamp( session, "commit", txn->commit_timestamp, &cval)); - if (!F_ISSET(txn, WT_TXN_HAS_TS_COMMIT)) { - __wt_writelock(session, &txn_global->rwlock); - __wt_timestamp_set(txn_state->commit_timestamp, - txn->commit_timestamp); - __wt_writeunlock(session, &txn_global->rwlock); - F_SET(txn, WT_TXN_HAS_TS_COMMIT); - } + __wt_txn_set_commit_timestamp(session); #else WT_RET_MSG(session, EINVAL, "commit_timestamp requires a " "version of WiredTiger built with timestamp support"); @@ -300,3 +297,110 @@ __wt_txn_set_timestamp(WT_SESSION_IMPL *session, const char *cfg[]) return (0); } + +/* + * __wt_txn_set_commit_timestamp -- + * Publish a transaction's commit timestamp. + */ +void +__wt_txn_set_commit_timestamp(WT_SESSION_IMPL *session) +{ + WT_TXN *prev, *txn; + WT_TXN_GLOBAL *txn_global; + + txn = &session->txn; + txn_global = &S2C(session)->txn_global; + + if (F_ISSET(txn, WT_TXN_HAS_TS_COMMIT)) + return; + + __wt_writelock(session, &txn_global->commit_timestamp_rwlock); + for (prev = TAILQ_LAST(&txn_global->commit_timestamph, __wt_txn_cts_qh); + prev != NULL && __wt_timestamp_cmp( + prev->commit_timestamp, txn->commit_timestamp) > 0; + prev = TAILQ_PREV(prev, __wt_txn_cts_qh, commit_timestampq)) + ; + if (prev == NULL) + TAILQ_INSERT_HEAD( + &txn_global->commit_timestamph, txn, commit_timestampq); + else + TAILQ_INSERT_AFTER(&txn_global->commit_timestamph, + prev, txn, commit_timestampq); + __wt_writeunlock(session, &txn_global->commit_timestamp_rwlock); + F_SET(txn, WT_TXN_HAS_TS_COMMIT); +} + +/* + * __wt_txn_clear_commit_timestamp -- + * Clear a transaction's published commit timestamp. + */ +void +__wt_txn_clear_commit_timestamp(WT_SESSION_IMPL *session) +{ + WT_TXN *txn; + WT_TXN_GLOBAL *txn_global; + + txn = &session->txn; + txn_global = &S2C(session)->txn_global; + + if (!F_ISSET(txn, WT_TXN_HAS_TS_COMMIT)) + return; + + __wt_writelock(session, &txn_global->commit_timestamp_rwlock); + TAILQ_REMOVE(&txn_global->commit_timestamph, txn, commit_timestampq); + __wt_writeunlock(session, &txn_global->commit_timestamp_rwlock); +} + +/* + * __wt_txn_set_read_timestamp -- + * Publish a transaction's read timestamp. + */ +void +__wt_txn_set_read_timestamp(WT_SESSION_IMPL *session) +{ + WT_TXN *prev, *txn; + WT_TXN_GLOBAL *txn_global; + + txn = &session->txn; + txn_global = &S2C(session)->txn_global; + + if (F_ISSET(txn, WT_TXN_HAS_TS_READ)) + return; + + __wt_writelock(session, &txn_global->read_timestamp_rwlock); + for (prev = TAILQ_LAST(&txn_global->read_timestamph, __wt_txn_rts_qh); + prev != NULL && __wt_timestamp_cmp( + prev->read_timestamp, txn->read_timestamp) > 0; + prev = TAILQ_PREV(prev, __wt_txn_rts_qh, read_timestampq)) + ; + if (prev == NULL) + TAILQ_INSERT_HEAD( + &txn_global->read_timestamph, txn, read_timestampq); + else + TAILQ_INSERT_AFTER( + &txn_global->read_timestamph, prev, txn, read_timestampq); + __wt_writeunlock(session, &txn_global->read_timestamp_rwlock); + F_SET(txn, WT_TXN_HAS_TS_READ); +} + +/* + * __wt_txn_clear_read_timestamp -- + * Clear a transaction's published read timestamp. + */ +void +__wt_txn_clear_read_timestamp(WT_SESSION_IMPL *session) +{ + WT_TXN *txn; + WT_TXN_GLOBAL *txn_global; + + txn = &session->txn; + txn_global = &S2C(session)->txn_global; + + if (!F_ISSET(txn, WT_TXN_HAS_TS_READ)) + return; + + __wt_writelock(session, &txn_global->read_timestamp_rwlock); + TAILQ_REMOVE(&txn_global->read_timestamph, txn, read_timestampq); + __wt_writeunlock(session, &txn_global->read_timestamp_rwlock); + F_CLR(txn, WT_TXN_HAS_TS_READ); +} diff --git a/src/third_party/wiredtiger/src/utilities/util.h b/src/third_party/wiredtiger/src/utilities/util.h index 0238915df07..adf3f844295 100644 --- a/src/third_party/wiredtiger/src/utilities/util.h +++ b/src/third_party/wiredtiger/src/utilities/util.h @@ -32,6 +32,7 @@ int util_cerr(WT_CURSOR *, const char *, int); int util_compact(WT_SESSION *, int, char *[]); void util_copyright(void); int util_create(WT_SESSION *, int, char *[]); +int util_downgrade(WT_SESSION *, WT_CONNECTION *, int, char *[]); int util_drop(WT_SESSION *, int, char *[]); int util_dump(WT_SESSION *, int, char *[]); int util_err(WT_SESSION *, int, const char *, ...) diff --git a/src/third_party/wiredtiger/src/utilities/util_downgrade.c b/src/third_party/wiredtiger/src/utilities/util_downgrade.c new file mode 100644 index 00000000000..4263727242c --- /dev/null +++ b/src/third_party/wiredtiger/src/utilities/util_downgrade.c @@ -0,0 +1,57 @@ +/*- + * Copyright (c) 2014-2017 MongoDB, Inc. + * Copyright (c) 2008-2014 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "util.h" + +static int usage(void); + +int +util_downgrade(WT_SESSION *session, WT_CONNECTION *conn, int argc, char *argv[]) +{ + WT_DECL_RET; + int ch; + char config_str[128], *release; + + release = NULL; + while ((ch = __wt_getopt(progname, argc, argv, "V:")) != EOF) + switch (ch) { + case 'V': + release = __wt_optarg; + break; + case '?': + default: + return (usage()); + } + argc -= __wt_optind; + argv += __wt_optind; + + /* + * The release argument is required. + * There should not be any more arguments. + */ + if (argc != 0 || release == NULL) + return (usage()); + + if ((ret = __wt_snprintf(config_str, sizeof(config_str), + "compatibility=(release=%s)", release)) != 0) + return (util_err(session, ret, NULL)); + if ((ret = conn->reconfigure(conn, config_str)) != 0) + return (util_err(session, ret, "conn.downgrade")); + + return (0); +} + +static int +usage(void) +{ + (void)fprintf(stderr, + "usage: %s %s " + "downgrade -V release\n", + progname, usage_prefix); + return (1); +} diff --git a/src/third_party/wiredtiger/src/utilities/util_list.c b/src/third_party/wiredtiger/src/utilities/util_list.c index 72888e03183..dcb1c97b529 100644 --- a/src/third_party/wiredtiger/src/utilities/util_list.c +++ b/src/third_party/wiredtiger/src/utilities/util_list.c @@ -231,7 +231,8 @@ list_print_checkpoint(WT_SESSION *session, const char *key) if (ci.root_size != 0) { printf("\t\t" "root offset: %" PRIuMAX " (0x%" PRIxMAX ")\n", - (intmax_t)ci.root_offset, (intmax_t)ci.root_offset); + (uintmax_t)ci.root_offset, + (uintmax_t)ci.root_offset); printf("\t\t" "root size: %" PRIu32 " (0x%" PRIx32 ")\n", ci.root_size, ci.root_size); diff --git a/src/third_party/wiredtiger/src/utilities/util_main.c b/src/third_party/wiredtiger/src/utilities/util_main.c index 010af63ea30..a068bfb320d 100644 --- a/src/third_party/wiredtiger/src/utilities/util_main.c +++ b/src/third_party/wiredtiger/src/utilities/util_main.c @@ -41,6 +41,7 @@ usage(void) "\t" "compact\t compact an object\n" "\t" "copyright copyright information\n" "\t" "create\t create an object\n" + "\t" "downgrade\t downgrade an database\n" "\t" "drop\t drop an object\n" "\t" "dump\t dump an object\n" "\t" "list\t list database objects\n" @@ -66,7 +67,8 @@ main(int argc, char *argv[]) WT_SESSION *session; size_t len; int ch, major_v, minor_v, tret, (*func)(WT_SESSION *, int, char *[]); - bool logoff, recover; + int (*cfunc)(WT_SESSION *, WT_CONNECTION *, int, char *[]); + bool logoff, needconn, recover; char *p, *secretkey; const char *cmd_config, *config, *p1, *p2, *p3, *rec_config; @@ -80,6 +82,8 @@ main(int argc, char *argv[]) ++progname; command = ""; + needconn = false; + /* Check the version against the library build. */ (void)wiredtiger_version(&major_v, & minor_v, NULL); if (major_v != WIREDTIGER_VERSION_MAJOR || @@ -156,6 +160,7 @@ main(int argc, char *argv[]) __wt_optreset = __wt_optind = 1; func = NULL; + cfunc = NULL; switch (command[0]) { case 'a': if (strcmp(command, "alter") == 0) @@ -177,7 +182,10 @@ main(int argc, char *argv[]) } break; case 'd': - if (strcmp(command, "drop") == 0) + if (strcmp(command, "downgrade") == 0) { + cfunc = util_downgrade; + needconn = true; + } else if (strcmp(command, "drop") == 0) func = util_drop; else if (strcmp(command, "dump") == 0) func = util_dump; @@ -234,7 +242,7 @@ main(int argc, char *argv[]) default: break; } - if (func == NULL) { + if (func == NULL && cfunc == NULL) { usage(); goto err; } @@ -278,7 +286,10 @@ main(int argc, char *argv[]) } /* Call the function. */ - ret = func(session, argc, argv); + if (needconn) + ret = cfunc(session, conn, argc, argv); + else + ret = func(session, argc, argv); if (0) { err: ret = 1; diff --git a/src/third_party/wiredtiger/test/csuite/rwlock/main.c b/src/third_party/wiredtiger/test/csuite/rwlock/main.c index 3610953a7eb..f1bad3f538f 100644 --- a/src/third_party/wiredtiger/test/csuite/rwlock/main.c +++ b/src/third_party/wiredtiger/test/csuite/rwlock/main.c @@ -73,13 +73,12 @@ main(int argc, char *argv[]) testutil_check(__wt_rwlock_init(NULL, &rwlock)); testutil_check(pthread_rwlock_init(&p_rwlock, NULL)); - testutil_check(pthread_create( - &dump_id, NULL, thread_dump, (void *)opts)); + testutil_check(pthread_create(&dump_id, NULL, thread_dump, opts)); __wt_epoch(NULL, &ts); for (i = 0; i < (int)opts->nthreads; ++i) - testutil_check(pthread_create( - &id[i], NULL, thread_rwlock, (void *)opts)); + testutil_check( + pthread_create(&id[i], NULL, thread_rwlock, opts)); while (--i >= 0) testutil_check(pthread_join(id[i], NULL)); diff --git a/src/third_party/wiredtiger/test/csuite/wt1965_col_efficiency/main.c b/src/third_party/wiredtiger/test/csuite/wt1965_col_efficiency/main.c index e6801d8d37e..265fef8e76f 100644 --- a/src/third_party/wiredtiger/test/csuite/wt1965_col_efficiency/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt1965_col_efficiency/main.c @@ -147,8 +147,8 @@ main(int argc, char *argv[]) testutil_check(session->close(session, NULL)); for (t = 0; t < NR_THREADS; ++t) - testutil_check(pthread_create( - &thr[t], NULL, thread_func, (void *)opts)); + testutil_check( + pthread_create(&thr[t], NULL, thread_func, opts)); for (t = 0; t < NR_THREADS; ++t) (void)pthread_join(thr[t], NULL); diff --git a/src/third_party/wiredtiger/test/csuite/wt2246_col_append/main.c b/src/third_party/wiredtiger/test/csuite/wt2246_col_append/main.c index c71edf6a0a9..bf2d4b8ffc4 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2246_col_append/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2246_col_append/main.c @@ -141,8 +141,8 @@ main(int argc, char *argv[]) id = 0; for (i = 0; i < opts->n_append_threads; ++i, ++id) { printf("append: %" PRIu64 "\n", id); - testutil_check(pthread_create( - &idlist[id], NULL, thread_append, (void *)opts)); + testutil_check( + pthread_create(&idlist[id], NULL, thread_append, opts)); } for (i = 0; i < id; ++i) diff --git a/src/third_party/wiredtiger/test/csuite/wt2323_join_visibility/main.c b/src/third_party/wiredtiger/test/csuite/wt2323_join_visibility/main.c index 89f9eeefb69..3f313a5bfde 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2323_join_visibility/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2323_join_visibility/main.c @@ -186,8 +186,8 @@ test_join(TEST_OPTS *opts, SHARED_OPTS *sharedopts, bool bloom, insert_args[i].nthread = N_INSERT_THREAD; insert_args[i].testopts = opts; insert_args[i].sharedopts = sharedopts; - testutil_check(pthread_create(&insert_tid[i], NULL, - thread_insert, (void *)&insert_args[i])); + testutil_check(pthread_create( + &insert_tid[i], NULL, thread_insert, &insert_args[i])); } for (i = 0; i < N_JOIN_THREAD; ++i) { @@ -195,8 +195,8 @@ test_join(TEST_OPTS *opts, SHARED_OPTS *sharedopts, bool bloom, join_args[i].nthread = N_JOIN_THREAD; join_args[i].testopts = opts; join_args[i].sharedopts = sharedopts; - testutil_check(pthread_create(&join_tid[i], NULL, - thread_join, (void *)&join_args[i])); + testutil_check(pthread_create( + &join_tid[i], NULL, thread_join, &join_args[i])); } /* diff --git a/src/third_party/wiredtiger/test/csuite/wt2535_insert_race/main.c b/src/third_party/wiredtiger/test/csuite/wt2535_insert_race/main.c index 92e69c5920c..f5848203f37 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2535_insert_race/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2535_insert_race/main.c @@ -81,8 +81,8 @@ main(int argc, char *argv[]) testutil_check(c->close(c)); cs = clock(); for (i = 0; i < (int)opts->nthreads; ++i) { - testutil_check(pthread_create( - &id[i], NULL, thread_insert_race, (void *)opts)); + testutil_check( + pthread_create(&id[i], NULL, thread_insert_race, opts)); } while (--i >= 0) testutil_check(pthread_join(id[i], NULL)); diff --git a/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c b/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c index 69e6b871dae..a671464d7e2 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c @@ -149,8 +149,8 @@ main(int argc, char *argv[]) insert_args[i].nthread = N_INSERT_THREAD; insert_args[i].testopts = opts; insert_args[i].sharedopts = sharedopts; - testutil_check(pthread_create(&insert_tid[i], NULL, - thread_insert, (void *)&insert_args[i])); + testutil_check(pthread_create( + &insert_tid[i], NULL, thread_insert, &insert_args[i])); } for (i = 0; i < N_GET_THREAD; ++i) { @@ -158,8 +158,8 @@ main(int argc, char *argv[]) get_args[i].nthread = N_GET_THREAD; get_args[i].testopts = opts; get_args[i].sharedopts = sharedopts; - testutil_check(pthread_create(&get_tid[i], NULL, - thread_get, (void *)&get_args[i])); + testutil_check(pthread_create( + &get_tid[i], NULL, thread_get, &get_args[i])); } /* diff --git a/src/third_party/wiredtiger/test/csuite/wt3363_checkpoint_op_races/main.c b/src/third_party/wiredtiger/test/csuite/wt3363_checkpoint_op_races/main.c index 5c9c3d96454..ffef7f5fa9f 100644 --- a/src/third_party/wiredtiger/test/csuite/wt3363_checkpoint_op_races/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt3363_checkpoint_op_races/main.c @@ -41,9 +41,9 @@ * any operation taking longer than 1/2 the delay time, we abort dumping a core * file which can be used to determine what operation was blocked. */ -void* do_checkpoints(void *); -void* do_ops(void *); -void* monitor(void *); +static WT_THREAD_RET do_checkpoints(void *); +static WT_THREAD_RET do_ops(void *); +static WT_THREAD_RET monitor(void *); /* * Time delay to introduce into checkpoints in seconds. Should be at-least @@ -51,8 +51,8 @@ void* monitor(void *); * this is set to 10 seconds and we expect no single operation to take longer * than 5 seconds. */ -#define MAX_EXECUTION_TIME 10 -#define N_THREADS 10 +#define MAX_EXECUTION_TIME 10 +#define N_THREADS 10 /* * Number of seconds to execute for. Initially set to 15 minutes, as we need to @@ -60,7 +60,7 @@ void* monitor(void *); * testing 5 minutes was enough to reproduce the issue, so we run for 3x that * here to ensure we reproduce before declaring success. */ -#define RUNTIME 900.0 +#define RUNTIME 900.0 static WT_EVENT_HANDLER event_handler = { handle_op_error, @@ -72,17 +72,10 @@ static WT_EVENT_HANDLER event_handler = { int main(int argc, char *argv[]) { - TEST_PER_THREAD_OPTS thread_args[N_THREADS]; TEST_OPTS *opts, _opts; + TEST_PER_THREAD_OPTS thread_args[N_THREADS]; pthread_t ckpt_thread, mon_thread, threads[N_THREADS]; int i; - bool diagnostic; - -#ifdef HAVE_DIAGNOSTIC - diagnostic = true; -#else - diagnostic = false; -#endif /* * This test should not run unless we have compiled with diagnostic @@ -90,7 +83,15 @@ main(int argc, char *argv[]) * attempting to set the option to add the delays to checkpoints if * diagnostic mode is not enable and runs for 15 minutes. */ - if (!testutil_is_flag_set("WT3363_CHECKPOINT_OP_RACES") || !diagnostic) +#if !defined(HAVE_DIAGNOSTIC) + /* + * Put the return in a conditional, otherwise some compilers will + * complain that code beyond the return is unreachable. + */ + if (true) + return (EXIT_SUCCESS); +#endif + if (!testutil_is_flag_set("WT3363_CHECKPOINT_OP_RACES")) return (EXIT_SUCCESS); opts = &_opts; @@ -105,14 +106,14 @@ main(int argc, char *argv[]) &opts->conn)); testutil_check(pthread_create( - &ckpt_thread, NULL, do_checkpoints, (void *)opts)); + &ckpt_thread, NULL, do_checkpoints, opts)); for (i = 0; i < N_THREADS; ++i) { thread_args[i].testopts = opts; thread_args[i].thread_counter = 0; thread_args[i].threadnum = i; testutil_check(pthread_create( - &threads[i], NULL, do_ops, (void *)&thread_args[i])); + &threads[i], NULL, do_ops, &thread_args[i])); } /* @@ -120,8 +121,7 @@ main(int argc, char *argv[]) * This thread will need to monitor each threads counter to track if it * is stuck. */ - testutil_check( - pthread_create(&mon_thread, NULL, monitor, &thread_args)); + testutil_check(pthread_create(&mon_thread, NULL, monitor, thread_args)); for (i = 0; i < N_THREADS; ++i) testutil_check(pthread_join(threads[i], NULL)); @@ -139,7 +139,7 @@ main(int argc, char *argv[]) /* * Function for repeatedly running checkpoint operations. */ -void * +static WT_THREAD_RET do_checkpoints(void *_opts) { TEST_OPTS *opts; @@ -159,7 +159,7 @@ do_checkpoints(void *_opts) if (ret != EBUSY && ret != ENOENT) testutil_die(ret, "session.checkpoint"); - testutil_check(session->close(session, NULL)); + testutil_check(session->close(session, NULL)); /* * A short sleep to let operations process and avoid back to @@ -169,21 +169,21 @@ do_checkpoints(void *_opts) (void)time(&now); } - return (NULL); + return (WT_THREAD_RET_VALUE); } /* * Function to monitor running operations and abort to dump core in the event * that we catch an operation running long. */ -void * +static WT_THREAD_RET monitor(void *args) { TEST_PER_THREAD_OPTS *thread_args; time_t now, start; int ctr, i, last_ops[N_THREADS]; - thread_args = (TEST_PER_THREAD_OPTS*)args; + thread_args = (TEST_PER_THREAD_OPTS *)args; (void)time(&start); (void)time(&now); @@ -191,13 +191,12 @@ monitor(void *args) memset(last_ops, 0, sizeof(int) + N_THREADS); while (difftime(now, start) < RUNTIME) { - /* * Checkpoints will run for slightly over MAX_EXECUTION_TIME. * MAX_EXECUTION_TIME should always be long enough that we can * complete any single operation in 1/2 that time. */ - sleep(MAX_EXECUTION_TIME/2); + sleep(MAX_EXECUTION_TIME / 2); for (i = 0; i < N_THREADS; i++) { ctr = thread_args[i].thread_counter; @@ -217,20 +216,20 @@ monitor(void *args) else { printf("Thread %d had a task running" " for more than %d seconds\n", - i, MAX_EXECUTION_TIME/2); + i, MAX_EXECUTION_TIME / 2); abort(); } } (void)time(&now); } - return (NULL); + return (WT_THREAD_RET_VALUE); } /* * Worker thread. Executes random operations from the set of 6. */ -void * +static WT_THREAD_RET do_ops(void *args) { WT_RAND_STATE rnd; @@ -242,27 +241,27 @@ do_ops(void *args) while (difftime(now, start) < RUNTIME) { switch (__wt_random(&rnd) % 6) { - case 0: - op_bulk(args); - break; - case 1: - op_create(args); - break; - case 2: - op_cursor(args); - break; - case 3: - op_drop(args); - break; - case 4: - op_bulk_unique(args); - break; - case 5: - op_create_unique(args); - break; + case 0: + op_bulk(args); + break; + case 1: + op_create(args); + break; + case 2: + op_cursor(args); + break; + case 3: + op_drop(args); + break; + case 4: + op_bulk_unique(args); + break; + case 5: + op_create_unique(args); + break; } (void)time(&now); } - return (NULL); + return (WT_THREAD_RET_VALUE); } diff --git a/src/third_party/wiredtiger/test/fops/fops.c b/src/third_party/wiredtiger/test/fops/fops.c index 911bfba55ad..cf182d757e2 100644 --- a/src/third_party/wiredtiger/test/fops/fops.c +++ b/src/third_party/wiredtiger/test/fops/fops.c @@ -161,7 +161,7 @@ print_stats(u_int nthreads) s = run_stats; for (id = 0; id < nthreads; ++id, ++s) printf( - "%2d:" + "%2u:" "\t" "bulk %3d, checkpoint %3d, create %3d, cursor %3d,\n" "\t" "drop %3d, rebalance %3d, upgrade %3d, verify %3d\n", id, s->bulk + s->bulk_unique, s->ckpt, diff --git a/src/third_party/wiredtiger/test/format/bulk.c b/src/third_party/wiredtiger/test/format/bulk.c index 0e7c54516e6..5d236ec5b42 100644 --- a/src/third_party/wiredtiger/test/format/bulk.c +++ b/src/third_party/wiredtiger/test/format/bulk.c @@ -58,8 +58,14 @@ wts_load(void) if (g.c_reverse) is_bulk = false; - testutil_check(session->open_cursor(session, g.uri, NULL, - is_bulk ? "bulk,append" : NULL, &cursor)); + /* + * open_cursor can return EBUSY if concurrent with a metadata + * operation, retry in that case. + */ + while ((ret = session->open_cursor(session, g.uri, NULL, + is_bulk ? "bulk,append" : NULL, &cursor)) == EBUSY) + __wt_yield(); + testutil_check(ret); /* Set up the key/value buffers. */ key_gen_setup(&key); diff --git a/src/third_party/wiredtiger/test/format/compact.c b/src/third_party/wiredtiger/test/format/compact.c index f2fa7521946..ba5efa5e53d 100644 --- a/src/third_party/wiredtiger/test/format/compact.c +++ b/src/third_party/wiredtiger/test/format/compact.c @@ -63,8 +63,12 @@ compact(void *arg) if (g.workers_finished) break; - if ((ret = session->compact( - session, g.uri, NULL)) != 0 && ret != WT_ROLLBACK) + /* + * Compact can return EBUSY if concurrent with alter. + */ + while ((ret = session->compact(session, g.uri, NULL)) == EBUSY) + __wt_yield(); + if (ret != 0 && ret != WT_ROLLBACK) testutil_die(ret, "session.compact"); } diff --git a/src/third_party/wiredtiger/test/format/config.c b/src/third_party/wiredtiger/test/format/config.c index 2685438af00..f7aad2ae0a7 100644 --- a/src/third_party/wiredtiger/test/format/config.c +++ b/src/third_party/wiredtiger/test/format/config.c @@ -30,6 +30,7 @@ #include "config.h" static void config_checksum(void); +static void config_compatibility(void); static void config_compression(const char *); static void config_encryption(void); static const char *config_file_type(u_int); @@ -40,6 +41,7 @@ static int config_is_perm(const char *); static void config_isolation(void); static void config_lrt(void); static void config_map_checksum(const char *, u_int *); +static void config_map_compatibility(const char *, u_int *); static void config_map_compression(const char *, u_int *); static void config_map_encryption(const char *, u_int *); static void config_map_file_type(const char *, u_int *); @@ -158,6 +160,7 @@ config_setup(void) g.c_threads = 1; config_checksum(); + config_compatibility(); config_compression("compression"); config_compression("logging_compression"); config_encryption(); @@ -308,6 +311,36 @@ config_compression(const char *conf_name) } /* + * config_compatibility -- + * Compatibility configuration. + */ +static void +config_compatibility(void) +{ + /* + * Compatibility is only relevant if logging is enabled. + * Skip it no matter what if we're not logging. + */ + if (g.c_logging == 0) { + config_single("compatibility=none", 0); + return; + } + /* Choose a compatibility mode if nothing was specified. */ + if (!config_is_perm("compatibility")) + switch (mmrand(NULL, 1, 10)) { + case 1: /* 10% */ + config_single("compatibility=v1", 0); + break; + case 2: /* 10% */ + config_single("compatibility=v2", 0); + break; + default: /* 80% */ + config_single("compatibility=none", 0); + break; + } +} + +/* * config_encryption -- * Encryption configuration. */ @@ -381,10 +414,14 @@ config_in_memory_check(void) return; /* Turn off a lot of stuff. */ + if (!config_is_perm("alter")) + config_single("alter=off", 0); if (!config_is_perm("backups")) config_single("backups=off", 0); if (!config_is_perm("checkpoints")) config_single("checkpoints=off", 0); + if (!config_is_perm("compatibility")) + config_single("compatibility=none", 0); if (!config_is_perm("compression")) config_single("compression=none", 0); if (!config_is_perm("logging")) @@ -732,6 +769,10 @@ config_single(const char *s, int perm) config_map_checksum(ep, &g.c_checksum_flag); *cp->vstr = dstrdup(ep); } else if (strncmp( + s, "compatibility", strlen("compatibility")) == 0) { + config_map_compatibility(ep, &g.c_compat_flag); + *cp->vstr = dstrdup(ep); + } else if (strncmp( s, "compression", strlen("compression")) == 0) { config_map_compression(ep, &g.c_compression_flag); *cp->vstr = dstrdup(ep); @@ -827,6 +868,24 @@ config_map_checksum(const char *s, u_int *vp) } /* + * config_map_compatibility -- + * Map a compatibility configuration to a flag. + */ +static void +config_map_compatibility(const char *s, u_int *vp) +{ + if (strcmp(s, "none") == 0) + *vp = COMPAT_NONE; + else if (strcmp(s, "v1") == 0) + *vp = COMPAT_V1; + else if (strcmp(s, "v2") == 0) + *vp = COMPAT_V2; + else + testutil_die(EINVAL, + "illegal compatibility configuration: %s", s); +} + +/* * config_map_compression -- * Map a compression configuration to a flag. */ diff --git a/src/third_party/wiredtiger/test/format/config.h b/src/third_party/wiredtiger/test/format/config.h index ea30986f453..ef3fcf8c028 100644 --- a/src/third_party/wiredtiger/test/format/config.h +++ b/src/third_party/wiredtiger/test/format/config.h @@ -117,6 +117,10 @@ static CONFIG c[] = { "if compaction is running", /* 10% */ C_BOOL, 10, 0, 0, &g.c_compact, NULL }, + { "compatibility", + "log file compatibility (none | v1 | v2 )", + C_IGNORE|C_STRING, 0, 0, 0, NULL, &g.c_compat }, + { "compression", "type of compression " COMPRESSION_LIST, C_IGNORE|C_STRING, 0, 0, 0, NULL, &g.c_compression }, diff --git a/src/third_party/wiredtiger/test/format/format.h b/src/third_party/wiredtiger/test/format/format.h index 7860bb539ed..30246ce69a1 100644 --- a/src/third_party/wiredtiger/test/format/format.h +++ b/src/third_party/wiredtiger/test/format/format.h @@ -155,6 +155,7 @@ typedef struct { char *c_checksum; uint32_t c_chunk_size; uint32_t c_compact; + char *c_compat; char *c_compression; char *c_config_open; uint32_t c_data_extend; @@ -220,6 +221,11 @@ typedef struct { #define CHECKSUM_UNCOMPRESSED 3 u_int c_checksum_flag; /* Checksum flag value */ +#define COMPAT_NONE 1 +#define COMPAT_V1 2 +#define COMPAT_V2 3 + u_int c_compat_flag; /* Compatibility flag value */ + #define COMPRESS_NONE 1 #define COMPRESS_LZ4 2 #define COMPRESS_LZ4_NO_RAW 3 @@ -285,6 +291,7 @@ void bdb_update(const void *, size_t, const void *, size_t); WT_THREAD_RET alter(void *); WT_THREAD_RET backup(void *); WT_THREAD_RET compact(void *); +WT_THREAD_RET compat(void *); void config_clear(void); void config_error(void); void config_file(const char *); diff --git a/src/third_party/wiredtiger/test/format/lrt.c b/src/third_party/wiredtiger/test/format/lrt.c index b9622cdb635..5073d5aad03 100644 --- a/src/third_party/wiredtiger/test/format/lrt.c +++ b/src/third_party/wiredtiger/test/format/lrt.c @@ -59,8 +59,14 @@ lrt(void *arg) /* Open a session and cursor. */ conn = g.wts_conn; testutil_check(conn->open_session(conn, NULL, NULL, &session)); - testutil_check(session->open_cursor( - session, g.uri, NULL, NULL, &cursor)); + /* + * open_cursor can return EBUSY if concurrent with a metadata + * operation, retry in that case. + */ + while ((ret = session->open_cursor( + session, g.uri, NULL, NULL, &cursor)) == EBUSY) + __wt_yield(); + testutil_check(ret); for (pinned = 0;;) { if (pinned) { diff --git a/src/third_party/wiredtiger/test/format/ops.c b/src/third_party/wiredtiger/test/format/ops.c index 76b1261e9ab..f8c275bb67d 100644 --- a/src/third_party/wiredtiger/test/format/ops.c +++ b/src/third_party/wiredtiger/test/format/ops.c @@ -62,7 +62,7 @@ wts_ops(int lastrun) TINFO **tinfo_list, *tinfo, total; WT_CONNECTION *conn; WT_SESSION *session; - wt_thread_t alter_tid, backup_tid, compact_tid, lrt_tid; + wt_thread_t alter_tid, backup_tid, compact_tid, compat_tid, lrt_tid; int64_t fourths, thread_ops; uint32_t i; int running; @@ -73,6 +73,7 @@ wts_ops(int lastrun) memset(&alter_tid, 0, sizeof(alter_tid)); memset(&backup_tid, 0, sizeof(backup_tid)); memset(&compact_tid, 0, sizeof(compact_tid)); + memset(&compat_tid, 0, sizeof(compat_tid)); memset(&lrt_tid, 0, sizeof(lrt_tid)); /* @@ -138,6 +139,9 @@ wts_ops(int lastrun) if (g.c_compact) testutil_check( __wt_thread_create(NULL, &compact_tid, compact, NULL)); + if (g.c_compat_flag != COMPAT_NONE) + testutil_check( + __wt_thread_create(NULL, &compat_tid, compat, NULL)); if (!SINGLETHREADED && g.c_long_running_txn) testutil_check(__wt_thread_create(NULL, &lrt_tid, lrt, NULL)); @@ -197,7 +201,7 @@ wts_ops(int lastrun) free(tinfo_list[i]); free(tinfo_list); - /* Wait for the backup, compaction, long-running reader threads. */ + /* Wait for the other threads. */ g.workers_finished = 1; if (g.c_alter) testutil_check(__wt_thread_join(NULL, alter_tid)); @@ -205,6 +209,8 @@ wts_ops(int lastrun) testutil_check(__wt_thread_join(NULL, backup_tid)); if (g.c_compact) testutil_check(__wt_thread_join(NULL, compact_tid)); + if (g.c_compat_flag != COMPAT_NONE) + testutil_check(__wt_thread_join(NULL, compat_tid)); if (!SINGLETHREADED && g.c_long_running_txn) testutil_check(__wt_thread_join(NULL, lrt_tid)); g.workers_finished = 0; @@ -525,8 +531,14 @@ ops(void *arg) */ if (!SINGLETHREADED && !DATASOURCE("lsm") && ckpt_available && mmrand(&tinfo->rnd, 1, 10) == 1) { - testutil_check(session->open_cursor(session, - g.uri, NULL, ckpt_name, &cursor)); + /* + * open_cursor can return EBUSY if concurrent + * with a metadata operation, retry. + */ + while ((ret = session->open_cursor(session, + g.uri, NULL, ckpt_name, &cursor)) == EBUSY) + __wt_yield(); + testutil_check(ret); /* Pick the next session/cursor close/open. */ session_op += 250; @@ -537,9 +549,13 @@ ops(void *arg) /* * Configure "append", in the case of column * stores, we append when inserting new rows. + * open_cursor can return EBUSY if concurrent + * with a metadata operation, retry. */ - testutil_check(session->open_cursor( - session, g.uri, NULL, "append", &cursor)); + while ((ret = session->open_cursor(session, + g.uri, NULL, "append", &cursor)) == EBUSY) + __wt_yield(); + testutil_check(ret); /* Pick the next session/cursor close/open. */ session_op += mmrand(&tinfo->rnd, 100, 5000); @@ -924,8 +940,14 @@ wts_read_scan(void) /* Open a session and cursor pair. */ testutil_check(conn->open_session(conn, NULL, NULL, &session)); - testutil_check( - session->open_cursor(session, g.uri, NULL, NULL, &cursor)); + /* + * open_cursor can return EBUSY if concurrent with a metadata + * operation, retry in that case. + */ + while ((ret = session->open_cursor( + session, g.uri, NULL, NULL, &cursor)) == EBUSY) + __wt_yield(); + testutil_check(ret); /* Check a random subset of the records using the key. */ for (last_keyno = keyno = 0; keyno < g.key_cnt;) { diff --git a/src/third_party/wiredtiger/test/format/t.c b/src/third_party/wiredtiger/test/format/t.c index 0cfe4e40421..c70f6facfc5 100644 --- a/src/third_party/wiredtiger/test/format/t.c +++ b/src/third_party/wiredtiger/test/format/t.c @@ -250,7 +250,7 @@ main(int argc, char *argv[]) /* Overwrite the progress line with a completion line. */ if (!g.c_quiet) printf("\r%78s\r", " "); - printf("%4d: %s, %s (%.0f seconds)\n", + printf("%4" PRIu32 ": %s, %s (%.0f seconds)\n", g.run_cnt, g.c_data_source, g.c_file_type, difftime(time(NULL), start)); fflush(stdout); diff --git a/src/third_party/wiredtiger/test/format/util.c b/src/third_party/wiredtiger/test/format/util.c index f09bb160893..6d24073da1a 100644 --- a/src/third_party/wiredtiger/test/format/util.c +++ b/src/third_party/wiredtiger/test/format/util.c @@ -134,7 +134,7 @@ key_gen_insert(WT_RAND_STATE *rnd, WT_ITEM *key, uint64_t keyno) "11", "12", "13", "14", "15" }; - key_gen_common(key, keyno, suffix[mmrand(rnd, 1, 15) - 1]); + key_gen_common(key, keyno, suffix[mmrand(rnd, 0, 14)]); } static uint32_t val_dup_data_len; /* Length of duplicate data items */ @@ -408,8 +408,9 @@ path_setup(const char *home) uint32_t rng(WT_RAND_STATE *rnd) { - char buf[64]; - uint32_t r; + u_long ulv; + uint32_t v; + char *endptr, buf[64]; /* * Threaded operations have their own RNG information, otherwise we @@ -439,16 +440,19 @@ rng(WT_RAND_STATE *rnd) testutil_die(errno, "random number log"); } - return ((uint32_t)strtoul(buf, NULL, 10)); + errno = 0; + ulv = strtoul(buf, &endptr, 10); + testutil_assert(errno == 0 && endptr[0] == '\n'); + return ((uint32_t)ulv); } - r = __wt_random(rnd); + v = __wt_random(rnd); /* Save and flush the random number so we're up-to-date on error. */ - (void)fprintf(g.randfp, "%" PRIu32 "\n", r); + (void)fprintf(g.randfp, "%" PRIu32 "\n", v); (void)fflush(g.randfp); - return (r); + return (v); } /* @@ -476,6 +480,7 @@ WT_THREAD_RET alter(void *arg) { WT_CONNECTION *conn; + WT_DECL_RET; WT_SESSION *session; u_int period; bool access_value; @@ -501,8 +506,12 @@ alter(void *arg) "access_pattern_hint=%s", access_value ? "random" : "none")); access_value = !access_value; - if (session->alter(session, g.uri, buf) != 0) - break; + /* + * Alter can return EBUSY if concurrent with other operations. + */ + while ((ret = session->alter(session, g.uri, buf)) != 0 && + ret != EBUSY) + testutil_die(ret, "session.alter"); while (period > 0 && !g.workers_finished) { --period; sleep(1); @@ -512,3 +521,45 @@ alter(void *arg) testutil_check(session->close(session, NULL)); return (WT_THREAD_RET_VALUE); } + +#define COMPATSTR_V1 "compatibility=(release=2.6)" +#define COMPATSTR_V2 "compatibility=(release=3.0)" + +/* + * compat -- + * Periodically reconfigure the compatibility option. + */ +WT_THREAD_RET +compat(void *arg) +{ + WT_CONNECTION *conn; + WT_DECL_RET; + u_int count, period; + const char *str; + + (void)(arg); + + conn = g.wts_conn; + str = NULL; + /* + * Perform compatibility swaps at somewhere under 10 seconds (so we + * get at least one done), and then at 7 second intervals. + */ + for (period = mmrand(NULL, 1, 10), count = 0;; ++count, period = 7) { + if (count % 2 == 0) + str = COMPATSTR_V1; + else + str = COMPATSTR_V2; + if ((ret = conn->reconfigure(conn, str)) != 0) + testutil_die(ret, "conn.reconfigure"); + + /* Sleep for short periods so we don't make the run wait. */ + while (period > 0 && !g.workers_finished) { + --period; + sleep(1); + } + if (g.workers_finished) + break; + } + return (WT_THREAD_RET_VALUE); +} diff --git a/src/third_party/wiredtiger/test/format/wts.c b/src/third_party/wiredtiger/test/format/wts.c index 6bfec79436c..6a58cad5403 100644 --- a/src/third_party/wiredtiger/test/format/wts.c +++ b/src/third_party/wiredtiger/test/format/wts.c @@ -72,6 +72,34 @@ compressor(uint32_t compress_flag) } /* + * compatibility -- + * Configure compatibility. + */ +static const char * +compatibility(uint32_t compat_flag) +{ + const char *p; + + p = "unrecognized compatibility flag"; + switch (compat_flag) { + case COMPAT_NONE: + p = ""; + break; + case COMPAT_V1: + p = "2.6"; + break; + case COMPAT_V2: + p = "3.0"; + break; + default: + testutil_die(EINVAL, + "illegal compatibility flag: %#" PRIx32, compat_flag); + /* NOTREACHED */ + } + return (p); +} + +/* * encryptor -- * Configure encryption. */ @@ -191,13 +219,17 @@ wts_open(const char *home, bool set_api, WT_CONNECTION **connp) ",eviction=(threads_max=%" PRIu32 ")", g.c_evict_max); /* Logging configuration. */ - if (g.c_logging) + if (g.c_logging) { CONFIG_APPEND(p, ",log=(enabled=true,archive=%d,prealloc=%d" ",compressor=\"%s\")", g.c_logging_archive ? 1 : 0, g.c_logging_prealloc ? 1 : 0, compressor(g.c_logging_compression_flag)); + CONFIG_APPEND(p, + ",compatibility=(release=%s)", + compatibility(g.c_compat_flag)); + } if (g.c_encryption) CONFIG_APPEND(p, diff --git a/src/third_party/wiredtiger/test/packing/intpack-test.c b/src/third_party/wiredtiger/test/packing/intpack-test.c index e7822015091..172575c3d3d 100644 --- a/src/third_party/wiredtiger/test/packing/intpack-test.c +++ b/src/third_party/wiredtiger/test/packing/intpack-test.c @@ -31,14 +31,14 @@ int main(void) { - const uint8_t *cp; - uint8_t buf[10], *p; uint64_t ncalls, r, r2, s; + uint8_t buf[WT_INTPACK64_MAXSIZE], *p; + const uint8_t *cp; int i; - ncalls = 0; + memset(buf, 0xff, sizeof(buf)); /* -Werror=maybe-uninitialized */ - for (i = 0; i < 10000000; i++) { + for (ncalls = 0, i = 0; i < 10000000; i++) { for (s = 0; s < 50; s += 5) { ++ncalls; r = 1ULL << s; @@ -60,14 +60,11 @@ main(void) cp = buf; memmove(&r2, cp, sizeof(r2)); #endif - if (r != r2) { - fprintf(stderr, "mismatch!\n"); - break; - } + testutil_assert(r == r2); } } - printf("Number of calls: %llu\n", (unsigned long long)ncalls); + printf("Number of calls: %" PRIu64 "\n", ncalls); return (0); } diff --git a/src/third_party/wiredtiger/test/packing/intpack-test2.c b/src/third_party/wiredtiger/test/packing/intpack-test2.c index e216899cebb..13f9ab0d2d7 100644 --- a/src/third_party/wiredtiger/test/packing/intpack-test2.c +++ b/src/third_party/wiredtiger/test/packing/intpack-test2.c @@ -31,7 +31,7 @@ int main(void) { - uint8_t buf[10], *p, *end; + uint8_t buf[WT_INTPACK64_MAXSIZE], *p, *end; int64_t i; for (i = 1; i < 1LL << 60; i <<= 1) { diff --git a/src/third_party/wiredtiger/test/packing/intpack-test3.c b/src/third_party/wiredtiger/test/packing/intpack-test3.c index 00fc80e24a2..dad31fd7f7d 100644 --- a/src/third_party/wiredtiger/test/packing/intpack-test3.c +++ b/src/third_party/wiredtiger/test/packing/intpack-test3.c @@ -35,7 +35,7 @@ void test_value(int64_t val) { const uint8_t *cp; - uint8_t buf[10], *p; + uint8_t buf[WT_INTPACK64_MAXSIZE], *p; int64_t sinput, soutput; uint64_t uinput, uoutput; size_t used_len; @@ -50,8 +50,8 @@ test_value(int64_t val) testutil_check(__wt_vunpack_int(&cp, used_len, &soutput)); /* Ensure we got the correct value back */ if (sinput != soutput) { - fprintf(stderr, "mismatch %" PRIu64 ", %" PRIu64 "\n", - sinput, soutput); + fprintf(stderr, + "mismatch %" PRId64 ", %" PRId64 "\n", sinput, soutput); abort(); } /* Ensure that decoding used the correct amount of buffer */ @@ -74,8 +74,8 @@ test_value(int64_t val) testutil_check(__wt_vunpack_uint(&cp, sizeof(buf), &uoutput)); /* Ensure we got the correct value back */ if (sinput != soutput) { - fprintf(stderr, "mismatch %" PRIu64 ", %" PRIu64 "\n", - sinput, soutput); + fprintf(stderr, + "mismatch %" PRId64 ", %" PRId64 "\n", sinput, soutput); abort(); } /* Ensure that decoding used the correct amount of buffer */ diff --git a/src/third_party/wiredtiger/test/packing/packing-test.c b/src/third_party/wiredtiger/test/packing/packing-test.c index bd48ac7125c..684cdec554a 100644 --- a/src/third_party/wiredtiger/test/packing/packing-test.c +++ b/src/third_party/wiredtiger/test/packing/packing-test.c @@ -51,7 +51,7 @@ check(const char *fmt, ...) printf("%s ", fmt); for (p = buf, end = p + len; p < end; p++) - printf("%02x", *p & 0xff); + printf("%02x", (u_char)*p & 0xff); printf("\n"); } diff --git a/src/third_party/wiredtiger/test/recovery/random-abort.c b/src/third_party/wiredtiger/test/recovery/random-abort.c index 7e76f61bd12..5588da00bd9 100644 --- a/src/third_party/wiredtiger/test/recovery/random-abort.c +++ b/src/third_party/wiredtiger/test/recovery/random-abort.c @@ -38,6 +38,7 @@ static char home[1024]; /* Program working dir */ */ static const char * const uri = "table:main"; static const char * const fs_main = "main.wt"; +static bool compat; static bool inmem; #define MAX_TH 12 @@ -46,6 +47,7 @@ static bool inmem; #define MIN_TIME 10 #define RECORDS_FILE "records-%" PRIu32 +#define ENV_CONFIG_COMPAT ",compatibility=(release=\"2.9\")" #define ENV_CONFIG_DEF \ "create,log=(file_max=10M,enabled)" #define ENV_CONFIG_TXNSYNC \ @@ -167,16 +169,19 @@ fill_db(uint32_t nth) wt_thread_t *thr; uint32_t i; int ret; - const char *envconf; + char envconf[512]; thr = dcalloc(nth, sizeof(*thr)); td = dcalloc(nth, sizeof(WT_THREAD_DATA)); if (chdir(home) != 0) testutil_die(errno, "Child chdir: %s", home); if (inmem) - envconf = ENV_CONFIG_DEF; + strcpy(envconf, ENV_CONFIG_DEF); else - envconf = ENV_CONFIG_TXNSYNC; + strcpy(envconf, ENV_CONFIG_TXNSYNC); + if (compat) + strcat(envconf, ENV_CONFIG_COMPAT); + if ((ret = wiredtiger_open(NULL, NULL, envconf, &conn)) != 0) testutil_die(ret, "wiredtiger_open"); if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) @@ -221,27 +226,30 @@ main(int argc, char *argv[]) FILE *fp; WT_CONNECTION *conn; WT_CURSOR *cursor; - WT_SESSION *session; WT_RAND_STATE rnd; + WT_SESSION *session; + pid_t pid; uint64_t absent, count, key, last_key, middle; uint32_t i, nth, timeout; int ch, status, ret; - pid_t pid; - bool fatal, rand_th, rand_time, verify_only; const char *working_dir; char fname[64], kname[64], statname[1024]; + bool fatal, rand_th, rand_time, verify_only; (void)testutil_set_progname(argv); - inmem = false; + compat = inmem = false; nth = MIN_TH; rand_th = rand_time = true; timeout = MIN_TIME; verify_only = false; working_dir = "WT_TEST.random-abort"; - while ((ch = __wt_getopt(progname, argc, argv, "h:mT:t:v")) != EOF) + while ((ch = __wt_getopt(progname, argc, argv, "Ch:mT:t:v")) != EOF) switch (ch) { + case 'C': + compat = true; + break; case 'h': working_dir = __wt_optarg; break; @@ -291,6 +299,8 @@ main(int argc, char *argv[]) if (nth < MIN_TH) nth = MIN_TH; } + printf("Parent: Compatibility %s in-mem log %s\n", + compat ? "true" : "false", inmem ? "true" : "false"); printf("Parent: Create %" PRIu32 " threads; sleep %" PRIu32 " seconds\n", nth, timeout); /* diff --git a/src/third_party/wiredtiger/test/recovery/smoke.sh b/src/third_party/wiredtiger/test/recovery/smoke.sh index ce0662d3b2b..ba4d77c642b 100755 --- a/src/third_party/wiredtiger/test/recovery/smoke.sh +++ b/src/third_party/wiredtiger/test/recovery/smoke.sh @@ -6,4 +6,6 @@ set -e $TEST_WRAPPER ./random-abort -t 10 -T 5 $TEST_WRAPPER ./random-abort -m -t 10 -T 5 +$TEST_WRAPPER ./random-abort -C -t 10 -T 5 +$TEST_WRAPPER ./random-abort -C -m -t 10 -T 5 $TEST_WRAPPER ./truncated-log diff --git a/src/third_party/wiredtiger/test/recovery/truncated-log.c b/src/third_party/wiredtiger/test/recovery/truncated-log.c index c9d73e0cf48..73ebe7dfae7 100644 --- a/src/third_party/wiredtiger/test/recovery/truncated-log.c +++ b/src/third_party/wiredtiger/test/recovery/truncated-log.c @@ -90,14 +90,17 @@ write_and_read_new(WT_SESSION *session) &optype, &fileid, &logrec_key, &logrec_value)) != 0) testutil_die(errno, "get_value"); /* - * We should never see a record from log file 2. We wrote + * We should never see a record from us in log file 2. We wrote * a record there, but then the record in log file 1 was * truncated to be a partial record, ending the log there. * So everything after that, including everything in log * file 2, is invalid until we get to log file 3 which is where * the post-recovery records will be written. + * The one exception in log file two is the system record for + * the previous log file's LSN. Although it is written by the + * system, we do walk it when using a cursor. */ - if (log_file == 2) + if (log_file == 2 && rectype != WT_LOGREC_SYSTEM) testutil_die(EINVAL, "Found LSN in Log 2"); #if 0 printf("LSN [%" PRIu32 "][%" PRIu32 "].%" PRIu32 @@ -348,9 +351,10 @@ main(int argc, char *argv[]) /* * The max key in the saved file is the key we truncated, but the * key space starts at 0 and we're counting the records here, so we - * expect the max key number of records. + * expect the max key number of records. Add one for the system + * record for the previous LSN that the cursor will see too. */ - if (count > max_key) { + if (count > (max_key + 1)) { printf("expected %" PRIu32 " records found %" PRIu32 "\n", max_key, count); return (EXIT_FAILURE); diff --git a/src/third_party/wiredtiger/test/suite/suite_subprocess.py b/src/third_party/wiredtiger/test/suite/suite_subprocess.py index 86134db5f88..626a6b5efd3 100644 --- a/src/third_party/wiredtiger/test/suite/suite_subprocess.py +++ b/src/third_party/wiredtiger/test/suite/suite_subprocess.py @@ -119,11 +119,13 @@ class suite_subprocess: # Run the wt utility. def runWt(self, args, infilename=None, - outfilename=None, errfilename=None, reopensession=True, failure=False): + outfilename=None, errfilename=None, closeconn=True, + reopensession=True, failure=False): # Close the connection to guarantee everything is flushed, and that # we can open it from another process. - self.close_conn() + if closeconn: + self.close_conn() wtoutname = outfilename or "wt.out" wterrname = errfilename or "wt.err" @@ -166,5 +168,5 @@ class suite_subprocess: self.check_empty_file(wtoutname) # Reestablish the connection if needed - if reopensession: + if reopensession and closeconn: self.open_conn() diff --git a/src/third_party/wiredtiger/test/suite/test_alter02.py b/src/third_party/wiredtiger/test/suite/test_alter02.py new file mode 100644 index 00000000000..3c5b67aeba5 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_alter02.py @@ -0,0 +1,234 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2017 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import sys, wiredtiger, wttest +from wtscenario import make_scenarios + +# test_alter02.py +# Smoke-test the session alter operations. +class test_alter02(wttest.WiredTigerTestCase): + entries = 500 + # Binary values. + value = u'\u0001\u0002abcd\u0003\u0004' + value2 = u'\u0001\u0002dcba\u0003\u0004' + + conn_log = [ + ('conn-always-logged', dict(conncreate=True, connreopen=True)), + ('conn-create-logged', dict(conncreate=True, connreopen=False)), + ('conn-reopen-logged', dict(conncreate=False, connreopen=True)), + ('conn-never-logged', dict(conncreate=False, connreopen=False)), + ] + + types = [ + ('file', dict(uri='file:', use_cg=False, use_index=False)), + ('lsm', dict(uri='lsm:', use_cg=False, use_index=False)), + ('table-cg', dict(uri='table:', use_cg=True, use_index=False)), + ('table-index', dict(uri='table:', use_cg=False, use_index=True)), + ('table-simple', dict(uri='table:', use_cg=False, use_index=False)), + ] + + tables = [ + ('always-logged', dict(name='table0', logcreate=True, logalter=True)), + ('create-logged', dict(name='table1', logcreate=True, logalter=False)), + ('alter-logged', dict(name='table2', logcreate=False, logalter=True)), + ('never-logged', dict(name='table3', logcreate=False, logalter=False)), + ] + + reopen = [ + ('no-reopen', dict(reopen=False)), + ('reopen', dict(reopen=True)), + ] + scenarios = make_scenarios(conn_log, types, tables, reopen) + + # This test varies the log setting. Override the standard methods. + def setUpConnectionOpen(self, dir): + return None + def setUpSessionOpen(self, conn): + return None + def ConnectionOpen(self): + self.home = '.' + + conn_params = 'create,log=(archive=false,file_max=100K,%s)' % self.uselog + + try: + self.conn = wiredtiger.wiredtiger_open(self.home, conn_params) + except wiredtiger.WiredTigerError as e: + print "Failed conn at '%s' with config '%s'" % (dir, conn_params) + self.session = self.conn.open_session() + + # Verify the metadata string for this URI and that its setting in the + # metdata file is correct. + def verify_metadata(self, metastr): + if metastr == '': + return + cursor = self.session.open_cursor('metadata:', None, None) + # + # Walk through all the metadata looking for the entries that are + # the file URIs for components of the table. + # + found = False + while True: + ret = cursor.next() + if ret != 0: + break + key = cursor.get_key() + check_meta = ((key.find("lsm:") != -1 or key.find("file:") != -1) \ + and key.find(self.name) != -1) + if check_meta: + value = cursor[key] + found = True + self.assertTrue(value.find(metastr) != -1) + cursor.close() + self.assertTrue(found == True) + + # Verify the data in the log. If the data should be logged we write one + # value. If it should not be logged, we write a different value. + def verify_logrecs(self, expected_keys): + c = self.session.open_cursor('log:', None, None) + count = 0 + while c.next() == 0: + # lsn.file, lsn.offset, opcount + keys = c.get_key() + # txnid, rectype, optype, fileid, logrec_key, logrec_value + values = c.get_value() + try: + if self.value in str(values[5]): # logrec_value + count += 1 + self.assertFalse(value2 in str(values[5])) + except: + pass + c.close() + self.assertEqual(count, expected_keys) + + # Alter: Change the log setting after creation + def test_alter02_log(self): + uri = self.uri + self.name + create_params = 'key_format=i,value_format=S,' + complex_params = '' + + # Set up logging for the connection. + if self.conncreate: + self.uselog = 'enabled=true' + conn_logged = 1 + else: + self.uselog = 'enabled=false' + conn_logged = 0 + self.ConnectionOpen() + + # Set up logging for the table. + if self.logcreate: + log_param = 'log=(enabled=true)' + table_logged = 1 + else: + log_param = 'log=(enabled=false)' + table_logged = 0 + create_params += '%s,' % log_param + complex_params += '%s,' % log_param + + cgparam = '' + if self.use_cg or self.use_index: + cgparam = 'columns=(k,v),' + if self.use_cg: + cgparam += 'colgroups=(g0),' + + self.session.create(uri, create_params + cgparam) + # Add in column group or index settings. + if self.use_cg: + cgparam = 'columns=(v),' + suburi = 'colgroup:' + self.name + ':g0' + self.session.create(suburi, complex_params + cgparam) + if self.use_index: + suburi = 'index:' + self.name + ':i0' + self.session.create(suburi, complex_params + cgparam) + + # Put some data in table. + c = self.session.open_cursor(uri, None) + if self.logcreate: + myvalue = self.value + else: + myvalue = self.value2 + for k in range(self.entries): + c[k] = myvalue + c.close() + + # Verify the logging string in the metadata. + self.verify_metadata(log_param) + + # Verify the logged operations only if logging is enabled. + expected_keys = conn_logged * table_logged * self.entries + if conn_logged != 0: + self.pr("EXPECTED KEYS 1: " + str(expected_keys)) + self.verify_logrecs(expected_keys) + + # Set the alter setting for the table. + if self.logalter: + log_str = 'log=(enabled=true)' + table_logged = 1 + else: + log_str = 'log=(enabled=false)' + table_logged = 0 + alter_param = '%s' % log_str + special = self.use_cg or self.use_index + + # Set the log setting on the new connection. + if self.reopen: + if self.connreopen: + self.uselog = 'enabled=true' + conn_logged = 1 + else: + self.uselog = 'enabled=false' + conn_logged = 0 + self.conn.close() + self.ConnectionOpen() + + self.session.alter(uri, alter_param) + if special: + self.session.alter(suburi, alter_param) + self.verify_metadata(log_str) + # Put some more data in table. + c = self.session.open_cursor(uri, None) + if self.logalter: + myvalue = self.value + else: + myvalue = self.value2 + for k in range(self.entries): + c[k + self.entries] = myvalue + c.close() + # If we logged the new connection and the table, add in the + # number of keys we expect. + expected_keys += conn_logged * table_logged * self.entries + # if self.logalter and self.connreopen == self.reopen: + # expected_keys += self.entries + # If we logged the connection at any time then check + # the log records. + if self.conncreate or (self.connreopen and self.reopen): + self.pr("EXPECTED KEYS 2: " + str(expected_keys)) + self.verify_logrecs(expected_keys) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_compat01.py b/src/third_party/wiredtiger/test/suite/test_compat01.py new file mode 100644 index 00000000000..460d38d5e08 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_compat01.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2017 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_compat01.py +# Check compatibility API + +import fnmatch, os, shutil, sys, time +from suite_subprocess import suite_subprocess +from wtscenario import make_scenarios +import wttest + +class test_compat01(wttest.WiredTigerTestCase, suite_subprocess): + # Add enough entries and use a small log size to generate more than + # one log file. + entries = 2000 + logmax = "100K" + tablename = 'test_compat01' + uri = 'table:' + tablename + sync_list = [ + '(method=fsync,enabled)', + '(method=none,enabled)', + ] + + # The API uses only the major and minor numbers but accepts with + # and without the patch number. Test both. + start_compat = [ + ('def', dict(compat1='none', current1=True)), + ('current', dict(compat1="3.0", current1=True)), + ('current_patch', dict(compat1="3.0.0", current1=True)), + ('minor_only', dict(compat1="2.6", current1=False)), + ('minor_patch', dict(compat1="2.6.1", current1=False)), + ('old', dict(compat1="1.8", current1=False)), + ('old_patch', dict(compat1="1.8.1", current1=False)), + ] + restart_compat = [ + ('def2', dict(compat2='none', current2=True)), + ('current2', dict(compat2="3.0", current2=True)), + ('current_patch2', dict(compat2="3.0.0", current2=True)), + ('minor_only2', dict(compat2="2.6", current2=False)), + ('minor_patch2', dict(compat2="2.6.1", current2=False)), + ('old2', dict(compat2="1.8", current2=False)), + ('old_patch2', dict(compat2="1.8.1", current2=False)), + ] + scenarios = make_scenarios(restart_compat, start_compat) + + def make_compat_str(self, create): + compat_str = '' + if (create == True and self.compat1 != 'none'): + #compat_str = 'verbose=(temporary),compatibility=(release="%s"),' % self.compat1 + compat_str = 'compatibility=(release="%s"),' % self.compat1 + elif (create == False and self.compat2 != 'none'): + #compat_str = 'verbose=(temporary),compatibility=(release="%s"),' % self.compat2 + compat_str = 'compatibility=(release="%s"),' % self.compat2 + return compat_str + + def conn_config(self): + # Cycle through the different transaction_sync values in a + # deterministic manner. + txn_sync = self.sync_list[ + self.scenario_number % len(self.sync_list)] + # Set archive false on the home directory. + log_str = 'log=(archive=false,enabled,file_max=%s),' % self.logmax + \ + 'transaction_sync="%s",' % txn_sync + compat_str = self.make_compat_str(True) + self.pr("Conn config:" + log_str + compat_str) + return log_str + compat_str + + def check_prev_lsn(self, conn_close, prev_lsn_count): + # + # Run printlog and look for the prev_lsn log record. Verify its + # existence with the passed in expected result. We don't use + # check_file_contains because that only looks in the first 100K and + # we don't know how big our text-based log output is. Look through + # the entire file if needed and set a boolean for comparison. + # + self.runWt(['printlog'], outfilename='printlog.out', closeconn=conn_close) + contains = 0 + with open('printlog.out') as logfile: + for line in logfile: + if 'prev_lsn' in line: + contains += 1 + self.assertEqual(prev_lsn_count, contains) + + def check_log(self, reconfig): + orig_logs = fnmatch.filter(os.listdir('.'), "*gerLog*") + compat_str = self.make_compat_str(False) + if self.current1: + prev_lsn_logs = len(orig_logs) + else: + prev_lsn_logs = 0 + + if not reconfig: + # + # Close and open the connection to force recovery and reset the + # compatibility string on startup. + # + self.conn.close() + log_str = 'log=(enabled,file_max=%s,archive=false),' % self.logmax + restart_config = log_str + compat_str + self.pr("Restart conn " + restart_config) + # + # Open a connection to force it to run recovery. + # + conn = self.wiredtiger_open('.', restart_config) + conn.close() + check_close = False + # + # If the version was upgraded we'll see a new log file containing + # the new log record no matter what the original setting was. + # + if self.current2: + prev_lsn_logs += 1 + else: + self.pr("Reconfigure: " + compat_str) + self.conn.reconfigure(compat_str) + check_close = True + # + # If we're reconfiguring, we'll only see another new log file + # when upgrading. Staying at the same version has no effect. + # + if self.current2 and not self.current1: + prev_lsn_logs += 1 + + # Run printlog and verify the new record does or does not exist. + # Need to check count of log files that should and should not have + # the prev_lsn record based on the count of log files that exist + # before and after. Pass that into this function and check the + # number of prev_lsn records we see. + self.check_prev_lsn(check_close, prev_lsn_logs) + + def run_test(self, reconfig): + # If reconfiguring with the empty string there is nothing to do. + if reconfig == True and self.compat2 == 'none': + return + self.session.create(self.uri, 'key_format=i,value_format=i') + c = self.session.open_cursor(self.uri, None) + # + # Add some entries to generate log files. + # + for i in range(self.entries): + c[i] = i + 1 + c.close() + + # Check the log state after the entire op completes + # and run recovery with the restart compatibility mode. + self.check_log(reconfig) + + # Run the same test but reset the compatibility via + # reconfigure or changing it when reopening the connection. + def test_reconfig(self): + self.run_test(True) + + def test_restart(self): + self.run_test(False) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_cursor07.py b/src/third_party/wiredtiger/test/suite/test_cursor07.py index a31d0d401e0..94989dd68a2 100644 --- a/src/third_party/wiredtiger/test/suite/test_cursor07.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor07.py @@ -38,8 +38,12 @@ import wttest class test_cursor07(wttest.WiredTigerTestCase, suite_subprocess): logmax = "100K" - tablename = 'test_cursor07' - uri = 'table:' + tablename + tablename1 = 'test_cursor07_log' + tablename2 = 'test_cursor07_nolog' + tablename3 = 'test_cursor07_nologtxn' + uri1 = 'table:' + tablename1 + uri2 = 'table:' + tablename2 + uri3 = 'table:' + tablename3 # A large number of keys will force a log file change which will # test that scenario for log cursors. nkeys = 7000 @@ -56,17 +60,34 @@ class test_cursor07(wttest.WiredTigerTestCase, suite_subprocess): def test_log_cursor(self): # print "Creating %s with config '%s'" % (self.uri, self.create_params) create_params = 'key_format=i,value_format=S' - self.session.create(self.uri, create_params) - c = self.session.open_cursor(self.uri, None) + create_nolog_params = 'key_format=i,value_format=S,log=(enabled=false)' + self.session.create(self.uri1, create_params) + c1 = self.session.open_cursor(self.uri1, None) + self.session.create(self.uri2, create_nolog_params) + c2 = self.session.open_cursor(self.uri2, None) + self.session.create(self.uri3, create_nolog_params) + c3 = self.session.open_cursor(self.uri3, None) # A binary value. value = u'\u0001\u0002abcd\u0003\u0004' + value_nolog = u'\u0001\u0002dcba\u0003\u0004' + # We want to test both adding data to a table that is not logged + # that is part of the same transaction as a table that is logged + # as well as in its own transaction. self.session.begin_transaction() for k in range(self.nkeys): - c[k] = value + c1[k] = value + c3[k] = value_nolog self.session.commit_transaction() - c.close() + c1.close() + c3.close() + + self.session.begin_transaction() + for k in range(self.nkeys): + c2[k] = value_nolog + self.session.commit_transaction() + c2.close() if self.reopen: self.reopen_conn() @@ -82,6 +103,7 @@ class test_cursor07(wttest.WiredTigerTestCase, suite_subprocess): try: if value in str(values[5]): # logrec_value count += 1 + self.assertFalse(value2 in str(values[5])) except: pass c.close() diff --git a/src/third_party/wiredtiger/test/suite/test_reconfig02.py b/src/third_party/wiredtiger/test/suite/test_reconfig02.py index 3bdc19fb2f8..bba224d1bb5 100644 --- a/src/third_party/wiredtiger/test/suite/test_reconfig02.py +++ b/src/third_party/wiredtiger/test/suite/test_reconfig02.py @@ -100,9 +100,9 @@ class test_reconfig02(wttest.WiredTigerTestCase): c.close() # Close and reopen connection to write a checkpoint, move to the # next log file and verify that archive did not run. - orig_logs = fnmatch.filter(os.listdir('.'), "*Log*") + orig_logs = fnmatch.filter(os.listdir('.'), "*gerLog*") self.reopen_conn() - cur_logs = fnmatch.filter(os.listdir('.'), "*Log*") + cur_logs = fnmatch.filter(os.listdir('.'), "*gerLog*") for o in orig_logs: self.assertEqual(True, o in cur_logs) @@ -111,7 +111,7 @@ class test_reconfig02(wttest.WiredTigerTestCase): self.conn.reconfigure("log=(archive=true)") self.session.checkpoint("force") time.sleep(2) - cur_logs = fnmatch.filter(os.listdir('.'), "*Log*") + cur_logs = fnmatch.filter(os.listdir('.'), "*gerLog*") for o in orig_logs: self.assertEqual(False, o in cur_logs) diff --git a/src/third_party/wiredtiger/test/suite/test_stat_log02.py b/src/third_party/wiredtiger/test/suite/test_stat_log02.py new file mode 100644 index 00000000000..322092c8190 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_stat_log02.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2017 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import glob +import json +import os.path +import time +import helper, wiredtiger, wttest +from wiredtiger import stat + +# test_stat_log02.py +# Statistics log JSON testing +class test_stat_log02(wttest.WiredTigerTestCase): + """ + Test statistics log JSON outputs + """ + + # Tests need to setup the connection in their own way. + def setUpConnectionOpen(self, dir): + return None + + def setUpSessionOpen(self, conn): + return None + + def test_stats_log_json(self): + self.conn = self.wiredtiger_open( + None, "create,statistics=(fast),statistics_log=(wait=1,json)") + + self.wait_for_stats_file(".") + self.check_stats_file(".") + + def test_stats_log_on_json_with_tables(self): + self.conn = self.wiredtiger_open(None, + "create,statistics=(fast),statistics_log=(wait=1,json,sources=[file:])") + + # Create a session and table to give us some stats + session = self.conn.open_session() + session.create("table:foo") + c = session.open_cursor("table:foo") + c["foo"] = "foo" + c.close() + session.close() + + self.wait_for_stats_file(".") + self.close_conn() + self.check_stats_file(".") + self.check_file_contains_tables(".") + + def wait_for_stats_file(self, dir): + # We wait for 30 sleeps then fail + number_sleeps = 0 + while True: + time.sleep(1) + files = glob.glob(dir + '/' + 'WiredTigerStat.[0-9]*') + for f in files: + if os.stat(f).st_size != 0: + return + + number_sleeps += 1 + self.assertLess(number_sleeps, 30) + + def check_stats_file(self, dir): + files = glob.glob(dir + '/' + 'WiredTigerStat.[0-9]*') + self.assertTrue(files) + self.check_file_is_json(files[0]) + + def check_file_is_json(self, file_name): + f = open(file_name, 'r') + for line in f: + # This will throw assertions if we don't have correctly formed JSON + json.loads(line) + + def check_file_contains_tables(self, dir): + files = glob.glob(dir + '/' + 'WiredTigerStat.[0-9]*') + f = open(files[0], 'r') + has_tables = False + for line in f: + data = json.loads(line) + if "wiredTigerTables" in data: + if "file:foo.wt" in data["wiredTigerTables"]: + has_tables = True + + self.assertTrue(has_tables) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_txn02.py b/src/third_party/wiredtiger/test/suite/test_txn02.py index b61a9ed9f99..6e54270d3ad 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn02.py +++ b/src/third_party/wiredtiger/test/suite/test_txn02.py @@ -163,7 +163,7 @@ class test_txn02(wttest.WiredTigerTestCase, suite_subprocess): self.scenario_number % len(self.archive_list)] backup_conn_params = \ 'log=(enabled,file_max=%s,archive=%s)' % (self.logmax, self.archive) - orig_logs = fnmatch.filter(os.listdir(self.backup_dir), "*Log*") + orig_logs = fnmatch.filter(os.listdir(self.backup_dir), "*gerLog*") endcount = 2 count = 0 while count < endcount: @@ -183,7 +183,7 @@ class test_txn02(wttest.WiredTigerTestCase, suite_subprocess): # have been archived if configured. Subsequent openings would not # archive because no checkpoint is written due to no modifications. # - cur_logs = fnmatch.filter(os.listdir(self.backup_dir), "*Log*") + cur_logs = fnmatch.filter(os.listdir(self.backup_dir), "*gerLog*") for o in orig_logs: if self.archive == 'true': self.assertEqual(False, o in cur_logs) @@ -195,7 +195,7 @@ class test_txn02(wttest.WiredTigerTestCase, suite_subprocess): # it does not. # self.runWt(['-h', self.backup_dir, 'printlog'], outfilename='printlog.out') - pr_logs = fnmatch.filter(os.listdir(self.backup_dir), "*Log*") + pr_logs = fnmatch.filter(os.listdir(self.backup_dir), "*gerLog*") self.assertEqual(cur_logs, pr_logs) def test_ops(self): diff --git a/src/third_party/wiredtiger/test/suite/test_txn05.py b/src/third_party/wiredtiger/test/suite/test_txn05.py index 6a5be0a5df4..045cd570d8a 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn05.py +++ b/src/third_party/wiredtiger/test/suite/test_txn05.py @@ -127,7 +127,7 @@ class test_txn05(wttest.WiredTigerTestCase, suite_subprocess): self.scenario_number % len(self.archive_list)] backup_conn_params = \ 'log=(enabled,file_max=%s,archive=%s)' % (self.logmax, self.archive) - orig_logs = fnmatch.filter(os.listdir(self.backup_dir), "*Log*") + orig_logs = fnmatch.filter(os.listdir(self.backup_dir), "*gerLog*") endcount = 2 count = 0 while count < endcount: @@ -142,7 +142,7 @@ class test_txn05(wttest.WiredTigerTestCase, suite_subprocess): time.sleep(1.0) if count == 0: first_logs = \ - fnmatch.filter(os.listdir(self.backup_dir), "*Log*") + fnmatch.filter(os.listdir(self.backup_dir), "*gerLog*") backup_conn.close() count += 1 # @@ -150,7 +150,7 @@ class test_txn05(wttest.WiredTigerTestCase, suite_subprocess): # have been archived if configured. Subsequent openings would not # archive because no checkpoint is written due to no modifications. # - cur_logs = fnmatch.filter(os.listdir(self.backup_dir), "*Log*") + cur_logs = fnmatch.filter(os.listdir(self.backup_dir), "*gerLog*") for o in orig_logs: # Creating the backup was effectively an unclean shutdown so # even after sleeping, we should never archive log files diff --git a/src/third_party/wiredtiger/test/suite/test_txn11.py b/src/third_party/wiredtiger/test/suite/test_txn11.py index 4b4db9ce315..0cb2dfd615a 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn11.py +++ b/src/third_party/wiredtiger/test/suite/test_txn11.py @@ -50,12 +50,12 @@ class test_txn11(wttest.WiredTigerTestCase, suite_subprocess): 'transaction_sync=(enabled=false),' def run_checkpoints(self): - orig_logs = fnmatch.filter(os.listdir(self.home), "*Log*") + orig_logs = fnmatch.filter(os.listdir(self.home), "*gerLog*") checkpoints = 0 sorig = set(orig_logs) while checkpoints < 500: self.session.checkpoint() - cur_logs = fnmatch.filter(os.listdir(self.home), "*Log*") + cur_logs = fnmatch.filter(os.listdir(self.home), "*gerLog*") scur = set(cur_logs) if scur.isdisjoint(sorig): break diff --git a/src/third_party/wiredtiger/test/suite/test_txn16.py b/src/third_party/wiredtiger/test/suite/test_txn16.py index 929da2291c7..921b67e3f6b 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn16.py +++ b/src/third_party/wiredtiger/test/suite/test_txn16.py @@ -82,14 +82,14 @@ class test_txn16(wttest.WiredTigerTestCase, suite_subprocess): loop = 0 # Record original log files. There should never be overlap # with these even after they're removed. - orig_logs = fnmatch.filter(os.listdir(homedir), "*Log*") + orig_logs = fnmatch.filter(os.listdir(homedir), "*gerLog*") while loop < 3: # Reopen with logging on to run recovery first time on_conn = self.wiredtiger_open(homedir, self.conn_on) on_conn.close() if loop > 0: # Get current log files. - cur_logs = fnmatch.filter(os.listdir(homedir), "*Log*") + cur_logs = fnmatch.filter(os.listdir(homedir), "*gerLog*") scur = set(cur_logs) sorig = set(orig_logs) # There should never be overlap with the log files that @@ -106,7 +106,7 @@ class test_txn16(wttest.WiredTigerTestCase, suite_subprocess): last_logs = cur_logs loop += 1 # Remove all log files before opening without logging. - cur_logs = fnmatch.filter(os.listdir(homedir), "*Log*") + cur_logs = fnmatch.filter(os.listdir(homedir), "*gerLog*") for l in cur_logs: path=homedir + "/" + l os.remove(path) diff --git a/src/third_party/wiredtiger/test/thread/rw.c b/src/third_party/wiredtiger/test/thread/rw.c index 3283f780b32..66f706d0d34 100644 --- a/src/third_party/wiredtiger/test/thread/rw.c +++ b/src/third_party/wiredtiger/test/thread/rw.c @@ -347,6 +347,6 @@ print_stats(u_int nthreads) s = run_info; for (id = 0; id < nthreads; ++id, ++s) - printf("%3d: read %6d, remove %6d, update %6d\n", + printf("%3u: read %6d, remove %6d, update %6d\n", id, s->reads, s->remove, s->update); } diff --git a/src/third_party/wiredtiger/test/utility/test_util.h b/src/third_party/wiredtiger/test/utility/test_util.h index f07fec98bfe..e53018ad4ea 100644 --- a/src/third_party/wiredtiger/test/utility/test_util.h +++ b/src/third_party/wiredtiger/test/utility/test_util.h @@ -217,21 +217,19 @@ const char *example_setup(int, char * const *); */ int handle_op_error(WT_EVENT_HANDLER *, WT_SESSION *, int, const char *); int handle_op_message(WT_EVENT_HANDLER *, WT_SESSION *, const char *); -void *op_bulk(void *); -void *op_bulk_unique(void *); -void *op_cursor(void *); -void *op_create(void *); -void *op_create_unique(void *); -void *op_drop(void *); -void testutil_clean_work_dir(const char *); -void testutil_cleanup(TEST_OPTS *); -bool testutil_is_flag_set(const char *); -void testutil_make_work_dir(const char *); -int testutil_parse_opts(int, char * const *, TEST_OPTS *); -void testutil_work_dir_from_path(char *, size_t, const char *); -void *thread_append(void *); -void *thread_insert_append(void *); -void *thread_prev(void *); +void op_bulk(void *); +void op_bulk_unique(void *); +void op_create(void *); +void op_create_unique(void *); +void op_cursor(void *); +void op_drop(void *); +void testutil_clean_work_dir(const char *); +void testutil_cleanup(TEST_OPTS *); +bool testutil_is_flag_set(const char *); +void testutil_make_work_dir(const char *); +int testutil_parse_opts(int, char * const *, TEST_OPTS *); +void testutil_work_dir_from_path(char *, size_t, const char *); +WT_THREAD_RET thread_append(void *); extern const char *progname; const char *testutil_set_progname(char * const *); diff --git a/src/third_party/wiredtiger/test/utility/thread.c b/src/third_party/wiredtiger/test/utility/thread.c index 4c5202016a5..ed8c7e5ae73 100644 --- a/src/third_party/wiredtiger/test/utility/thread.c +++ b/src/third_party/wiredtiger/test/utility/thread.c @@ -34,7 +34,7 @@ * One thread (the first thread created by an application) checks for a * terminating condition after each insert. */ -void * +WT_THREAD_RET thread_append(void *arg) { TEST_OPTS *opts; @@ -70,75 +70,7 @@ thread_append(void *arg) } } - return (NULL); -} - -/* - * Append into a row store table. - */ -void * -thread_insert_append(void *arg) -{ - TEST_OPTS *opts; - WT_CONNECTION *conn; - WT_CURSOR *cursor; - WT_SESSION *session; - uint64_t i; - char kbuf[64]; - - opts = (TEST_OPTS *)arg; - conn = opts->conn; - - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - testutil_check(session->open_cursor( - session, opts->uri, NULL, NULL, &cursor)); - - for (i = 0; i < opts->nrecords; ++i) { - testutil_check(__wt_snprintf( - kbuf, sizeof(kbuf), "%010d KEY------", (int)i)); - cursor->set_key(cursor, kbuf); - cursor->set_value(cursor, "========== VALUE ======="); - testutil_check(cursor->insert(cursor)); - if (i % 100000 == 0) { - printf("insert: %" PRIu64 "\r", i); - fflush(stdout); - } - } - printf("\n"); - - opts->running = false; - - return (NULL); -} - -/* - * Repeatedly walk backwards through the records in a table. - */ -void * -thread_prev(void *arg) -{ - TEST_OPTS *opts; - WT_CURSOR *cursor; - WT_SESSION *session; - int ret; - - opts = (TEST_OPTS *)arg; - ret = 0; - - testutil_check( - opts->conn->open_session(opts->conn, NULL, NULL, &session)); - testutil_check( - session->open_cursor(session, opts->uri, NULL, NULL, &cursor)); - while (opts->running) { - while (opts->running && (ret = cursor->prev(cursor)) == 0) - ; - if (ret == WT_NOTFOUND) - ret = 0; - testutil_check(ret); - } - - testutil_check(session->close(session, NULL)); - return (NULL); + return (WT_THREAD_RET_VALUE); } /* @@ -198,7 +130,7 @@ handle_op_message(WT_EVENT_HANDLER *handler, /* * Create a table and open a bulk cursor on it. */ -void * +void op_bulk(void *arg) { TEST_OPTS *opts; @@ -228,14 +160,12 @@ op_bulk(void *arg) testutil_check(session->close(session, NULL)); args->thread_counter++; - - return (NULL); } /* * Create a guaranteed unique table and open and close a bulk cursor on it. */ -void * +void op_bulk_unique(void *arg) { TEST_OPTS *opts; @@ -286,14 +216,12 @@ op_bulk_unique(void *arg) testutil_check(session->close(session, NULL)); args->thread_counter++; - - return (NULL); } /* * Open and close cursor on a table. */ -void * +void op_cursor(void *arg) { TEST_OPTS *opts; @@ -317,14 +245,12 @@ op_cursor(void *arg) testutil_check(session->close(session, NULL)); args->thread_counter++; - - return (NULL); } /* * Create a table. */ -void * +void op_create(void *arg) { TEST_OPTS *opts; @@ -344,14 +270,12 @@ op_create(void *arg) testutil_check(session->close(session, NULL)); args->thread_counter++; - - return (NULL); } /* * Create and drop a unique guaranteed table. */ -void * +void op_create_unique(void *arg) { TEST_OPTS *opts; @@ -389,14 +313,12 @@ op_create_unique(void *arg) testutil_check(session->close(session, NULL)); args->thread_counter++; - - return (NULL); } /* * Drop a table. */ -void * +void op_drop(void *arg) { TEST_PER_THREAD_OPTS *args; @@ -419,6 +341,4 @@ op_drop(void *arg) testutil_check(session->close(session, NULL)); args->thread_counter++; - - return (NULL); } diff --git a/src/third_party/wiredtiger/tools/wtstats/stat_data.py b/src/third_party/wiredtiger/tools/wtstats/stat_data.py index 09fca2b9525..41bc26abbaf 100644 --- a/src/third_party/wiredtiger/tools/wtstats/stat_data.py +++ b/src/third_party/wiredtiger/tools/wtstats/stat_data.py @@ -13,7 +13,6 @@ no_scale_per_second_list = [ 'cache: hazard pointer maximum array length', 'cache: maximum bytes configured', 'cache: maximum page size at eviction', - 'cache: overflow values cached in memory', 'cache: pages currently held in the cache', 'cache: percentage overhead', 'cache: tracked bytes belonging to internal pages in the cache', @@ -94,10 +93,11 @@ no_scale_per_second_list = [ 'btree: row-store internal pages', 'btree: row-store leaf pages', 'cache: bytes currently in the cache', - 'cache: overflow values cached in memory', 'cache: tracked dirty bytes in the cache', 'cache_walk: Average difference between current eviction generation when the page was last considered', 'cache_walk: Average on-disk page image size seen', + 'cache_walk: Average time in cache for pages that have been visited by the eviction server', + 'cache_walk: Average time in cache for pages that have not been visited by the eviction server', 'cache_walk: Clean pages currently in cache', 'cache_walk: Current eviction generation', 'cache_walk: Dirty pages currently in cache', @@ -107,6 +107,7 @@ no_scale_per_second_list = [ 'cache_walk: Maximum difference between current eviction generation when the page was last considered', 'cache_walk: Maximum page size seen', 'cache_walk: Minimum on-disk page image size seen', + 'cache_walk: Number of pages never visited by eviction server', 'cache_walk: On-disk page image sizes smaller than a single allocation unit', 'cache_walk: Pages created in memory and never written', 'cache_walk: Pages currently queued for eviction', @@ -194,6 +195,8 @@ no_clear_list = [ 'cache: tracked dirty bytes in the cache', 'cache_walk: Average difference between current eviction generation when the page was last considered', 'cache_walk: Average on-disk page image size seen', + 'cache_walk: Average time in cache for pages that have been visited by the eviction server', + 'cache_walk: Average time in cache for pages that have not been visited by the eviction server', 'cache_walk: Clean pages currently in cache', 'cache_walk: Current eviction generation', 'cache_walk: Dirty pages currently in cache', @@ -203,6 +206,7 @@ no_clear_list = [ 'cache_walk: Maximum difference between current eviction generation when the page was last considered', 'cache_walk: Maximum page size seen', 'cache_walk: Minimum on-disk page image size seen', + 'cache_walk: Number of pages never visited by eviction server', 'cache_walk: On-disk page image sizes smaller than a single allocation unit', 'cache_walk: Pages created in memory and never written', 'cache_walk: Pages currently queued for eviction', |