diff options
| author | Luke Chen <luke.chen@mongodb.com> | 2018-08-07 13:55:40 +1000 |
|---|---|---|
| committer | Luke Chen <luke.chen@mongodb.com> | 2018-08-07 13:55:40 +1000 |
| commit | 96747600635ffe73bf1443857a76a9ccdc90c45c (patch) | |
| tree | 4cb1eb058bdea550caca6fe5526a03693cfb07c2 /src/third_party/wiredtiger | |
| parent | a37441baa3d5d3dc6f70c7b0d7825f9dbbc90b3f (diff) | |
| download | mongo-96747600635ffe73bf1443857a76a9ccdc90c45c.tar.gz | |
Import wiredtiger: 2bb717625d1e81e6a0abfb910ead00afdde7fe2a from branch mongodb-4.0
ref: e6c7496532..2bb717625d
for: 4.0.2
WT-3276 Add recover=salvage to recover from a corrupted log file
WT-3943 Include full error message when a python test asserts
WT-3955 Add verbose option to log more messages on error returns
WT-3963 Add a schema intensive abort testing
WT-3968 Use compression ratio to tune page sizes
WT-4010 Simplify test/format timestamp handling.
WT-4134 Rework assertion that we don't discard required history
WT-4147 Log recovery should not ignore corruption outside of log records in a log file
WT-4160 Restore performance when timestamps are not in use
WT-4168 Update upgrading documentation for 3.1.0 release
WT-4169 Fix wt verify dump-pages failure
WT-4171 Enabling tree walk timing stress causes excessive slowdown
WT-4172 Add diagnostic hazard pointer checks in more places before freeing refs
WT-4174 Do not access the lookaside file in rollback_to_stable when running with in_memory=true
WT-4178 Fixes for wt_btree_immediately_durable needed for in-memory
WT-4179 Expose WiredTiger crc32c functions
WT-4182 Use conservative approach for log checksum errors
WT-4183 Extend verbose option to log more messages on error returns
WT-4186 Log recovery should detect and report corruption within log records
WT-4187 Coverity: unused value complaints
WT-4188 Coverity: unchecked return value complaints
WT-4189 Potential infinite loop in __async_flush_wait().
WT-4191 Fix Coverity static analysis errors
WT-4195 When encountering an illegal value, log the value that failed
WT-4196 Make log corruption checking work regardless of the machine byte order
WT-4198 Some supported MongoDB architectures don't support crc32 hardware
WT-4199 Fix an incorrect report of log corruption
WT-4201 Fix Coverity static analysis issues
WT-4206 Fix error handling in cursor close routines
WT-4208 tree walks can be interrupted by locked internal pages
WT-4213 Rename lock statistics that have redundant or misleading text
WT-4226 test/format LSM configurations can misconfigure prepare and timestamps
Diffstat (limited to 'src/third_party/wiredtiger')
192 files changed, 4018 insertions, 1402 deletions
diff --git a/src/third_party/wiredtiger/NEWS b/src/third_party/wiredtiger/NEWS index 7bf3b0e7edb..1e821835386 100644 --- a/src/third_party/wiredtiger/NEWS +++ b/src/third_party/wiredtiger/NEWS @@ -1,6 +1,14 @@ Ticket reference tags refer to tickets in the MongoDB JIRA tracking system: https://jira.mongodb.org +WiredTiger release 3.1.0, 2018-07-12 +------------------------------------ + +See the upgrading documentation for details of API and behavior changes. + +See JIRA changelog for a full listing: +https://jira.mongodb.org/projects/WT/versions/19708 + WiredTiger release 3.0.0, 2018-01-08 ------------------------------------ diff --git a/src/third_party/wiredtiger/README b/src/third_party/wiredtiger/README index f21ff213a7c..fd8757621bf 100644 --- a/src/third_party/wiredtiger/README +++ b/src/third_party/wiredtiger/README @@ -1,6 +1,6 @@ -WiredTiger 3.1.0: (April 23, 2018) +WiredTiger 3.1.1: (July 12, 2018) -This is version 3.1.0 of WiredTiger. +This is version 3.1.1 of WiredTiger. WiredTiger release packages and documentation can be found at: @@ -8,7 +8,7 @@ WiredTiger release packages and documentation can be found at: The documentation for this specific release can be found at: - http://source.wiredtiger.com/3.1.0/index.html + http://source.wiredtiger.com/3.1.1/index.html The WiredTiger source code can be found at: diff --git a/src/third_party/wiredtiger/RELEASE_INFO b/src/third_party/wiredtiger/RELEASE_INFO index ee25ecd6c56..2014ba3ee74 100644 --- a/src/third_party/wiredtiger/RELEASE_INFO +++ b/src/third_party/wiredtiger/RELEASE_INFO @@ -1,6 +1,6 @@ WIREDTIGER_VERSION_MAJOR=3 WIREDTIGER_VERSION_MINOR=1 -WIREDTIGER_VERSION_PATCH=0 +WIREDTIGER_VERSION_PATCH=1 WIREDTIGER_VERSION="$WIREDTIGER_VERSION_MAJOR.$WIREDTIGER_VERSION_MINOR.$WIREDTIGER_VERSION_PATCH" WIREDTIGER_RELEASE_DATE=`date "+%B %e, %Y"` diff --git a/src/third_party/wiredtiger/bench/workgen/runner/compress_ratio.py b/src/third_party/wiredtiger/bench/workgen/runner/compress_ratio.py new file mode 100644 index 00000000000..2c5552bfa5d --- /dev/null +++ b/src/third_party/wiredtiger/bench/workgen/runner/compress_ratio.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2018 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +# Drive a constant high workload through, even if WiredTiger isn't keeping +# up by dividing the workload across a lot of threads. This needs to be +# tuned to the particular machine so the workload is close to capacity in the +# steady state, but not overwhelming. +# +################ +# Note: This looks similar to multi_btree_heavy_stress.py with values altered +# for run time, #ops, #threads, #throttle to maintain dirty cache around the +# eviction target of 5% on the AWS perf machines. These values being machine +# dependant might need to be altered as per the machine this workload gets +# run on. +# +from runner import * +from wiredtiger import * +from workgen import * + +def op_append(ops, op): + if ops == None: + ops = op + else: + ops += op + return ops + +def make_op(optype, table, key, value = None): + if value == None: + return Operation(optype, table, key) + else: + return Operation(optype, table, key, value) + +logkey = Key(Key.KEYGEN_APPEND, 8) ## should be 8 bytes format 'Q' +def operations(optype, tables, key, value = None, ops_per_txn = 0, logtable = None): + txn_list = [] + ops = None + nops = 0 + for table in tables: + ops = op_append(ops, make_op(optype, table, key, value)) + if logtable != None: + ops = op_append(ops, make_op(optype, logtable, logkey, value)) + nops += 1 + if ops_per_txn > 0 and nops % ops_per_txn == 0: + txn_list.append(txn(ops)) + ops = None + if ops_per_txn > 0: + if ops != None: + txn_list.append(txn(ops)) + ops = None + for t in txn_list: + ops = op_append(ops, t) + return ops + +context = Context() +conn_config="create,cache_size=2GB,session_max=1000,eviction=(threads_min=4,threads_max=4),log=(enabled=false),transaction_sync=(enabled=false),checkpoint_sync=false,checkpoint=(wait=20),statistics=(fast),statistics_log=(json,wait=1)" +table_config="allocation_size=4k,memory_page_max=10MB,prefix_compression=false,split_pct=90,leaf_page_max=32k,internal_page_max=16k,type=file" +compression_opts = { + "none" : "block_compressor=none", + "zlib_noraw" : "block_compressor=zlib-noraw", + "zlib_noraw_onepage" : "block_compressor=zlib-noraw,memory_page_image_max=32k", + "zlib_noraw_tenpage" : "block_compressor=zlib-noraw,memory_page_image_max=320k", + "zlib_raw" : "block_compressor=zlib", + "snappy" : "block_compressor=snappy" +} +#conn_config += extensions_config(['compressors/snappy']) +conn = wiredtiger_open("WT_TEST", conn_config) +s = conn.open_session() + +tables = [] +for name_ext, compress_config in compression_opts.iteritems(): + tname = "table:test_" + name_ext + s.create(tname, 'key_format=S,value_format=S,' + table_config + "," + compress_config) + table = Table(tname) + table.options.value_compressibility = 70 + tables.append(table) + +icount=500000 +ins_ops = operations(Operation.OP_INSERT, tables, Key(Key.KEYGEN_APPEND, 20), Value(500)) +thread = Thread(ins_ops * icount) +pop_workload = Workload(context, thread) +print('populate:') +pop_workload.run(conn) + +ins_ops = operations(Operation.OP_INSERT, tables, Key(Key.KEYGEN_APPEND, 20), Value(500), 0) +upd_ops = operations(Operation.OP_UPDATE, tables, Key(Key.KEYGEN_UNIFORM, 20), Value(500), 0) + +ins_thread = Thread(ins_ops) +upd_thread = Thread(upd_ops) +ins_thread.options.throttle = 1000 +ins_thread.options.name = "Insert" +upd_thread.options.throttle = 1000 +upd_thread.options.name = "Update" +threads = ins_thread * 2 + upd_thread * 10 +workload = Workload(context, threads) +workload.options.run_time = 60 +workload.options.report_interval = 1 +workload.options.sample_interval = 1 +workload.options.sample_rate = 1 +print('Update heavy workload:') +workload.run(conn) diff --git a/src/third_party/wiredtiger/bench/workgen/runner/insert_stress.py b/src/third_party/wiredtiger/bench/workgen/runner/insert_stress.py new file mode 100644 index 00000000000..be33396db70 --- /dev/null +++ b/src/third_party/wiredtiger/bench/workgen/runner/insert_stress.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2018 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +from runner import * +from wiredtiger import * +from workgen import * + +context = Context() +conn_config="create,cache_size=4GB,session_max=1000,eviction=(threads_min=4,threads_max=8),log=(enabled=false),transaction_sync=(enabled=false),checkpoint_sync=true,checkpoint=(wait=10),statistics=(fast),statistics_log=(json,wait=1)" +table_config="allocation_size=4k,memory_page_max=10MB,prefix_compression=false,split_pct=90,leaf_page_max=32k,internal_page_max=16k,type=file,block_compressor=snappy" +conn = wiredtiger_open("WT_TEST", conn_config) +s = conn.open_session() +tname = "file:test.wt" +table_config="key_format=S,value_format=S,allocation_size=4k,memory_page_max=10MB,prefix_compression=false,split_pct=90,leaf_page_max=32k,leaf_value_max=64MB,internal_page_max=16k,type=file,block_compressor=snappy" +s.create(tname, table_config) +table = Table(tname) +table.options.key_size = 20 +table.options.value_size = 130 * 1024 +table.options.range = 100000000 # 100 million + +op = Operation(Operation.OP_INSERT, table) +thread = Thread(op * 500) +pop_workload = Workload(context, thread) +print('populate:') +pop_workload.run(conn) + +op = Operation(Operation.OP_INSERT, table, Key(Key.KEYGEN_UNIFORM, 10), Value(130 * 1024)) +op2 = Operation(Operation.OP_INSERT, table, Key(Key.KEYGEN_UNIFORM, 10), Value(100)) +op3 = Operation(Operation.OP_INSERT, table, Key(Key.KEYGEN_APPEND, 10), Value(130 * 1024)) +t = Thread(op + 10 * op2 + op3) + +read_op = Operation(Operation.OP_SEARCH, table, Key(Key.KEYGEN_UNIFORM, 10)) +read_txn_ops = op_group_transaction(read_op, 100, "") +read_thread = Thread(read_txn_ops) + +workload = Workload(context, t * 8 + read_thread) +workload.options.run_time = 240 +workload.options.report_interval = 5 +print('workload:') +workload.run(conn) diff --git a/src/third_party/wiredtiger/bench/workgen/workgen.cxx b/src/third_party/wiredtiger/bench/workgen/workgen.cxx index f89356b836a..74393b13681 100644 --- a/src/third_party/wiredtiger/bench/workgen/workgen.cxx +++ b/src/third_party/wiredtiger/bench/workgen/workgen.cxx @@ -55,11 +55,11 @@ extern "C" { #define THROTTLE_PER_SEC 20 // times per sec we will throttle -#define MIN(a, b) ((a) < (b) ? (a) : (b)) -#define MAX(a, b) ((a) < (b) ? (b) : (a)) -#define TIMESPEC_DOUBLE(ts) ((double)(ts).tv_sec + ts.tv_nsec * 0.000000001) -#define PCT(n, total) ((total) == 0 ? 0 : ((n) * 100) / (total)) -#define OPS_PER_SEC(ops, ts) (int) ((ts) == 0 ? 0.0 : \ +#define MIN(a, b) ((a) < (b) ? (a) : (b)) +#define MAX(a, b) ((a) < (b) ? (b) : (a)) +#define TIMESPEC_DOUBLE(ts) ((double)(ts).tv_sec + ts.tv_nsec * 0.000000001) +#define PCT(n, total) ((total) == 0 ? 0 : ((n) * 100) / (total)) +#define OPS_PER_SEC(ops, ts) (int) ((ts) == 0 ? 0.0 : \ (ops) / TIMESPEC_DOUBLE(ts)) // Get the value of a STL container, even if it is not present @@ -345,7 +345,7 @@ int Monitor::run() { << std::endl; if (_json != NULL) { -#define WORKGEN_TIMESTAMP_JSON "%Y-%m-%dT%H:%M:%S.000Z" +#define WORKGEN_TIMESTAMP_JSON "%Y-%m-%dT%H:%M:%S.000Z" (void)strftime(time_buf, sizeof(time_buf), WORKGEN_TIMESTAMP_JSON, tm); @@ -598,7 +598,7 @@ void ThreadRunner::op_create_all(Operation *op, size_t &keysize, } -#define PARETO_SHAPE 1.5 +#define PARETO_SHAPE 1.5 // Return a value within the interval [ 0, recno_max ) // that is weighted toward lower numbers with pareto_param at 0 (the minimum), @@ -653,7 +653,7 @@ uint64_t ThreadRunner::op_get_key_recno(Operation *op, uint64_t range, if (recno_count == 0) // The file has no entries, returning 0 forces a WT_NOTFOUND return. return (0); - rval = workgen_random(_rand_state); + rval = random_value(); if (op->_key._keytype == Key::KEYGEN_PARETO) rval = pareto_calculation(rval, recno_count, op->_key._pareto); return (rval % recno_count + 1); // recnos are one-based. @@ -742,13 +742,12 @@ int ThreadRunner::op_run(Operation *op) { _in_transaction = true; } if (op->_optype != Operation::OP_NONE) { - op->kv_gen(true, 0, recno, _keybuf); + op->kv_gen(this, true, 100, recno, _keybuf); cursor->set_key(cursor, _keybuf); if (OP_HAS_VALUE(op)) { - uint32_t r = 0; - if (op->_table.options.random_value) - r = workgen_random(_rand_state); - op->kv_gen(false, r, recno, _valuebuf); + uint64_t compressibility = op->_table.options.random_value ? + 0 : op->_table.options.value_compressibility; + op->kv_gen(this, false, compressibility, recno, _valuebuf); cursor->set_value(cursor, _valuebuf); } switch (op->_optype) { @@ -806,6 +805,18 @@ std::string ThreadRunner::get_debug() { } #endif +uint32_t ThreadRunner::random_value() { + return (workgen_random(_rand_state)); +} + +// Generate a random 32-bit value then return a float value equally distributed +// between -1.0 and 1.0. +float ThreadRunner::random_signed() { + uint32_t r = random_value(); + int sign = ((r & 0x1) == 0 ? 1 : -1); + return (((float)r * sign) / UINT32_MAX); +} + Throttle::Throttle(ThreadRunner &runner, double throttle, double throttle_burst) : _runner(runner), _throttle(throttle), _burst(throttle_burst), _next_div(), _ops_delta(0), _ops_prev(0), @@ -817,13 +828,6 @@ Throttle::Throttle(ThreadRunner &runner, double throttle, Throttle::~Throttle() {} -// Given a random 32-bit value, return a float value equally distributed -// between -1.0 and 1.0. -static float rand_signed(uint32_t r) { - int sign = ((r & 0x1) == 0 ? 1 : -1); - return (((float)r * sign) / UINT32_MAX); -} - // Each time throttle is called, we sleep and return a number of operations to // perform next. To implement this we keep a time calculation in _next_div set // initially to the current time + 1/THROTTLE_PER_SEC. Each call to throttle @@ -851,8 +855,7 @@ int Throttle::throttle(uint64_t op_count, uint64_t *op_limit) { _ops_delta += (op_count - _ops_prev); if (now < _next_div) { sleep_ms = ts_ms(_next_div - now); - sleep_ms += (_ms_per_div * _burst * - rand_signed(workgen_random(_runner._rand_state))); + sleep_ms += (_ms_per_div * _burst * _runner.random_signed()); if (sleep_ms > 0) { DEBUG_CAPTURE(_runner, ", sleep=" << sleep_ms); usleep((useconds_t)ms_to_us(sleep_ms)); @@ -1066,10 +1069,6 @@ void Operation::kv_compute_max(bool iskey, bool has_random) { if (has_random) { if (iskey) THROW("Random keys not allowed"); - size -= RANDOMIZER_SIZE; - if (size < 1) - THROW("Value.size with random values too small for table '" - << _table._uri << "'"); } if (size > 1) @@ -1096,8 +1095,8 @@ void Operation::kv_size_buffer(bool iskey, size_t &maxsize) const { } } -void Operation::kv_gen(bool iskey, uint32_t randomizer, uint64_t n, - char *result) const { +void Operation::kv_gen(ThreadRunner *runner, bool iskey, + uint64_t compressibility, uint64_t n, char *result) const { uint64_t max; int size; @@ -1106,13 +1105,40 @@ void Operation::kv_gen(bool iskey, uint32_t randomizer, uint64_t n, if (n > max) THROW((iskey ? "Key" : "Value") << " (" << n << ") too large for size (" << size << ")"); - if (randomizer != 0) { - randomizer %= 1000; - snprintf(result, 6, ":%3.3d:", randomizer); - n -= RANDOMIZER_SIZE; - result += RANDOMIZER_SIZE; - } + /* Setup the buffer, defaulting to zero filled. */ workgen_u64_to_string_zf(n, result, size); + + /* + * Compressibility is a percentage, 100 is all zeroes, it applies to the + * proportion of the value that can't be used for the identifier. + */ + if (size > 20 && compressibility < 100) { + static const char alphanum[] = + "0123456789" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz"; + /* + * The random length is the proportion of the string that should not + * be compressible. As an example a compressibility of 25 in a value + * of length 100 should be: + * 100 - ((100 * 25) / 100) = 75 + * That means that 75% of the string will be random numbers, and 25 + * will be easily compressible zero-fill. + */ + uint64_t random_len = size - ((size * compressibility) / 100); + + /* Never overwrite the record number identifier */ + if (random_len > size - 20) + random_len = size - 20; + + for (int i = 0; i < random_len; ++i) + /* + * TODO: It'd be nice to use workgen_rand here, but this class + * is without the context of a runner thread, so it's not easy + * to get access to a state. + */ + result[i] = alphanum[runner->random_value() % (sizeof(alphanum) - 1)]; + } } void Operation::size_check() const { @@ -1436,18 +1462,21 @@ void Stats::track_latency(bool latency) { } TableOptions::TableOptions() : key_size(0), value_size(0), - random_value(false), range(0), _options() { + value_compressibility(100), random_value(false), range(0), _options() { _options.add_int("key_size", key_size, "default size of the key, unless overridden by Key.size"); _options.add_int("value_size", value_size, "default size of the value, unless overridden by Value.size"); _options.add_bool("random_value", random_value, "generate random content for the value"); + _options.add_bool("value_compressibility", value_compressibility, + "How compressible the generated value should be"); _options.add_int("range", range, "if zero, keys are inserted at the end and reads/updates are in the current range, if non-zero, inserts/reads/updates are at a random key between 0 and the given range"); } TableOptions::TableOptions(const TableOptions &other) : key_size(other.key_size), value_size(other.value_size), + value_compressibility(other.value_compressibility), random_value(other.random_value), range(other.range), _options(other._options) {} TableOptions::~TableOptions() {} diff --git a/src/third_party/wiredtiger/bench/workgen/workgen.h b/src/third_party/wiredtiger/bench/workgen/workgen.h index dc15ab06bf8..7de03a90f17 100644 --- a/src/third_party/wiredtiger/bench/workgen/workgen.h +++ b/src/third_party/wiredtiger/bench/workgen/workgen.h @@ -35,6 +35,7 @@ namespace workgen { struct ContextInternal; struct OperationInternal; struct TableInternal; +struct ThreadRunner; struct Thread; struct Transaction; @@ -171,6 +172,7 @@ struct Context { struct TableOptions { int key_size; int value_size; + uint64_t value_compressibility; bool random_value; int range; @@ -308,8 +310,8 @@ struct Operation { void create_all(); void get_static_counts(Stats &stats, int multiplier); void kv_compute_max(bool iskey, bool has_random); - void kv_gen(bool iskey, uint32_t randomizer, uint64_t n, - char *result) const; + void kv_gen(ThreadRunner *runner, bool iskey, uint64_t compressibility, + uint64_t n, char *result) const; void kv_size_buffer(bool iskey, size_t &size) const; void size_check() const; #endif diff --git a/src/third_party/wiredtiger/bench/workgen/workgen_int.h b/src/third_party/wiredtiger/bench/workgen/workgen_int.h index 33a8a8f492f..c38f709efa1 100644 --- a/src/third_party/wiredtiger/bench/workgen/workgen_int.h +++ b/src/third_party/wiredtiger/bench/workgen/workgen_int.h @@ -36,8 +36,6 @@ extern "C" { } #endif -#define RANDOMIZER_SIZE 5 /* ":000:" prefix */ - namespace workgen { // A 'tint' or ('table integer') is a unique small value integer @@ -131,6 +129,8 @@ struct ThreadRunner { uint64_t op_get_key_recno(Operation *, uint64_t range, tint_t tint); void op_get_static_counts(Operation *, Stats &, int); int op_run(Operation *); + float random_signed(); + uint32_t random_value(); #ifdef _DEBUG std::stringstream _debug_messages; diff --git a/src/third_party/wiredtiger/bench/wtperf/idle_table_cycle.c b/src/third_party/wiredtiger/bench/wtperf/idle_table_cycle.c index f84e9ddaed5..baab2177507 100644 --- a/src/third_party/wiredtiger/bench/wtperf/idle_table_cycle.c +++ b/src/third_party/wiredtiger/bench/wtperf/idle_table_cycle.c @@ -129,7 +129,7 @@ cycle_idle_tables(void *arg) session, uri, "force,checkpoint_wait=false")) == EBUSY) __wt_sleep(1, 0); - if (ret != 0 && ret != EBUSY) { + if (ret != 0) { lprintf(wtperf, ret, 0, "Table drop failed in cycle_idle_tables."); wtperf->error = true; @@ -178,5 +178,5 @@ stop_idle_table_cycle(WTPERF *wtperf, wt_thread_t idle_table_cycle_thread) return; wtperf->idle_cycle_run = false; - testutil_check(__wt_thread_join(NULL, idle_table_cycle_thread)); + testutil_check(__wt_thread_join(NULL, &idle_table_cycle_thread)); } diff --git a/src/third_party/wiredtiger/bench/wtperf/wtperf.c b/src/third_party/wiredtiger/bench/wtperf/wtperf.c index 4adb3db3c6c..047ce549746 100644 --- a/src/third_party/wiredtiger/bench/wtperf/wtperf.c +++ b/src/third_party/wiredtiger/bench/wtperf/wtperf.c @@ -473,44 +473,31 @@ do_range_reads(WTPERF *wtperf, WT_CURSOR *cursor, int64_t read_range) /* pre_load_data -- * Pull everything into cache before starting the workload phase. */ -static int +static void pre_load_data(WTPERF *wtperf) { CONFIG_OPTS *opts; WT_CONNECTION *conn; WT_CURSOR *cursor; WT_SESSION *session; - char *key; - int ret; size_t i; + int ret; + char *key; opts = wtperf->opts; conn = wtperf->conn; - if ((ret = conn->open_session( - conn, NULL, opts->sess_config, &session)) != 0) { - lprintf(wtperf, ret, 0, "worker: WT_CONNECTION.open_session"); - goto err; - } + testutil_check(conn->open_session( + conn, NULL, opts->sess_config, &session)); for (i = 0; i < opts->table_count; i++) { - if ((ret = session->open_cursor(session, - wtperf->uris[i], NULL, NULL, &cursor)) != 0) { - lprintf(wtperf, ret, 0, - "worker: WT_SESSION.open_cursor: %s", - wtperf->uris[i]); - goto err; - } - while (cursor->next(cursor) == 0) - if ((ret = cursor->get_key(cursor, &key)) != 0) - goto err; - if ((ret = cursor->close(cursor)) != 0) - goto err; - } - if ((ret = session->close(session, NULL)) != 0) - goto err; - if (ret != 0) -err: lprintf(wtperf, ret, 0, "Pre-workload traverse error"); - return (ret); + testutil_check(session->open_cursor( + session, wtperf->uris[i], NULL, NULL, &cursor)); + while ((ret = cursor->next(cursor)) == 0) + testutil_check(cursor->get_key(cursor, &key)); + testutil_assert(ret == WT_NOTFOUND); + testutil_check(cursor->close(cursor)); + } + testutil_check(session->close(session, NULL)); } static WT_THREAD_RET @@ -608,8 +595,7 @@ worker(void *arg) /* Setup for truncate */ if (workload->truncate != 0) - if ((ret = setup_truncate(wtperf, thread, session)) != 0) - goto err; + setup_truncate(wtperf, thread, session); key_buf = thread->key_buf; value_buf = thread->value_buf; @@ -1268,7 +1254,7 @@ static WT_THREAD_RET monitor(void *arg) { struct timespec t; - struct tm *tm, _tm; + struct tm localt; CONFIG_OPTS *opts; FILE *fp; WTPERF *wtperf; @@ -1337,8 +1323,9 @@ monitor(void *arg) continue; __wt_epoch(NULL, &t); - tm = localtime_r(&t.tv_sec, &_tm); - (void)strftime(buf, sizeof(buf), "%b %d %H:%M:%S", tm); + testutil_check(__wt_localtime(NULL, &t.tv_sec, &localt)); + testutil_assert( + strftime(buf, sizeof(buf), "%b %d %H:%M:%S", &localt) != 0); reads = sum_read_ops(wtperf); inserts = sum_insert_ops(wtperf); @@ -2197,7 +2184,7 @@ start_all_runs(WTPERF *wtperf) /* Wait for threads to finish. */ for (i = 0; i < opts->database_count; i++) - testutil_check(__wt_thread_join(NULL, threads[i])); + testutil_check(__wt_thread_join(NULL, &threads[i])); for (i = 0; i < opts->database_count && wtperfs[i] != NULL; i++) { wtperf_free(wtperfs[i]); @@ -2286,8 +2273,9 @@ start_run(WTPERF *wtperf) start_threads(wtperf, NULL, wtperf->ckptthreads, opts->checkpoint_threads, checkpoint_worker); } - if (opts->pre_load_data && (ret = pre_load_data(wtperf)) != 0) - goto err; + if (opts->pre_load_data) + pre_load_data(wtperf); + /* Execute the workload. */ if ((ret = execute_workload(wtperf)) != 0) goto err; @@ -2341,7 +2329,7 @@ err: if (ret == 0) stop_threads(1, wtperf->ckptthreads); if (monitor_created != 0) - testutil_check(__wt_thread_join(NULL, monitor_thread)); + testutil_check(__wt_thread_join(NULL, &monitor_thread)); if (wtperf->conn != NULL && opts->close_conn && (t_ret = wtperf->conn->close(wtperf->conn, NULL)) != 0) { @@ -2761,7 +2749,7 @@ stop_threads(u_int num, WTPERF_THREAD *threads) return; for (i = 0; i < num; ++i, ++threads) { - testutil_check(__wt_thread_join(NULL, threads->handle)); + testutil_check(__wt_thread_join(NULL, &threads->handle)); free(threads->key_buf); threads->key_buf = NULL; diff --git a/src/third_party/wiredtiger/bench/wtperf/wtperf.h b/src/third_party/wiredtiger/bench/wtperf/wtperf.h index 7fb370e0b5c..5efbe5f6e13 100644 --- a/src/third_party/wiredtiger/bench/wtperf/wtperf.h +++ b/src/third_party/wiredtiger/bench/wtperf/wtperf.h @@ -268,7 +268,7 @@ int run_truncate( WTPERF *, WTPERF_THREAD *, WT_CURSOR *, WT_SESSION *, int *); int setup_log_file(WTPERF *); void setup_throttle(WTPERF_THREAD *); -int setup_truncate(WTPERF *, WTPERF_THREAD *, WT_SESSION *); +void setup_truncate(WTPERF *, WTPERF_THREAD *, WT_SESSION *); void start_idle_table_cycle(WTPERF *, wt_thread_t *); void stop_idle_table_cycle(WTPERF *, wt_thread_t); void worker_throttle(WTPERF_THREAD *); diff --git a/src/third_party/wiredtiger/bench/wtperf/wtperf_truncate.c b/src/third_party/wiredtiger/bench/wtperf/wtperf_truncate.c index 1f910b9a3a4..83cfe3ffae0 100644 --- a/src/third_party/wiredtiger/bench/wtperf/wtperf_truncate.c +++ b/src/third_party/wiredtiger/bench/wtperf/wtperf_truncate.c @@ -34,7 +34,7 @@ decode_key(char *key_buf) return (strtoull(key_buf, NULL, 10)); } -int +void setup_truncate(WTPERF *wtperf, WTPERF_THREAD *thread, WT_SESSION *session) { CONFIG_OPTS *opts; @@ -42,9 +42,8 @@ setup_truncate(WTPERF *wtperf, WTPERF_THREAD *thread, WT_SESSION *session) TRUNCATE_QUEUE_ENTRY *truncate_item; WORKLOAD *workload; WT_CURSOR *cursor; - char *key; - int ret; uint64_t end_point, final_stone_gap, i, start_point; + char *key; opts = wtperf->opts; end_point = final_stone_gap = start_point = 0; @@ -52,9 +51,8 @@ setup_truncate(WTPERF *wtperf, WTPERF_THREAD *thread, WT_SESSION *session) workload = thread->workload; /* We are limited to only one table when running truncate. */ - if ((ret = session->open_cursor( - session, wtperf->uris[0], NULL, NULL, &cursor)) != 0) - goto err; + testutil_check(session->open_cursor( + session, wtperf->uris[0], NULL, NULL, &cursor)); /* * If we find the workload getting behind we multiply the number of @@ -79,18 +77,13 @@ setup_truncate(WTPERF *wtperf, WTPERF_THREAD *thread, WT_SESSION *session) * data available, then we need to setup some initial truncation * stones. */ - if ((ret = cursor->next(cursor)) != 0 || - (ret = cursor->get_key(cursor, &key)) != 0) { - lprintf(wtperf, ret, 0, "truncate setup start: failed"); - goto err; - } + testutil_check(cursor->next(cursor)); + testutil_check(cursor->get_key(cursor, &key)); start_point = decode_key(key); - if ((cursor->reset(cursor)) != 0 || (ret = cursor->prev(cursor)) != 0 || - (ret = cursor->get_key(cursor, &key)) != 0) { - lprintf(wtperf, ret, 0, "truncate setup end: failed"); - goto err; - } + testutil_check(cursor->reset(cursor)); + testutil_check(cursor->prev(cursor)); + testutil_check(cursor->get_key(cursor, &key)); end_point = decode_key(key); /* Assign stones if there are enough documents. */ @@ -119,10 +112,7 @@ setup_truncate(WTPERF *wtperf, WTPERF_THREAD *thread, WT_SESSION *session) } trunc_cfg->stone_gap = final_stone_gap; -err: if ((ret = cursor->close(cursor)) != 0) { - lprintf(wtperf, ret, 0, "truncate setup: cursor close failed"); - } - return (ret); + testutil_check(cursor->close(cursor)); } int diff --git a/src/third_party/wiredtiger/build_posix/aclocal/options.m4 b/src/third_party/wiredtiger/build_posix/aclocal/options.m4 index 7d0df5d65ac..9d07958bad9 100644 --- a/src/third_party/wiredtiger/build_posix/aclocal/options.m4 +++ b/src/third_party/wiredtiger/build_posix/aclocal/options.m4 @@ -53,19 +53,6 @@ AM_CONDITIONAL([HAVE_BUILTIN_EXTENSION_ZSTD], [test "$wt_cv_with_builtin_extension_zstd" = "yes"]) AC_MSG_RESULT($with_builtins) -AH_TEMPLATE( - HAVE_CRC32_HARDWARE, [Define to 1 to configure CRC32 hardware support.]) -AC_MSG_CHECKING(if --enable-crc32-hardware option specified) -AC_ARG_ENABLE(crc32-hardware, - AS_HELP_STRING([--enable-crc32-hardware], - [Enable CRC32 hardware support.]), r=$enableval, r=yes) -case "$r" in -no) wt_cv_enable_crc32_hardware=no;; -*) AC_DEFINE(HAVE_CRC32_HARDWARE) - wt_cv_enable_crc32_hardware=yes;; -esac -AC_MSG_RESULT($wt_cv_enable_crc32_hardware) - AH_TEMPLATE(HAVE_DIAGNOSTIC, [Define to 1 for diagnostic tests.]) AC_MSG_CHECKING(if --enable-diagnostic option specified) AC_ARG_ENABLE(diagnostic, diff --git a/src/third_party/wiredtiger/build_posix/aclocal/version-set.m4 b/src/third_party/wiredtiger/build_posix/aclocal/version-set.m4 index 3f87f7f6507..8b39a5d09d6 100644 --- a/src/third_party/wiredtiger/build_posix/aclocal/version-set.m4 +++ b/src/third_party/wiredtiger/build_posix/aclocal/version-set.m4 @@ -2,8 +2,8 @@ dnl build by dist/s_version VERSION_MAJOR=3 VERSION_MINOR=1 -VERSION_PATCH=0 -VERSION_STRING='"WiredTiger 3.1.0: (April 23, 2018)"' +VERSION_PATCH=1 +VERSION_STRING='"WiredTiger 3.1.1: (July 12, 2018)"' AC_SUBST(VERSION_MAJOR) AC_SUBST(VERSION_MINOR) diff --git a/src/third_party/wiredtiger/build_win/wiredtiger.def b/src/third_party/wiredtiger/build_win/wiredtiger.def index 3ee9f6b6a9d..79fa84a11e0 100644 --- a/src/third_party/wiredtiger/build_win/wiredtiger.def +++ b/src/third_party/wiredtiger/build_win/wiredtiger.def @@ -1,8 +1,8 @@ LIBRARY WIREDTIGER EXPORTS - wiredtiger_checksum_crc32c wiredtiger_config_parser_open wiredtiger_config_validate + wiredtiger_crc32c_func wiredtiger_open wiredtiger_pack_close wiredtiger_pack_int diff --git a/src/third_party/wiredtiger/build_win/wiredtiger_config.h b/src/third_party/wiredtiger/build_win/wiredtiger_config.h index 55431f59fae..bb4cc7848f8 100644 --- a/src/third_party/wiredtiger/build_win/wiredtiger_config.h +++ b/src/third_party/wiredtiger/build_win/wiredtiger_config.h @@ -25,9 +25,6 @@ /* Define to 1 if you have the `clock_gettime' function. */ /* #undef HAVE_CLOCK_GETTIME */ -/* Define to 1 to enable CRC32 hardware support. */ -/* #undef HAVE_CRC32_HARDWARE */ - /* Define to 1 for diagnostic tests. */ /* #undef HAVE_DIAGNOSTIC */ diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index d29e9655fb3..66825754a6a 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -302,6 +302,14 @@ file_config = format_meta + file_runtime_config + [ Config('leaf_item_max', '0', r''' historic term for leaf_key_max and leaf_value_max''', min=0, undoc=True), + Config('memory_page_image_max', '0', r''' + the maximum in-memory page image represented by a single storage block. + Depending on compression efficiency, compression can create storage + blocks which require significant resources to re-instantiate in the + cache, penalizing the performance of future point updates. The value + limits the maximum in-memory page image a storage block will need. If + set to 0, a default of 4 times \c leaf_page_max is used''', + min='0'), Config('memory_page_max', '5MB', r''' the maximum size a page can grow to in memory before being reconciled to disk. The specified size will be adjusted to a lower @@ -590,8 +598,7 @@ connection_runtime_config = [ type='list', undoc=True, choices=[ 'checkpoint_slow', 'lookaside_sweep_race', 'split_1', 'split_2', - 'split_3', 'split_4', 'split_5', 'split_6', 'split_7', 'split_8', - 'split_9']), + 'split_3', 'split_4', 'split_5', 'split_6', 'split_7', 'split_8']), Config('verbose', '', r''' enable messages for various events. Options are given as a list, such as <code>"verbose=[evictserver,read]"</code>''', @@ -601,6 +608,7 @@ connection_runtime_config = [ 'checkpoint', 'checkpoint_progress', 'compact', + 'error_returns', 'evict', 'evict_stuck', 'evictserver', @@ -709,7 +717,7 @@ wiredtiger_open_log_configuration = [ Config('recover', 'on', r''' run recovery or error if recovery needs to run after an unclean shutdown''', - choices=['error','on']) + choices=['error', 'on', 'salvage']) ]), ] diff --git a/src/third_party/wiredtiger/dist/log.py b/src/third_party/wiredtiger/dist/log.py index 2da8e5eae66..2503edea49d 100644 --- a/src/third_party/wiredtiger/dist/log.py +++ b/src/third_party/wiredtiger/dist/log.py @@ -331,7 +331,7 @@ for optype in log_data.optypes: }) tfile.write(''' -\tWT_ILLEGAL_VALUE(session); +\tWT_ILLEGAL_VALUE(session, optype); \t} \treturn (0); diff --git a/src/third_party/wiredtiger/dist/s_export.list b/src/third_party/wiredtiger/dist/s_export.list index 72ce553ac9b..e49fa113d96 100644 --- a/src/third_party/wiredtiger/dist/s_export.list +++ b/src/third_party/wiredtiger/dist/s_export.list @@ -1,7 +1,7 @@ # List of OK external symbols. -wiredtiger_checksum_crc32c wiredtiger_config_parser_open wiredtiger_config_validate +wiredtiger_crc32c_func wiredtiger_open wiredtiger_pack_close wiredtiger_pack_int diff --git a/src/third_party/wiredtiger/dist/s_funcs.list b/src/third_party/wiredtiger/dist/s_funcs.list index eed29e91fc1..95c568a19ff 100644 --- a/src/third_party/wiredtiger/dist/s_funcs.list +++ b/src/third_party/wiredtiger/dist/s_funcs.list @@ -33,7 +33,6 @@ __wt_stat_join_aggregate __wt_stat_join_clear_all __wt_stream_set_no_buffer __wt_try_readlock -wiredtiger_checksum_crc32c wiredtiger_config_parser_open wiredtiger_config_validate wiredtiger_pack_int diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index 64b9758877e..d9b93d709e3 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -75,6 +75,7 @@ Checksum Checksums CityHash CloseHandle +Cmvxz Cmvz Collet Comparator @@ -207,8 +208,8 @@ LLLLLLL LOGREC LOGSCAN LOOKASIDE +LRSVv LRU -LRVv LSB LSM LSN @@ -222,6 +223,7 @@ LevelDB Levyx Llqr Llqrt +LmT LoadLoad LockFile Lookaside @@ -1318,6 +1320,7 @@ vtype vunpack vw vxr +vxz vz waitpid waker diff --git a/src/third_party/wiredtiger/dist/s_void b/src/third_party/wiredtiger/dist/s_void index 9c5f6711da0..8798893597d 100755 --- a/src/third_party/wiredtiger/dist/s_void +++ b/src/third_party/wiredtiger/dist/s_void @@ -55,8 +55,9 @@ func_ok() -e '/int __wt_block_compact_start$/d' \ -e '/int __wt_block_manager_size$/d' \ -e '/int __wt_block_write_size$/d' \ + -e '/int __wt_buf_catfmt$/d' \ + -e '/int __wt_buf_fmt$/d' \ -e '/int __wt_curjoin_joined$/d' \ - -e '/int __wt_cursor_close$/d' \ -e '/int __wt_cursor_noop$/d' \ -e '/int __wt_epoch$/d' \ -e '/int __wt_errno$/d' \ @@ -65,6 +66,7 @@ func_ok() -e '/int __wt_once$/d' \ -e '/int __wt_posix_directory_list_free$/d' \ -e '/int __wt_session_breakpoint$/d' \ + -e '/int __wt_set_return_func$/d' \ -e '/int __wt_spin_init$/d' \ -e '/int __wt_spin_trylock$/d' \ -e '/int __wt_stat_connection_desc$/d' \ diff --git a/src/third_party/wiredtiger/dist/stat_data.py b/src/third_party/wiredtiger/dist/stat_data.py index f4f8f61ee1e..3144f7a5f65 100644 --- a/src/third_party/wiredtiger/dist/stat_data.py +++ b/src/third_party/wiredtiger/dist/stat_data.py @@ -332,19 +332,19 @@ connection_stats = [ LockStat('lock_checkpoint_wait_application', 'checkpoint lock application thread wait time (usecs)'), LockStat('lock_checkpoint_wait_internal', 'checkpoint lock internal thread wait time (usecs)'), LockStat('lock_commit_timestamp_read_count', 'commit timestamp queue read lock acquisitions'), - LockStat('lock_commit_timestamp_wait_application', 'commit timestamp queue lock application thread time waiting for the dhandle lock (usecs)'), - LockStat('lock_commit_timestamp_wait_internal', 'commit timestamp queue lock internal thread time waiting for the dhandle lock (usecs)'), + LockStat('lock_commit_timestamp_wait_application', 'commit timestamp queue lock application thread time waiting (usecs)'), + LockStat('lock_commit_timestamp_wait_internal', 'commit timestamp queue lock internal thread time waiting (usecs)'), LockStat('lock_commit_timestamp_write_count', 'commit timestamp queue write lock acquisitions'), LockStat('lock_dhandle_read_count', 'dhandle read lock acquisitions'), - LockStat('lock_dhandle_wait_application', 'dhandle lock application thread time waiting for the dhandle lock (usecs)'), - LockStat('lock_dhandle_wait_internal', 'dhandle lock internal thread time waiting for the dhandle lock (usecs)'), + LockStat('lock_dhandle_wait_application', 'dhandle lock application thread time waiting (usecs)'), + LockStat('lock_dhandle_wait_internal', 'dhandle lock internal thread time waiting (usecs)'), LockStat('lock_dhandle_write_count', 'dhandle write lock acquisitions'), LockStat('lock_metadata_count', 'metadata lock acquisitions'), LockStat('lock_metadata_wait_application', 'metadata lock application thread wait time (usecs)'), LockStat('lock_metadata_wait_internal', 'metadata lock internal thread wait time (usecs)'), LockStat('lock_read_timestamp_read_count', 'read timestamp queue read lock acquisitions'), - LockStat('lock_read_timestamp_wait_application', 'read timestamp queue lock application thread time waiting for the dhandle lock (usecs)'), - LockStat('lock_read_timestamp_wait_internal', 'read timestamp queue lock internal thread time waiting for the dhandle lock (usecs)'), + LockStat('lock_read_timestamp_wait_application', 'read timestamp queue lock application thread time waiting (usecs)'), + LockStat('lock_read_timestamp_wait_internal', 'read timestamp queue lock internal thread time waiting (usecs)'), LockStat('lock_read_timestamp_write_count', 'read timestamp queue write lock acquisitions'), LockStat('lock_schema_count', 'schema lock acquisitions'), LockStat('lock_schema_wait_application', 'schema lock application thread wait time (usecs)'), @@ -354,8 +354,8 @@ connection_stats = [ LockStat('lock_table_wait_internal', 'table lock internal thread time waiting for the table lock (usecs)'), LockStat('lock_table_write_count', 'table write lock acquisitions'), LockStat('lock_txn_global_read_count', 'txn global read lock acquisitions'), - LockStat('lock_txn_global_wait_application', 'txn global lock application thread time waiting for the dhandle lock (usecs)'), - LockStat('lock_txn_global_wait_internal', 'txn global lock internal thread time waiting for the dhandle lock (usecs)'), + LockStat('lock_txn_global_wait_application', 'txn global lock application thread time waiting (usecs)'), + LockStat('lock_txn_global_wait_internal', 'txn global lock internal thread time waiting (usecs)'), LockStat('lock_txn_global_write_count', 'txn global write lock acquisitions'), ########################################## diff --git a/src/third_party/wiredtiger/examples/c/ex_all.c b/src/third_party/wiredtiger/examples/c/ex_all.c index 139f39fe673..190c2c421d3 100644 --- a/src/third_party/wiredtiger/examples/c/ex_all.c +++ b/src/third_party/wiredtiger/examples/c/ex_all.c @@ -1356,8 +1356,9 @@ main(int argc, char *argv[]) const char *buffer = "some string"; size_t len = strlen(buffer); /*! [Checksum a buffer] */ - uint32_t crc32c; - crc32c = wiredtiger_checksum_crc32c(buffer, len); + uint32_t crc32c, (*func)(const void *, size_t); + func = wiredtiger_crc32c_func(); + crc32c = func(buffer, len); /*! [Checksum a buffer] */ (void)crc32c; } diff --git a/src/third_party/wiredtiger/examples/c/ex_thread.c b/src/third_party/wiredtiger/examples/c/ex_thread.c index f709707bffc..79aef70d644 100644 --- a/src/third_party/wiredtiger/examples/c/ex_thread.c +++ b/src/third_party/wiredtiger/examples/c/ex_thread.c @@ -95,7 +95,7 @@ main(int argc, char *argv[]) __wt_thread_create(NULL, &threads[i], scan_thread, conn)); for (i = 0; i < NUM_THREADS; i++) - error_check(__wt_thread_join(NULL, threads[i])); + error_check(__wt_thread_join(NULL, &threads[i])); error_check(conn->close(conn, NULL)); diff --git a/src/third_party/wiredtiger/ext/collators/revint/revint_collator.c b/src/third_party/wiredtiger/ext/collators/revint/revint_collator.c index 00c8bf93acb..b080a5e4d6a 100644 --- a/src/third_party/wiredtiger/ext/collators/revint/revint_collator.c +++ b/src/third_party/wiredtiger/ext/collators/revint/revint_collator.c @@ -80,9 +80,10 @@ revint_compare(WT_COLLATOR *collator, * when comparing primary keys. */ if ((ret = wt_api->unpack_start( - wt_api, session, "ii", k1->data, k1->size, &pstream)) != 0 || - (ret = wt_api->unpack_int(wt_api, pstream, &i1)) != 0) + wt_api, session, "ii", k1->data, k1->size, &pstream)) != 0) return (ret); + if ((ret = wt_api->unpack_int(wt_api, pstream, &i1)) != 0) + goto err; if ((ret = wt_api->unpack_int(wt_api, pstream, &p1)) != 0) /* A missing primary key is OK and sorts first. */ p1 = INT64_MIN; @@ -91,9 +92,12 @@ revint_compare(WT_COLLATOR *collator, /* Unpack the second pair of numbers. */ if ((ret = wt_api->unpack_start( - wt_api, session, "ii", k2->data, k2->size, &pstream)) != 0 || - (ret = wt_api->unpack_int(wt_api, pstream, &i2)) != 0) + wt_api, session, "ii", k2->data, k2->size, &pstream)) != 0) return (ret); + if ((ret = wt_api->unpack_int(wt_api, pstream, &i2)) != 0) { +err: (void)wt_api->pack_close(wt_api, pstream, NULL); + return (ret); + } if ((ret = wt_api->unpack_int(wt_api, pstream, &p2)) != 0) /* A missing primary key is OK and sorts first. */ p2 = INT64_MIN; @@ -138,6 +142,7 @@ int wiredtiger_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) { REVINT_COLLATOR *revint_collator; + int ret; (void)config; /* Unused parameters */ @@ -148,6 +153,10 @@ wiredtiger_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) revint_collator->collator.terminate = revint_terminate; revint_collator->wt_api = connection->get_extension_api(connection); - return (connection->add_collator( - connection, "revint", &revint_collator->collator, NULL)); + if ((ret = connection->add_collator( + connection, "revint", (WT_COLLATOR *)revint_collator, NULL)) == 0) + return (0); + + free(revint_collator); + return (ret); } diff --git a/src/third_party/wiredtiger/ext/compressors/lz4/lz4_compress.c b/src/third_party/wiredtiger/ext/compressors/lz4/lz4_compress.c index 1f32ff910d6..dc90500dcdb 100644 --- a/src/third_party/wiredtiger/ext/compressors/lz4/lz4_compress.c +++ b/src/third_party/wiredtiger/ext/compressors/lz4/lz4_compress.c @@ -375,6 +375,7 @@ static int lz_add_compressor(WT_CONNECTION *connection, bool raw, const char *name) { LZ4_COMPRESSOR *lz4_compressor; + int ret; /* * There are two almost identical LZ4 compressors: one using raw @@ -392,8 +393,12 @@ lz_add_compressor(WT_CONNECTION *connection, bool raw, const char *name) lz4_compressor->wt_api = connection->get_extension_api(connection); /* Load the compressor */ - return (connection->add_compressor( - connection, name, (WT_COMPRESSOR *)lz4_compressor, NULL)); + if ((ret = connection->add_compressor( + connection, name, (WT_COMPRESSOR *)lz4_compressor, NULL)) == 0) + return (0); + + free(lz4_compressor); + return (ret); } int lz4_extension_init(WT_CONNECTION *, WT_CONFIG_ARG *); diff --git a/src/third_party/wiredtiger/ext/compressors/nop/nop_compress.c b/src/third_party/wiredtiger/ext/compressors/nop/nop_compress.c index f739ffa4777..586f6c8831b 100644 --- a/src/third_party/wiredtiger/ext/compressors/nop/nop_compress.c +++ b/src/third_party/wiredtiger/ext/compressors/nop/nop_compress.c @@ -155,6 +155,7 @@ int wiredtiger_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) { NOP_COMPRESSOR *nop_compressor; + int ret; (void)config; /* Unused parameters */ @@ -177,7 +178,11 @@ wiredtiger_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) nop_compressor->wt_api = connection->get_extension_api(connection); /* Load the compressor */ - return (connection->add_compressor( - connection, "nop", (WT_COMPRESSOR *)nop_compressor, NULL)); + if ((ret = connection->add_compressor( + connection, "nop", (WT_COMPRESSOR *)nop_compressor, NULL)) == 0) + return (0); + + free(nop_compressor); + return (ret); } /*! [WT_COMPRESSOR initialization function] */ diff --git a/src/third_party/wiredtiger/ext/compressors/snappy/snappy_compress.c b/src/third_party/wiredtiger/ext/compressors/snappy/snappy_compress.c index 26aa3082bc0..03a17d28a1b 100644 --- a/src/third_party/wiredtiger/ext/compressors/snappy/snappy_compress.c +++ b/src/third_party/wiredtiger/ext/compressors/snappy/snappy_compress.c @@ -252,6 +252,7 @@ int snappy_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) { SNAPPY_COMPRESSOR *snappy_compressor; + int ret; (void)config; /* Unused parameters */ @@ -266,8 +267,12 @@ snappy_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) snappy_compressor->wt_api = connection->get_extension_api(connection); - return (connection->add_compressor( - connection, "snappy", (WT_COMPRESSOR *)snappy_compressor, NULL)); + if ((ret = connection->add_compressor(connection, + "snappy", (WT_COMPRESSOR *)snappy_compressor, NULL)) == 0) + return (0); + + free(snappy_compressor); + return (ret); } /* diff --git a/src/third_party/wiredtiger/ext/compressors/zlib/zlib_compress.c b/src/third_party/wiredtiger/ext/compressors/zlib/zlib_compress.c index 5ae54a25163..d5c0d0fb318 100644 --- a/src/third_party/wiredtiger/ext/compressors/zlib/zlib_compress.c +++ b/src/third_party/wiredtiger/ext/compressors/zlib/zlib_compress.c @@ -452,6 +452,7 @@ zlib_add_compressor( WT_CONNECTION *connection, bool raw, const char *name, int zlib_level) { ZLIB_COMPRESSOR *zlib_compressor; + int ret; /* * There are two almost identical zlib compressors: one using raw @@ -471,8 +472,12 @@ zlib_add_compressor( zlib_compressor->zlib_level = zlib_level; /* Load the compressor. */ - return (connection->add_compressor( - connection, name, (WT_COMPRESSOR *)zlib_compressor, NULL)); + if ((ret = connection->add_compressor( + connection, name, (WT_COMPRESSOR *)zlib_compressor, NULL)) == 0) + return (0); + + free(zlib_compressor); + return (ret); } /* diff --git a/src/third_party/wiredtiger/ext/compressors/zstd/zstd_compress.c b/src/third_party/wiredtiger/ext/compressors/zstd/zstd_compress.c index 4f80694b0d1..40a872f92e2 100644 --- a/src/third_party/wiredtiger/ext/compressors/zstd/zstd_compress.c +++ b/src/third_party/wiredtiger/ext/compressors/zstd/zstd_compress.c @@ -315,8 +315,12 @@ zstd_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) zstd_compressor->compression_level = compression_level; /* Load the compressor */ - return (connection->add_compressor( - connection, "zstd", (WT_COMPRESSOR *)zstd_compressor, NULL)); + if ((ret = connection->add_compressor( + connection, "zstd", (WT_COMPRESSOR *)zstd_compressor, NULL)) == 0) + return (0); + + free(zstd_compressor); + return (ret); } /* diff --git a/src/third_party/wiredtiger/ext/encryptors/nop/nop_encrypt.c b/src/third_party/wiredtiger/ext/encryptors/nop/nop_encrypt.c index 0d04e51e395..edefab450a0 100644 --- a/src/third_party/wiredtiger/ext/encryptors/nop/nop_encrypt.c +++ b/src/third_party/wiredtiger/ext/encryptors/nop/nop_encrypt.c @@ -168,6 +168,7 @@ int wiredtiger_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) { NOP_ENCRYPTOR *nop_encryptor; + int ret; (void)config; /* Unused parameters */ @@ -189,7 +190,11 @@ wiredtiger_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) nop_encryptor->wt_api = connection->get_extension_api(connection); /* Load the encryptor */ - return (connection->add_encryptor( - connection, "nop", (WT_ENCRYPTOR *)nop_encryptor, NULL)); + if ((ret = connection->add_encryptor( + connection, "nop", (WT_ENCRYPTOR *)nop_encryptor, NULL)) == 0) + return (0); + + free(nop_encryptor); + return (ret); } /*! [WT_ENCRYPTOR initialization function] */ diff --git a/src/third_party/wiredtiger/ext/encryptors/rotn/rotn_encrypt.c b/src/third_party/wiredtiger/ext/encryptors/rotn/rotn_encrypt.c index f1e26eeaa23..6f46a950bfc 100644 --- a/src/third_party/wiredtiger/ext/encryptors/rotn/rotn_encrypt.c +++ b/src/third_party/wiredtiger/ext/encryptors/rotn/rotn_encrypt.c @@ -479,7 +479,11 @@ wiredtiger_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) return (ret); } /* Load the encryptor */ - return (connection->add_encryptor( - connection, "rotn", (WT_ENCRYPTOR *)rotn_encryptor, NULL)); + if ((ret = connection->add_encryptor( + connection, "rotn", (WT_ENCRYPTOR *)rotn_encryptor, NULL)) == 0) + return (0); + + free(rotn_encryptor); + return (ret); } /*! [WT_ENCRYPTOR initialization function] */ diff --git a/src/third_party/wiredtiger/ext/extractors/csv/csv_extractor.c b/src/third_party/wiredtiger/ext/extractors/csv/csv_extractor.c index 6ce64e240fe..f3c01bc3f41 100644 --- a/src/third_party/wiredtiger/ext/extractors/csv/csv_extractor.c +++ b/src/third_party/wiredtiger/ext/extractors/csv/csv_extractor.c @@ -147,7 +147,6 @@ csv_customize(WT_EXTRACTOR *extractor, WT_SESSION *session, long field_num; int ret; - (void)session; /* Unused parameters */ (void)uri; /* Unused parameters */ orig = (const CSV_EXTRACTOR *)extractor; @@ -155,34 +154,61 @@ csv_customize(WT_EXTRACTOR *extractor, WT_SESSION *session, if ((ret = wt_api->config_parser_open(wt_api, session, appcfg->str, appcfg->len, &parser)) != 0) return (ret); - if ((ret = parser->get(parser, "field", &field)) != 0 || - (ret = parser->get(parser, "format", &format)) != 0) { - if (ret == WT_NOTFOUND) { + if ((ret = parser->get(parser, "field", &field)) != 0) { + if (ret == WT_NOTFOUND) (void)wt_api->err_printf( - wt_api, session, "field or format not found"); - return (WT_NOTFOUND); - } - return (ret); + wt_api, session, "field not found"); + else + (void)wt_api->err_printf( + wt_api, session, "WT_CONFIG_PARSER.get: field: %s", + wt_api->strerror(wt_api, session, ret)); + goto err; + } + if ((ret = parser->get(parser, "format", &format)) != 0) { + if (ret == WT_NOTFOUND) + (void)wt_api->err_printf( + wt_api, session, "format not found"); + else + (void)wt_api->err_printf( + wt_api, session, "WT_CONFIG_PARSER.get: format: %s", + wt_api->strerror(wt_api, session, ret)); + goto err; + } + ret = parser->close(parser); + parser = NULL; + if (ret != 0) { + (void)wt_api->err_printf( + wt_api, session, "WT_CONFIG_PARSER.close: %s", + wt_api->strerror(wt_api, session, ret)); } + field_num = strtol(field.str, NULL, 10); if (field_num < 0 || field_num > INT_MAX) { (void)wt_api->err_printf( wt_api, session, "field: invalid format"); - return (EINVAL); + ret = EINVAL; + goto err; } if (format.len != 1 || (format.str[0] != 'S' && format.str[0] != 'i')) { (void)wt_api->err_printf( wt_api, session, "format: invalid format"); - return (EINVAL); + ret = EINVAL; + goto err; + } + if ((csv_extractor = calloc(1, sizeof(CSV_EXTRACTOR))) == NULL) { + ret = errno; + goto err; } - if ((csv_extractor = calloc(1, sizeof(CSV_EXTRACTOR))) == NULL) - return (errno); *csv_extractor = *orig; csv_extractor->field = (int)field_num; csv_extractor->format_isnum = (format.str[0] == 'i'); *customp = (WT_EXTRACTOR *)csv_extractor; return (0); + +err: if (parser != NULL) + (void)parser->close(parser); + return (ret); } /* @@ -207,6 +233,7 @@ int wiredtiger_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) { CSV_EXTRACTOR *csv_extractor; + int ret; (void)config; /* Unused parameters */ @@ -218,6 +245,10 @@ wiredtiger_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) csv_extractor->extractor.terminate = csv_terminate; csv_extractor->wt_api = connection->get_extension_api(connection); - return (connection->add_extractor( - connection, "csv", (WT_EXTRACTOR *)csv_extractor, NULL)); + if ((ret = connection->add_extractor( + connection, "csv", (WT_EXTRACTOR *)csv_extractor, NULL)) == 0) + return (0); + + free(csv_extractor); + return (ret); } diff --git a/src/third_party/wiredtiger/ext/test/fail_fs/fail_fs.c b/src/third_party/wiredtiger/ext/test/fail_fs/fail_fs.c index bdb4669a637..b74144eb252 100644 --- a/src/third_party/wiredtiger/ext/test/fail_fs/fail_fs.c +++ b/src/third_party/wiredtiger/ext/test/fail_fs/fail_fs.c @@ -767,8 +767,10 @@ wiredtiger_extension_init(WT_CONNECTION *conn, WT_CONFIG_ARG *config) int64_t argval; int ret; - ret = 0; + config_parser = NULL; wtext = conn->get_extension_api(conn); + ret = 0; + if ((fail_fs = calloc(1, sizeof(FAIL_FILE_SYSTEM))) == NULL) { (void)wtext->err_printf(wtext, NULL, "fail_file_system extension_init: %s", @@ -813,7 +815,9 @@ wiredtiger_extension_init(WT_CONNECTION *conn, WT_CONFIG_ARG *config) wtext->strerror(wtext, NULL, ret)); goto err; } - if ((ret = config_parser->close(config_parser)) != 0) { + ret = config_parser->close(config_parser); + config_parser = NULL; + if (ret != 0) { (void)wtext->err_printf(wtext, NULL, "WT_CONFIG_PARSER.close: config: %s", wtext->strerror(wtext, NULL, ret)); @@ -840,6 +844,8 @@ wiredtiger_extension_init(WT_CONNECTION *conn, WT_CONFIG_ARG *config) } return (0); -err: free(fail_fs); +err: if (config_parser != NULL) + (void)config_parser->close(config_parser); + free(fail_fs); return (ret); } diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 0d25134e924..e7f5dc2f681 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -1,5 +1,5 @@ { - "commit": "e6c749653220cf701c23634cd704ae0c2d882dd9", + "commit": "2bb717625d1e81e6a0abfb910ead00afdde7fe2a", "github": "wiredtiger/wiredtiger.git", "vendor": "wiredtiger", "branch": "mongodb-4.0" diff --git a/src/third_party/wiredtiger/src/async/async_api.c b/src/third_party/wiredtiger/src/async/async_api.c index db755db198a..a4754addfdc 100644 --- a/src/third_party/wiredtiger/src/async/async_api.c +++ b/src/third_party/wiredtiger/src/async/async_api.c @@ -145,7 +145,7 @@ retry: */ if (op == NULL || op->state != WT_ASYNCOP_FREE) { WT_STAT_CONN_INCR(session, async_full); - return (EBUSY); + return (__wt_set_return(session, EBUSY)); } /* * Set the state of this op handle as READY for the user to use. @@ -400,7 +400,7 @@ __wt_async_reconfig(WT_SESSION_IMPL *session, const char *cfg[]) F_CLR(async->worker_sessions[i], WT_SESSION_SERVER_ASYNC); WT_TRET(__wt_thread_join( - session, async->worker_tids[i])); + session, &async->worker_tids[i])); wt_session = &async->worker_sessions[i]->iface; WT_TRET(wt_session->close(wt_session, NULL)); async->worker_sessions[i] = NULL; @@ -434,7 +434,7 @@ __wt_async_destroy(WT_SESSION_IMPL *session) F_CLR(conn, WT_CONN_SERVER_ASYNC); for (i = 0; i < conn->async_workers; i++) - WT_TRET(__wt_thread_join(session, async->worker_tids[i])); + WT_TRET(__wt_thread_join(session, &async->worker_tids[i])); __wt_cond_destroy(session, &async->flush_cond); /* Close the server threads' sessions. */ diff --git a/src/third_party/wiredtiger/src/async/async_worker.c b/src/third_party/wiredtiger/src/async/async_worker.c index 6dfddced8e7..3fe7a1fbe5b 100644 --- a/src/third_party/wiredtiger/src/async/async_worker.c +++ b/src/third_party/wiredtiger/src/async/async_worker.c @@ -106,8 +106,10 @@ static void __async_flush_wait(WT_SESSION_IMPL *session, WT_ASYNC *async, uint64_t my_gen) { while (async->flush_state == WT_ASYNC_FLUSHING && - async->flush_gen == my_gen) + async->flush_gen == my_gen) { __wt_cond_wait(session, async->flush_cond, 10000, NULL); + WT_BARRIER(); + } } /* diff --git a/src/third_party/wiredtiger/src/block/block_ckpt.c b/src/third_party/wiredtiger/src/block/block_ckpt.c index 73a3d13e307..ba7c2f13b1e 100644 --- a/src/third_party/wiredtiger/src/block/block_ckpt.c +++ b/src/third_party/wiredtiger/src/block/block_ckpt.c @@ -373,8 +373,7 @@ __ckpt_verify(WT_SESSION_IMPL *session, WT_CKPT *ckptbase) * on some gcc compilers because they don't understand * FALLTHROUGH as part of a macro. */ - return ( - __wt_illegal_value(session, "checkpoint array")); + return (__wt_illegal_value(session, ckpt->flags)); } return (0); } diff --git a/src/third_party/wiredtiger/src/block/block_read.c b/src/third_party/wiredtiger/src/block/block_read.c index ec44885f56a..5b3388b4d2e 100644 --- a/src/third_party/wiredtiger/src/block/block_read.c +++ b/src/third_party/wiredtiger/src/block/block_read.c @@ -278,18 +278,20 @@ __wt_block_read_off(WT_SESSION_IMPL *session, WT_BLOCK *block, if (!F_ISSET(session, WT_SESSION_QUIET_CORRUPT_FILE)) __wt_errx(session, - "read checksum error for %" PRIu32 "B block at " + "%s: read checksum error for %" PRIu32 "B block at " "offset %" PRIuMAX ": calculated block checksum " "of %" PRIu32 " doesn't match expected checksum " "of %" PRIu32, + block->name, size, (uintmax_t)offset, page_checksum, checksum); } else if (!F_ISSET(session, WT_SESSION_QUIET_CORRUPT_FILE)) __wt_errx(session, - "read checksum error for %" PRIu32 "B block at " + "%s: read checksum error for %" PRIu32 "B block at " "offset %" PRIuMAX ": block header checksum " "of %" PRIu32 " doesn't match expected checksum " "of %" PRIu32, + block->name, size, (uintmax_t)offset, swap.checksum, checksum); if (!F_ISSET(session, WT_SESSION_QUIET_CORRUPT_FILE)) @@ -297,7 +299,7 @@ __wt_block_read_off(WT_SESSION_IMPL *session, WT_BLOCK *block, __wt_bm_corrupt_dump(session, buf, offset, size, checksum)); /* Panic if a checksum fails during an ordinary read. */ - return (block->verify || - F_ISSET(session, WT_SESSION_QUIET_CORRUPT_FILE) ? - WT_ERROR : __wt_illegal_value(session, block->name)); + if (block->verify || F_ISSET(session, WT_SESSION_QUIET_CORRUPT_FILE)) + return (WT_ERROR); + WT_PANIC_RET(session, WT_ERROR, "%s: fatal read error", block->name); } diff --git a/src/third_party/wiredtiger/src/btree/bt_curnext.c b/src/third_party/wiredtiger/src/btree/bt_curnext.c index 3a031b49db5..0aa446fc974 100644 --- a/src/third_party/wiredtiger/src/btree/bt_curnext.c +++ b/src/third_party/wiredtiger/src/btree/bt_curnext.c @@ -457,7 +457,7 @@ __wt_cursor_key_order_check( return (__cursor_key_order_check_col(session, cbt, next)); case WT_PAGE_ROW_LEAF: return (__cursor_key_order_check_row(session, cbt, next)); - WT_ILLEGAL_VALUE(session); + WT_ILLEGAL_VALUE(session, cbt->ref->page->type); } /* NOTREACHED */ } @@ -482,7 +482,7 @@ __wt_cursor_key_order_init(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) case WT_PAGE_ROW_LEAF: return (__wt_buf_set(session, cbt->lastkey, cbt->iface.key.data, cbt->iface.key.size)); - WT_ILLEGAL_VALUE(session); + WT_ILLEGAL_VALUE(session, cbt->ref->page->type); } /* NOTREACHED */ } @@ -642,7 +642,7 @@ __wt_btcur_next(WT_CURSOR_BTREE *cbt, bool truncating) case WT_PAGE_COL_VAR: ret = __cursor_var_append_next(cbt, newpage); break; - WT_ILLEGAL_VALUE_ERR(session); + WT_ILLEGAL_VALUE_ERR(session, page->type); } if (ret == 0) break; @@ -660,7 +660,7 @@ __wt_btcur_next(WT_CURSOR_BTREE *cbt, bool truncating) case WT_PAGE_ROW_LEAF: ret = __cursor_row_next(cbt, newpage); break; - WT_ILLEGAL_VALUE_ERR(session); + WT_ILLEGAL_VALUE_ERR(session, page->type); } if (ret != WT_NOTFOUND) break; diff --git a/src/third_party/wiredtiger/src/btree/bt_curprev.c b/src/third_party/wiredtiger/src/btree/bt_curprev.c index 9b8ca471749..9647fdc9b2d 100644 --- a/src/third_party/wiredtiger/src/btree/bt_curprev.c +++ b/src/third_party/wiredtiger/src/btree/bt_curprev.c @@ -606,7 +606,7 @@ __wt_btcur_prev(WT_CURSOR_BTREE *cbt, bool truncating) case WT_PAGE_COL_VAR: ret = __cursor_var_append_prev(cbt, newpage); break; - WT_ILLEGAL_VALUE_ERR(session); + WT_ILLEGAL_VALUE_ERR(session, page->type); } if (ret == 0) break; @@ -626,7 +626,7 @@ __wt_btcur_prev(WT_CURSOR_BTREE *cbt, bool truncating) case WT_PAGE_ROW_LEAF: ret = __cursor_row_prev(cbt, newpage); break; - WT_ILLEGAL_VALUE_ERR(session); + WT_ILLEGAL_VALUE_ERR(session, page->type); } if (ret != WT_NOTFOUND) break; diff --git a/src/third_party/wiredtiger/src/btree/bt_cursor.c b/src/third_party/wiredtiger/src/btree/bt_cursor.c index 5ddddc5ff6e..b6172966eae 100644 --- a/src/third_party/wiredtiger/src/btree/bt_cursor.c +++ b/src/third_party/wiredtiger/src/btree/bt_cursor.c @@ -915,17 +915,17 @@ __curfile_update_check(WT_CURSOR_BTREE *cbt) int __wt_btcur_insert_check(WT_CURSOR_BTREE *cbt) { - WT_BTREE *btree; WT_CURSOR *cursor; WT_DECL_RET; WT_SESSION_IMPL *session; uint64_t yield_count, sleep_usecs; cursor = &cbt->iface; - btree = cbt->btree; session = (WT_SESSION_IMPL *)cursor->session; yield_count = sleep_usecs = 0; + WT_ASSERT(session, cbt->btree->type == BTREE_ROW); + /* * The pinned page goes away if we do a search, get a local copy of any * pinned key and discard any pinned value. Unlike most of the btree @@ -936,14 +936,10 @@ __wt_btcur_insert_check(WT_CURSOR_BTREE *cbt) __cursor_novalue(cursor); retry: WT_ERR(__cursor_func_init(cbt, true)); + WT_ERR(__cursor_row_search(session, cbt, NULL, true)); - if (btree->type == BTREE_ROW) { - WT_ERR(__cursor_row_search(session, cbt, NULL, true)); - - /* Just check for conflicts. */ - ret = __curfile_update_check(cbt); - } else - WT_ERR(__wt_illegal_value(session, NULL)); + /* Just check for conflicts. */ + ret = __curfile_update_check(cbt); err: if (ret == WT_RESTART) { __cursor_restart(session, &yield_count, &sleep_usecs); @@ -1305,8 +1301,8 @@ done: switch (modify_type) { /* * WT_CURSOR.update returns a key and a value. */ - WT_TRET(__cursor_kv_return( - session, cbt, cbt->modify_update)); + ret = __cursor_kv_return( + session, cbt, cbt->modify_update); break; case WT_UPDATE_RESERVE: /* @@ -1319,13 +1315,11 @@ done: switch (modify_type) { * WT_CURSOR.modify has already created the return value * and our job is to leave it untouched. */ - WT_TRET(__wt_key_return(session, cbt)); + ret = __wt_key_return(session, cbt); break; case WT_UPDATE_BIRTHMARK: case WT_UPDATE_TOMBSTONE: - default: - WT_TRET(__wt_illegal_value(session, NULL)); - break; + WT_ILLEGAL_VALUE(session, modify_type); } } diff --git a/src/third_party/wiredtiger/src/btree/bt_debug.c b/src/third_party/wiredtiger/src/btree/bt_debug.c index 16e25c1fe25..6eff8c53481 100644 --- a/src/third_party/wiredtiger/src/btree/bt_debug.c +++ b/src/third_party/wiredtiger/src/btree/bt_debug.c @@ -53,7 +53,7 @@ static int __debug_ref(WT_DBG *, WT_REF *); static int __debug_row_skip(WT_DBG *, WT_INSERT_HEAD *); static int __debug_tree(WT_SESSION_IMPL *, WT_REF *, const char *, uint32_t); static int __debug_update(WT_DBG *, WT_UPDATE *, bool); -static int __dmsg_wrapup(WT_DBG *); +static int __debug_wrapup(WT_DBG *); /* * __wt_debug_set_verbose -- @@ -270,7 +270,7 @@ __debug_config(WT_SESSION_IMPL *session, WT_DBG *ds, const char *ofile) ds->f = __dmsg_event; } else { if ((ds->fp = fopen(ofile, "w")) == NULL) - return (EIO); + return (__wt_set_return(session, EIO)); __wt_stream_set_line_buffer(ds->fp); ds->f = __dmsg_file; } @@ -282,12 +282,13 @@ __debug_config(WT_SESSION_IMPL *session, WT_DBG *ds, const char *ofile) } /* - * __dmsg_wrapup -- + * __debug_wrapup -- * Flush any remaining output, release resources. */ static int -__dmsg_wrapup(WT_DBG *ds) +__debug_wrapup(WT_DBG *ds) { + WT_DECL_RET; WT_ITEM *msg; WT_SESSION_IMPL *session; @@ -303,7 +304,7 @@ __dmsg_wrapup(WT_DBG *ds) */ if (msg != NULL) { if (msg->size != 0) - WT_RET(__wt_msg(session, "%s", (char *)msg->mem)); + ret = __wt_msg(session, "%s", (char *)msg->mem); __wt_scr_free(session, &ds->msg); } @@ -311,7 +312,7 @@ __dmsg_wrapup(WT_DBG *ds) if (ds->fp != NULL) (void)fclose(ds->fp); - return (0); + return (ret); } /* @@ -435,59 +436,61 @@ __wt_debug_disk( WT_SESSION_IMPL *session, const WT_PAGE_HEADER *dsk, const char *ofile) { WT_DBG *ds, _ds; + WT_DECL_RET; ds = &_ds; WT_RET(__debug_config(session, ds, ofile)); - WT_RET(ds->f(ds, "%s page", __wt_page_type_string(dsk->type))); + WT_ERR(ds->f(ds, "%s page", __wt_page_type_string(dsk->type))); switch (dsk->type) { case WT_PAGE_BLOCK_MANAGER: break; case WT_PAGE_COL_FIX: case WT_PAGE_COL_INT: case WT_PAGE_COL_VAR: - WT_RET(ds->f(ds, ", recno %" PRIu64, dsk->recno)); + WT_ERR(ds->f(ds, ", recno %" PRIu64, dsk->recno)); /* FALLTHROUGH */ case WT_PAGE_ROW_INT: case WT_PAGE_ROW_LEAF: - WT_RET(ds->f(ds, ", entries %" PRIu32, dsk->u.entries)); + WT_ERR(ds->f(ds, ", entries %" PRIu32, dsk->u.entries)); break; case WT_PAGE_OVFL: - WT_RET(ds->f(ds, ", datalen %" PRIu32, dsk->u.datalen)); + WT_ERR(ds->f(ds, ", datalen %" PRIu32, dsk->u.datalen)); break; - WT_ILLEGAL_VALUE(session); + WT_ILLEGAL_VALUE_ERR(session, dsk->type); } if (F_ISSET(dsk, WT_PAGE_COMPRESSED)) - WT_RET(ds->f(ds, ", compressed")); + WT_ERR(ds->f(ds, ", compressed")); if (F_ISSET(dsk, WT_PAGE_ENCRYPTED)) - WT_RET(ds->f(ds, ", encrypted")); + WT_ERR(ds->f(ds, ", encrypted")); if (F_ISSET(dsk, WT_PAGE_EMPTY_V_ALL)) - WT_RET(ds->f(ds, ", empty-all")); + WT_ERR(ds->f(ds, ", empty-all")); if (F_ISSET(dsk, WT_PAGE_EMPTY_V_NONE)) - WT_RET(ds->f(ds, ", empty-none")); + WT_ERR(ds->f(ds, ", empty-none")); if (F_ISSET(dsk, WT_PAGE_LAS_UPDATE)) - WT_RET(ds->f(ds, ", LAS-update")); + WT_ERR(ds->f(ds, ", LAS-update")); - WT_RET(ds->f(ds, ", generation %" PRIu64 "\n", dsk->write_gen)); + WT_ERR(ds->f(ds, ", generation %" PRIu64 "\n", dsk->write_gen)); switch (dsk->type) { case WT_PAGE_BLOCK_MANAGER: break; case WT_PAGE_COL_FIX: - WT_RET(__debug_dsk_col_fix(ds, dsk)); + WT_ERR(__debug_dsk_col_fix(ds, dsk)); break; case WT_PAGE_COL_INT: case WT_PAGE_COL_VAR: case WT_PAGE_ROW_INT: case WT_PAGE_ROW_LEAF: - WT_RET(__debug_dsk_cell(ds, dsk)); + WT_ERR(__debug_dsk_cell(ds, dsk)); break; default: break; } - return (__dmsg_wrapup(ds)); +err: WT_TRET(__debug_wrapup(ds)); + return (ret); } /* @@ -620,9 +623,9 @@ __wt_debug_tree_shape( WT_WITH_PAGE_INDEX(session, ret = __debug_tree_shape_worker(ds, page, 1)); - WT_RET(ret); - return (__dmsg_wrapup(ds)); + WT_TRET(__debug_wrapup(ds)); + return (ret); } /* AUTOMATIC FLAG VALUE GENERATION START */ @@ -705,7 +708,7 @@ __wt_debug_page( WT_WITH_BTREE(session, btree, ret = __debug_page(ds, ref, WT_DEBUG_TREE_LEAF)); - WT_TRET(__dmsg_wrapup(ds)); + WT_TRET(__debug_wrapup(ds)); return (ret); } @@ -744,7 +747,7 @@ __debug_tree( ret = __debug_page(ds, ref, flags); - WT_TRET(__dmsg_wrapup(ds)); + WT_TRET(__debug_wrapup(ds)); return (ret); } @@ -788,7 +791,7 @@ __debug_page(WT_DBG *ds, WT_REF *ref, uint32_t flags) if (LF_ISSET(WT_DEBUG_TREE_LEAF)) WT_RET(__debug_page_row_leaf(ds, ref->page)); break; - WT_ILLEGAL_VALUE(session); + WT_ILLEGAL_VALUE(session, ref->page->type); } return (0); @@ -838,7 +841,7 @@ __debug_page_metadata(WT_DBG *ds, WT_REF *ref) case WT_PAGE_ROW_LEAF: entries = page->entries; break; - WT_ILLEGAL_VALUE(session); + WT_ILLEGAL_VALUE(session, page->type); } WT_RET(ds->f(ds, ": %s\n", __wt_page_type_string(page->type))); @@ -878,7 +881,7 @@ __debug_page_metadata(WT_DBG *ds, WT_REF *ref) break; case 0: break; - WT_ILLEGAL_VALUE(session); + WT_ILLEGAL_VALUE(session, mod->rec_result); } if (split_gen != 0) WT_RET(ds->f(ds, ", split-gen=%" PRIu64, split_gen)); @@ -1398,7 +1401,7 @@ __debug_cell_data(WT_DBG *ds, case WT_CELL_VALUE_SHORT: WT_ERR(__debug_item_value(ds, tag, buf->data, buf->size)); break; - WT_ILLEGAL_VALUE_ERR(session); + WT_ILLEGAL_VALUE_ERR(session, unpack->raw); } err: __wt_scr_free(session, &buf); diff --git a/src/third_party/wiredtiger/src/btree/bt_delete.c b/src/third_party/wiredtiger/src/btree/bt_delete.c index a2f9afaf409..66127c0f0df 100644 --- a/src/third_party/wiredtiger/src/btree/bt_delete.c +++ b/src/third_party/wiredtiger/src/btree/bt_delete.c @@ -209,9 +209,7 @@ __wt_delete_page_rollback(WT_SESSION_IMPL *session, WT_REF *ref) case WT_REF_LIMBO: case WT_REF_LOOKASIDE: case WT_REF_READING: - default: - return (__wt_illegal_value(session, - "illegal WT_REF.state rolling back deleted page")); + WT_ILLEGAL_VALUE(session, current_state); } if (locked) diff --git a/src/third_party/wiredtiger/src/btree/bt_discard.c b/src/third_party/wiredtiger/src/btree/bt_discard.c index d31f76f629c..0d49adc19ca 100644 --- a/src/third_party/wiredtiger/src/btree/bt_discard.c +++ b/src/third_party/wiredtiger/src/btree/bt_discard.c @@ -32,29 +32,14 @@ __wt_ref_out(WT_SESSION_IMPL *session, WT_REF *ref) */ WT_ASSERT(session, S2BT(session)->evict_ref != ref); -#ifdef HAVE_DIAGNOSTIC - { - WT_HAZARD *hp; - int i; /* * Make sure no other thread has a hazard pointer on the page we are * about to discard. This is complicated by the fact that readers * publish their hazard pointer before re-checking the page state, so * our check can race with readers without indicating a real problem. - * Wait for up to a second for hazard pointers to be cleared. + * If we find a hazard pointer, wait for it to be cleared. */ - for (hp = NULL, i = 0; i < 100; i++) { - if ((hp = __wt_hazard_check(session, ref)) == NULL) - break; - __wt_sleep(0, 10000); - } - if (hp != NULL) - __wt_errx(session, - "discarded page has hazard pointer: (%p: %s, line %d)", - (void *)hp->ref, hp->file, hp->line); - WT_ASSERT(session, hp == NULL); - } -#endif + WT_ASSERT(session, __wt_hazard_check_assert(session, ref, true)); __wt_page_out(session, &ref->page); } @@ -263,6 +248,9 @@ __wt_free_ref( if (ref == NULL) return; + /* Assert there are no hazard pointers. */ + WT_ASSERT(session, __wt_hazard_check_assert(session, ref, false)); + /* * Optionally free the referenced pages. (The path to free referenced * page is used for error cleanup, no instantiated and then discarded diff --git a/src/third_party/wiredtiger/src/btree/bt_handle.c b/src/third_party/wiredtiger/src/btree/bt_handle.c index 9160ff1dd21..bcca39a1f17 100644 --- a/src/third_party/wiredtiger/src/btree/bt_handle.c +++ b/src/third_party/wiredtiger/src/btree/bt_handle.c @@ -447,6 +447,48 @@ __btree_conf(WT_SESSION_IMPL *session, WT_CKPT *ckpt) WT_RET(__wt_compressor_config(session, &cval, &btree->compressor)); /* + * Configure compression adjustment. + * When doing compression, assume compression rates that will result in + * pages larger than the maximum in-memory images allowed. If we're + * wrong, we adjust downward (but we're almost certainly correct, the + * maximum in-memory images allowed are only 4x the maximum page size, + * and compression always gives us more than 4x). + * Don't do compression adjustment for fixed-size column store, the + * leaf page sizes don't change. (We could adjust internal pages but not + * internal pages, but that seems an unlikely use case.) + * XXX + * Don't do compression adjustment of snappy-compressed blocks. + */ + btree->intlpage_compadjust = false; + btree->maxintlpage_precomp = btree->maxintlpage; + btree->leafpage_compadjust = false; + btree->maxleafpage_precomp = btree->maxleafpage; + if (btree->compressor != NULL && btree->compressor->compress != NULL && + !WT_STRING_MATCH("snappy", cval.str, cval.len) && + btree->type != BTREE_COL_FIX) { + /* + * Don't do compression adjustment when on-disk page sizes are + * less than 16KB. There's not enough compression going on to + * fine-tune the size, all we end up doing is hammering shared + * memory. + * + * Don't do compression adjustment when on-disk page sizes are + * equal to the maximum in-memory page image, the bytes taken + * for compression can't grow past the base value. + */ + if (btree->maxintlpage >= 16 * 1024 && + btree->maxmempage_image > btree->maxintlpage) { + btree->intlpage_compadjust = true; + btree->maxintlpage_precomp = btree->maxmempage_image; + } + if (btree->maxleafpage >= 16 * 1024 && + btree->maxmempage_image > btree->maxleafpage) { + btree->leafpage_compadjust = true; + btree->maxleafpage_precomp = btree->maxmempage_image; + } + } + + /* * We do not use __wt_config_gets_none here because "none" and the empty * string have different meanings. The empty string means inherit the * system encryption setting and "none" means this table is in the clear @@ -753,7 +795,7 @@ __btree_page_sizes(WT_SESSION_IMPL *session) WT_CONFIG_ITEM cval; WT_CONNECTION_IMPL *conn; uint64_t cache_size; - uint32_t intl_split_size, leaf_split_size; + uint32_t intl_split_size, leaf_split_size, max; const char **cfg; btree = S2BT(session); @@ -787,6 +829,22 @@ __btree_page_sizes(WT_SESSION_IMPL *session) "size (%" PRIu32 "B)", btree->allocsize); /* + * Default in-memory page image size for compression is 4x the maximum + * internal or leaf page size, and enforce the on-disk page sizes as a + * lower-limit for the in-memory image size. + */ + WT_RET(__wt_config_gets(session, cfg, "memory_page_image_max", &cval)); + btree->maxmempage_image = (uint32_t)cval.val; + max = WT_MAX(btree->maxintlpage, btree->maxleafpage); + if (btree->maxmempage_image == 0) + btree->maxmempage_image = 4 * max; + else if (btree->maxmempage_image < max) + WT_RET_MSG(session, EINVAL, + "in-memory page image size must be larger than the maximum " + "page size (%" PRIu32 "B < %" PRIu32 "B)", + btree->maxmempage_image, max); + + /* * Don't let pages grow large compared to the cache size or we can end * up in a situation where nothing can be evicted. Make sure at least * 10 pages fit in cache when it is at the dirty trigger where threads @@ -811,7 +869,7 @@ __btree_page_sizes(WT_SESSION_IMPL *session) * size. This gives multi-threaded append workloads a better chance of * not stalling. */ - btree->splitmempage = 8 * btree->maxmempage / 10; + btree->splitmempage = (8 * btree->maxmempage) / 10; /* * Get the split percentage (reconciliation splits pages into smaller @@ -827,8 +885,10 @@ __btree_page_sizes(WT_SESSION_IMPL *session) "%d%%.", session->dhandle->name, WT_BTREE_MIN_SPLIT_PCT)); } else btree->split_pct = (int)cval.val; - intl_split_size = __wt_split_page_size(btree, btree->maxintlpage); - leaf_split_size = __wt_split_page_size(btree, btree->maxleafpage); + intl_split_size = __wt_split_page_size( + btree->split_pct, btree->maxintlpage, btree->allocsize); + leaf_split_size = __wt_split_page_size( + btree->split_pct, btree->maxleafpage, btree->allocsize); /* * In-memory split configuration. @@ -915,10 +975,11 @@ __wt_btree_immediately_durable(WT_SESSION_IMPL *session) /* * This is used to determine whether timestamp updates should - * be rolled back for this btree. It's likely that the particular - * test required here will change when rollback to stable is - * supported with in-memory configurations. + * be rolled back for this btree. With in-memory, the logging + * setting on tables is still important and when enabled they + * should be considered "durable". */ - return (FLD_ISSET(S2C(session)->log_flags, WT_CONN_LOG_ENABLED) && + return ((FLD_ISSET(S2C(session)->log_flags, WT_CONN_LOG_ENABLED) || + (F_ISSET(S2C(session), WT_CONN_IN_MEMORY))) && !F_ISSET(btree, WT_BTREE_NO_LOGGING)); } diff --git a/src/third_party/wiredtiger/src/btree/bt_io.c b/src/third_party/wiredtiger/src/btree/bt_io.c index 1379553c211..eb4fdc2e0ae 100644 --- a/src/third_party/wiredtiger/src/btree/bt_io.c +++ b/src/third_party/wiredtiger/src/btree/bt_io.c @@ -153,10 +153,10 @@ corrupt: if (ret == 0) ret = WT_ERROR; if (!F_ISSET(btree, WT_BTREE_VERIFY) && !F_ISSET(session, WT_SESSION_QUIET_CORRUPT_FILE)) { - __wt_err(session, ret, "%s", fail_msg); WT_TRET(bm->corrupt(bm, session, addr, addr_size)); - WT_TRET( - __wt_illegal_value(session, btree->dhandle->name)); + WT_PANIC_ERR(session, ret, + "%s: fatal read error: %s", + btree->dhandle->name, fail_msg); } } @@ -172,7 +172,7 @@ err: __wt_scr_free(session, &tmp); */ int __wt_bt_write(WT_SESSION_IMPL *session, WT_ITEM *buf, - uint8_t *addr, size_t *addr_sizep, + uint8_t *addr, size_t *addr_sizep, size_t *compressed_sizep, bool checkpoint, bool checkpoint_io, bool compressed) { WT_BM *bm; @@ -189,6 +189,9 @@ __wt_bt_write(WT_SESSION_IMPL *session, WT_ITEM *buf, int compression_failed; /* Extension API, so not a bool. */ bool data_checksum, encrypted, timer; + if (compressed_sizep != NULL) + *compressed_sizep = 0; + btree = S2BT(session); bm = btree->bm; encrypted = false; @@ -306,6 +309,10 @@ __wt_bt_write(WT_SESSION_IMPL *session, WT_ITEM *buf, memcpy(ctmp->mem, buf->mem, WT_BLOCK_COMPRESS_SKIP); ctmp->size = result_len; ip = ctmp; + + /* Optionally return the compressed size. */ + if (compressed_sizep != NULL) + *compressed_sizep = result_len; } } /* diff --git a/src/third_party/wiredtiger/src/btree/bt_ovfl.c b/src/third_party/wiredtiger/src/btree/bt_ovfl.c index ce0ee706923..6032364fff7 100644 --- a/src/third_party/wiredtiger/src/btree/bt_ovfl.c +++ b/src/third_party/wiredtiger/src/btree/bt_ovfl.c @@ -241,7 +241,7 @@ __wt_ovfl_discard(WT_SESSION_IMPL *session, WT_CELL *cell) __wt_cell_type_reset(session, unpack->cell, WT_CELL_VALUE_OVFL, WT_CELL_VALUE_OVFL_RM); break; - WT_ILLEGAL_VALUE(session); + WT_ILLEGAL_VALUE(session, unpack->raw); } __wt_writeunlock(session, &btree->ovfl_lock); diff --git a/src/third_party/wiredtiger/src/btree/bt_page.c b/src/third_party/wiredtiger/src/btree/bt_page.c index 612540956b7..e3f5d64deb9 100644 --- a/src/third_party/wiredtiger/src/btree/bt_page.c +++ b/src/third_party/wiredtiger/src/btree/bt_page.c @@ -57,7 +57,7 @@ __wt_page_alloc(WT_SESSION_IMPL *session, */ size += alloc_entries * sizeof(WT_ROW); break; - WT_ILLEGAL_VALUE(session); + WT_ILLEGAL_VALUE(session, type); } WT_RET(__wt_calloc(session, 1, size, &page)); @@ -112,7 +112,7 @@ err: if ((pindex = WT_INTL_INDEX_GET_SAFE(page)) != NULL) { NULL : (WT_ROW *)((uint8_t *)page + sizeof(WT_PAGE)); page->entries = alloc_entries; break; - WT_ILLEGAL_VALUE(session); + WT_ILLEGAL_VALUE(session, type); } /* Increment the cache statistics. */ @@ -186,7 +186,7 @@ __wt_page_inmem(WT_SESSION_IMPL *session, WT_RET(__inmem_row_leaf_entries( session, dsk, &alloc_entries)); break; - WT_ILLEGAL_VALUE(session); + WT_ILLEGAL_VALUE(session, dsk->type); } /* Allocate and initialize a new WT_PAGE. */ @@ -222,7 +222,7 @@ __wt_page_inmem(WT_SESSION_IMPL *session, case WT_PAGE_ROW_LEAF: WT_ERR(__inmem_row_leaf(session, page)); break; - WT_ILLEGAL_VALUE_ERR(session); + WT_ILLEGAL_VALUE_ERR(session, page->type); } /* Update the page's cache statistics. */ @@ -503,7 +503,7 @@ __inmem_row_int(WT_SESSION_IMPL *session, WT_PAGE *page, size_t *sizep) ref->addr = cell; ++refp; break; - WT_ILLEGAL_VALUE_ERR(session); + WT_ILLEGAL_VALUE_ERR(session, unpack->type); } } @@ -556,7 +556,7 @@ __inmem_row_leaf_entries( case WT_CELL_VALUE: case WT_CELL_VALUE_OVFL: break; - WT_ILLEGAL_VALUE(session); + WT_ILLEGAL_VALUE(session, unpack->type); } } @@ -614,7 +614,7 @@ __inmem_row_leaf(WT_SESSION_IMPL *session, WT_PAGE *page) break; case WT_CELL_VALUE_OVFL: break; - WT_ILLEGAL_VALUE(session); + WT_ILLEGAL_VALUE(session, unpack->type); } } diff --git a/src/third_party/wiredtiger/src/btree/bt_read.c b/src/third_party/wiredtiger/src/btree/bt_read.c index c8368624d3c..58853bff6ac 100644 --- a/src/third_party/wiredtiger/src/btree/bt_read.c +++ b/src/third_party/wiredtiger/src/btree/bt_read.c @@ -221,7 +221,7 @@ __las_page_instantiate(WT_SESSION_IMPL *session, WT_REF *ref) WT_ERR(__wt_buf_set(session, current_key, las_key.data, las_key.size)); break; - WT_ILLEGAL_VALUE_ERR(session); + WT_ILLEGAL_VALUE_ERR(session, page->type); } /* Append the latest update to the list. */ @@ -251,7 +251,7 @@ __las_page_instantiate(WT_SESSION_IMPL *session, WT_REF *ref) current_key, ref, &cbt, first_upd)); first_upd = NULL; break; - WT_ILLEGAL_VALUE_ERR(session); + WT_ILLEGAL_VALUE_ERR(session, page->type); } /* Discard the cursor. */ @@ -788,7 +788,7 @@ skip_evict: /* return (LF_ISSET(WT_READ_IGNORE_CACHE_SIZE) && !F_ISSET(session, WT_SESSION_IGNORE_CACHE_SIZE) ? 0 : __wt_txn_autocommit_check(session)); - WT_ILLEGAL_VALUE(session); + WT_ILLEGAL_VALUE(session, current_state); } /* diff --git a/src/third_party/wiredtiger/src/btree/bt_rebalance.c b/src/third_party/wiredtiger/src/btree/bt_rebalance.c index 17adcdd6da6..a509bbb88bc 100644 --- a/src/third_party/wiredtiger/src/btree/bt_rebalance.c +++ b/src/third_party/wiredtiger/src/btree/bt_rebalance.c @@ -235,7 +235,7 @@ __rebalance_col_walk( unpack.type == WT_CELL_ADDR_LEAF ? WT_ADDR_LEAF : WT_ADDR_LEAF_NO, rs)); break; - WT_ILLEGAL_VALUE_ERR(session); + WT_ILLEGAL_VALUE_ERR(session, unpack.type); } } @@ -386,7 +386,7 @@ __rebalance_row_walk( first_cell = false; break; - WT_ILLEGAL_VALUE_ERR(session); + WT_ILLEGAL_VALUE_ERR(session, unpack.type); } } @@ -440,7 +440,7 @@ __wt_bt_rebalance(WT_SESSION_IMPL *session, const char *cfg[]) WT_ERR( __rebalance_col_walk(session, btree->root.page->dsk, rs)); break; - WT_ILLEGAL_VALUE_ERR(session); + WT_ILLEGAL_VALUE_ERR(session, rs->type); } /* Build a new root page. */ diff --git a/src/third_party/wiredtiger/src/btree/bt_slvg.c b/src/third_party/wiredtiger/src/btree/bt_slvg.c index 54f4eaa8f52..afb1657406a 100644 --- a/src/third_party/wiredtiger/src/btree/bt_slvg.c +++ b/src/third_party/wiredtiger/src/btree/bt_slvg.c @@ -1770,7 +1770,8 @@ __slvg_row_trk_update_start( * would have discarded it, we wouldn't be here. Therefore, this test * is safe. (But, it never hurts to check.) */ - WT_ERR_TEST(!found, WT_ERROR); + if (!found) + WT_ERR_MSG(session, WT_ERROR, "expected on-page key not found"); WT_ERR(__slvg_key_copy(session, &trk->row_start, key)); /* @@ -2350,7 +2351,7 @@ __slvg_ovfl_ref(WT_SESSION_IMPL *session, WT_TRACK *trk, bool multi_panic) { if (F_ISSET(trk, WT_TRACK_OVFL_REFD)) { if (!multi_panic) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); WT_PANIC_RET(session, EINVAL, "overflow record unexpectedly referenced multiple times " "during leaf page merge"); diff --git a/src/third_party/wiredtiger/src/btree/bt_split.c b/src/third_party/wiredtiger/src/btree/bt_split.c index a98de6c6c9f..1b4c16a8e1d 100644 --- a/src/third_party/wiredtiger/src/btree/bt_split.c +++ b/src/third_party/wiredtiger/src/btree/bt_split.c @@ -287,7 +287,7 @@ __split_ref_move(WT_SESSION_IMPL *session, WT_PAGE *from_home, case WT_CELL_ADDR_LEAF_NO: addr->type = WT_ADDR_LEAF_NO; break; - WT_ILLEGAL_VALUE_ERR(session); + WT_ILLEGAL_VALUE_ERR(session, unpack.raw); } if (__wt_atomic_cas_ptr(&ref->addr, ref_addr, addr)) addr = NULL; @@ -449,7 +449,7 @@ __split_root(WT_SESSION_IMPL *session, WT_PAGE *root) children = pindex->entries / btree->split_deepen_per_child; if (children < 10) { if (pindex->entries < 100) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); children = 10; } chunk = pindex->entries / children; @@ -872,6 +872,8 @@ __split_parent(WT_SESSION_IMPL *session, WT_REF *ref, WT_REF **ref_new, /* Free the backing block and address. */ WT_TRET(__wt_ref_block_free(session, next_ref)); + WT_ASSERT(session, + __wt_hazard_check_assert(session, next_ref, false)); WT_TRET(__split_safe_free( session, split_gen, exclusive, next_ref, sizeof(WT_REF))); parent_decr += sizeof(WT_REF); @@ -915,7 +917,7 @@ err: __wt_scr_free(session, &scr); * being deleted, but don't be noisy, there's nothing wrong. */ if (empty_parent) - ret = EBUSY; + ret = __wt_set_return(session, EBUSY); break; case WT_ERR_PANIC: __wt_err(session, ret, "fatal error during parent page split"); @@ -982,7 +984,7 @@ __split_internal(WT_SESSION_IMPL *session, WT_PAGE *parent, WT_PAGE *page) children = pindex->entries / btree->split_deepen_per_child; if (children < 10) { if (pindex->entries < 100) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); children = 10; } chunk = pindex->entries / children; @@ -1214,7 +1216,7 @@ __split_internal_lock( * the parent, give up to avoid that deadlock. */ if (!trylock && !__wt_btree_can_evict_dirty(session)) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); /* * Get a page-level lock on the parent to single-thread splits into the @@ -1511,7 +1513,7 @@ __split_multi_inmem( WT_ERR(__wt_row_modify(session, &cbt, key, NULL, upd, WT_UPDATE_INVALID, true)); break; - WT_ILLEGAL_VALUE_ERR(session); + WT_ILLEGAL_VALUE_ERR(session, orig->type); } } diff --git a/src/third_party/wiredtiger/src/btree/bt_stat.c b/src/third_party/wiredtiger/src/btree/bt_stat.c index 0fbd5ce869f..2fd23596cd7 100644 --- a/src/third_party/wiredtiger/src/btree/bt_stat.c +++ b/src/third_party/wiredtiger/src/btree/bt_stat.c @@ -122,7 +122,7 @@ __stat_page(WT_SESSION_IMPL *session, WT_PAGE *page, WT_DSRC_STATS **stats) case WT_PAGE_ROW_LEAF: __stat_page_row_leaf(session, page, stats); break; - WT_ILLEGAL_VALUE(session); + WT_ILLEGAL_VALUE(session, page->type); } return (0); } diff --git a/src/third_party/wiredtiger/src/btree/bt_sync.c b/src/third_party/wiredtiger/src/btree/bt_sync.c index 24eea097cdf..fee6f534321 100644 --- a/src/third_party/wiredtiger/src/btree/bt_sync.c +++ b/src/third_party/wiredtiger/src/btree/bt_sync.c @@ -354,7 +354,7 @@ __sync_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) break; case WT_SYNC_CLOSE: case WT_SYNC_DISCARD: - WT_ERR(__wt_illegal_value(session, NULL)); + WT_ERR(__wt_illegal_value(session, syncop)); break; } diff --git a/src/third_party/wiredtiger/src/btree/bt_vrfy_dsk.c b/src/third_party/wiredtiger/src/btree/bt_vrfy_dsk.c index aae50ed636c..727a8e5b1a7 100644 --- a/src/third_party/wiredtiger/src/btree/bt_vrfy_dsk.c +++ b/src/third_party/wiredtiger/src/btree/bt_vrfy_dsk.c @@ -166,7 +166,7 @@ __wt_verify_dsk_image(WT_SESSION_IMPL *session, case WT_PAGE_BLOCK_MANAGER: case WT_PAGE_OVFL: return (__verify_dsk_chunk(session, tag, dsk, dsk->u.datalen)); - WT_ILLEGAL_VALUE(session); + WT_ILLEGAL_VALUE(session, dsk->type); } /* NOTREACHED */ } diff --git a/src/third_party/wiredtiger/src/btree/bt_walk.c b/src/third_party/wiredtiger/src/btree/bt_walk.c index a2386d907c7..6434142824e 100644 --- a/src/third_party/wiredtiger/src/btree/bt_walk.c +++ b/src/third_party/wiredtiger/src/btree/bt_walk.c @@ -414,24 +414,41 @@ restart: /* empty_internal = false; } - /* - * Optionally return internal pages. Swap our previous - * hazard pointer for the page we'll return. We don't - * handle restart or not-found returns, it would require - * additional complexity and is not a possible return: - * we're moving to the parent of the current child page, - * the parent can't have been evicted. - */ - if (!LF_ISSET(WT_READ_SKIP_INTL)) { - WT_ERR(__wt_page_swap( - session, couple, ref, flags)); - couple = NULL; - *refp = ref; - goto done; - } - /* Encourage races. */ __wt_timing_stress(session, WT_TIMING_STRESS_SPLIT_8); + + /* Optionally return internal pages. */ + if (LF_ISSET(WT_READ_SKIP_INTL)) + continue; + + for (;;) { + /* + * Swap our previous hazard pointer for the page + * we'll return. + * + * Not-found is an expected return, as eviction + * might have been attempted. The page can't be + * evicted, we're holding a hazard pointer on a + * child, spin until we're successful. + * + * Restart is not expected, our parent WT_REF + * should not have split. + */ + ret = __wt_page_swap(session, + couple, ref, WT_READ_NOTFOUND_OK | flags); + if (ret == 0) { + /* Success, "couple" released. */ + couple = NULL; + *refp = ref; + goto done; + } + + WT_ASSERT(session, ret == WT_NOTFOUND); + WT_ERR_NOTFOUND_OK(ret); + + __wt_spin_backoff(&yield_count, &sleep_usecs); + } + /* NOTREACHED */ } if (prev) diff --git a/src/third_party/wiredtiger/src/btree/col_srch.c b/src/third_party/wiredtiger/src/btree/col_srch.c index 123b640cdf4..4b7b3d3d727 100644 --- a/src/third_party/wiredtiger/src/btree/col_srch.c +++ b/src/third_party/wiredtiger/src/btree/col_srch.c @@ -180,7 +180,7 @@ descend: /* } /* Encourage races. */ - __wt_timing_stress(session, WT_TIMING_STRESS_SPLIT_9); + WT_DIAGNOSTIC_YIELD; /* * Swap the current page for the child page. If the page splits diff --git a/src/third_party/wiredtiger/src/btree/row_srch.c b/src/third_party/wiredtiger/src/btree/row_srch.c index a3f05a2700f..38921471d74 100644 --- a/src/third_party/wiredtiger/src/btree/row_srch.c +++ b/src/third_party/wiredtiger/src/btree/row_srch.c @@ -432,7 +432,7 @@ append: if (__wt_split_descent_race( } descend: /* Encourage races. */ - __wt_timing_stress(session, WT_TIMING_STRESS_SPLIT_9); + WT_DIAGNOSTIC_YIELD; /* * Swap the current page for the child page. If the page splits diff --git a/src/third_party/wiredtiger/src/cache/cache_las.c b/src/third_party/wiredtiger/src/cache/cache_las.c index cd11a3793c5..c6ee854d8cb 100644 --- a/src/third_party/wiredtiger/src/cache/cache_las.c +++ b/src/third_party/wiredtiger/src/cache/cache_las.c @@ -657,7 +657,7 @@ __wt_las_insert_block(WT_SESSION_IMPL *session, WT_CURSOR *cursor, key->size = WT_INSERT_KEY_SIZE(list->ins); } break; - WT_ILLEGAL_VALUE_ERR(session); + WT_ILLEGAL_VALUE_ERR(session, page->type); } /* @@ -699,7 +699,7 @@ __wt_las_insert_block(WT_SESSION_IMPL *session, WT_CURSOR *cursor, case WT_UPDATE_TOMBSTONE: las_value.size = 0; break; - WT_ILLEGAL_VALUE_ERR(session); + WT_ILLEGAL_VALUE_ERR(session, upd->type); } cursor->set_key(cursor, diff --git a/src/third_party/wiredtiger/src/checksum/arm64/crc32-arm64.c b/src/third_party/wiredtiger/src/checksum/arm64/crc32-arm64.c index 01740dcd953..240c2a421bf 100644 --- a/src/third_party/wiredtiger/src/checksum/arm64/crc32-arm64.c +++ b/src/third_party/wiredtiger/src/checksum/arm64/crc32-arm64.c @@ -26,9 +26,16 @@ * OTHER DEALINGS IN THE SOFTWARE. */ -#include "wt_internal.h" +#include <inttypes.h> +#include <stddef.h> -#if defined(__linux__) && defined(HAVE_CRC32_HARDWARE) +/* + * The checksum code doesn't include WiredTiger configuration or include files. + * This means the HAVE_NO_CRC32_HARDWARE #define isn't configurable as part of + * standalone WiredTiger configuration, there's no way to turn off the checksum + * hardware. + */ +#if defined(__linux__) && !defined(HAVE_NO_CRC32_HARDWARE) #include <asm/hwcap.h> #include <sys/auxv.h> @@ -84,23 +91,27 @@ __wt_checksum_hw(const void *chunk, size_t len) } #endif +extern uint32_t __wt_checksum_sw(const void *chunk, size_t len); +#if defined(__GNUC__) +extern uint32_t (*wiredtiger_crc32c_func(void))(const void *, size_t) + __attribute__((visibility("default"))); +#else +extern uint32_t (*wiredtiger_crc32c_func(void))(const void *, size_t); +#endif + /* - * __wt_checksum_init -- - * WiredTiger: detect CRC hardware and set the checksum function. + * wiredtiger_crc32c_func -- + * WiredTiger: detect CRC hardware and return the checksum function. */ -void -__wt_checksum_init(void) - WT_GCC_FUNC_ATTRIBUTE((cold)) +uint32_t (*wiredtiger_crc32c_func(void))(const void *, size_t) { -#if defined(__linux__) && defined(HAVE_CRC32_HARDWARE) +#if defined(__linux__) && !defined(HAVE_NO_CRC32_HARDWARE) unsigned long caps = getauxval(AT_HWCAP); if (caps & HWCAP_CRC32) - __wt_process.checksum = __wt_checksum_hw; - else - __wt_process.checksum = __wt_checksum_sw; - + return (__wt_checksum_hw); + return (__wt_checksum_sw); #else - __wt_process.checksum = __wt_checksum_sw; + return (__wt_checksum_sw); #endif } diff --git a/src/third_party/wiredtiger/src/checksum/power8/crc32.sx b/src/third_party/wiredtiger/src/checksum/power8/crc32.sx index 0b7870668b5..2c9f77d3428 100644 --- a/src/third_party/wiredtiger/src/checksum/power8/crc32.sx +++ b/src/third_party/wiredtiger/src/checksum/power8/crc32.sx @@ -1,4 +1,11 @@ -#if defined(__powerpc64__) +/* + * The checksum code doesn't include WiredTiger configuration or include files. + * This means the HAVE_NO_CRC32_HARDWARE #define isn't configurable as part of + * standalone WiredTiger configuration, there's no way to turn off the checksum + * hardware. + */ +#if defined(__powerpc64__) && !defined(HAVE_NO_CRC32_HARDWARE) + /* * Calculate the checksum of data that is 16 byte aligned and a multiple of * 16 bytes. @@ -768,7 +775,6 @@ FUNC_START(__crc32_vpmsum) b .Lout FUNC_END(__crc32_vpmsum) -#endif /* * Make sure the stack isn't executable with GCC (regardless of platform). @@ -776,3 +782,4 @@ FUNC_END(__crc32_vpmsum) #ifdef __ELF__ .section .note.GNU-stack,"",@progbits #endif +#endif diff --git a/src/third_party/wiredtiger/src/checksum/power8/crc32_wrapper.c b/src/third_party/wiredtiger/src/checksum/power8/crc32_wrapper.c index 8626fa42136..074891eb0e7 100644 --- a/src/third_party/wiredtiger/src/checksum/power8/crc32_wrapper.c +++ b/src/third_party/wiredtiger/src/checksum/power8/crc32_wrapper.c @@ -1,6 +1,13 @@ -#if defined(__powerpc64__) -#include "wt_internal.h" +#include <inttypes.h> +#include <stddef.h> +/* + * The checksum code doesn't include WiredTiger configuration or include files. + * This means the HAVE_NO_CRC32_HARDWARE #define isn't configurable as part of + * standalone WiredTiger configuration, there's no way to turn off the checksum + * hardware. + */ +#if defined(__powerpc64__) && !defined(HAVE_NO_CRC32_HARDWARE) #define CRC_TABLE #include "crc32_constants.h" @@ -68,7 +75,6 @@ out: return crc; } -#endif /* * __wt_checksum_hw -- @@ -79,18 +85,25 @@ __wt_checksum_hw(const void *chunk, size_t len) { return (crc32_vpmsum(0, chunk, len)); } +#endif + +extern uint32_t __wt_checksum_sw(const void *chunk, size_t len); +#if defined(__GNUC__) +extern uint32_t (*wiredtiger_crc32c_func(void))(const void *, size_t) + __attribute__((visibility("default"))); +#else +extern uint32_t (*wiredtiger_crc32c_func(void))(const void *, size_t); +#endif /* - * __wt_checksum_init -- - * WiredTiger: detect CRC hardware and set the checksum function. + * wiredtiger_crc32c_func -- + * WiredTiger: detect CRC hardware and return the checksum function. */ -void -__wt_checksum_init(void) - WT_GCC_FUNC_ATTRIBUTE((cold)) +uint32_t (*wiredtiger_crc32c_func(void))(const void *, size_t) { -#if defined(HAVE_CRC32_HARDWARE) - __wt_process.checksum = __wt_checksum_hw; +#if defined(__powerpc64__) && !defined(HAVE_NO_CRC32_HARDWARE) + return (__wt_checksum_hw); #else - __wt_process.checksum = __wt_checksum_sw; + return (__wt_checksum_sw); #endif } diff --git a/src/third_party/wiredtiger/src/checksum/software/checksum.c b/src/third_party/wiredtiger/src/checksum/software/checksum.c index 1228c9a0ce1..4d93f8bf1ea 100644 --- a/src/third_party/wiredtiger/src/checksum/software/checksum.c +++ b/src/third_party/wiredtiger/src/checksum/software/checksum.c @@ -38,7 +38,8 @@ * little endian. */ -#include "wt_internal.h" +#include <inttypes.h> +#include <stddef.h> /* * The CRC slicing tables. @@ -1095,13 +1096,14 @@ static const uint32_t g_crc_slicing[8][256] = { #endif }; +extern uint32_t __wt_checksum_sw(const void *chunk, size_t len); + /* * __wt_checksum_sw -- * Return a checksum for a chunk of memory, computed in software. */ uint32_t __wt_checksum_sw(const void *chunk, size_t len) - WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { uint32_t crc, next; size_t nqwords; diff --git a/src/third_party/wiredtiger/src/checksum/x86/crc32-x86.c b/src/third_party/wiredtiger/src/checksum/x86/crc32-x86.c index 73199018a7d..70860019e02 100644 --- a/src/third_party/wiredtiger/src/checksum/x86/crc32-x86.c +++ b/src/third_party/wiredtiger/src/checksum/x86/crc32-x86.c @@ -26,9 +26,16 @@ * OTHER DEALINGS IN THE SOFTWARE. */ -#include "wt_internal.h" +#include <inttypes.h> +#include <stddef.h> -#if defined(HAVE_CRC32_HARDWARE) +/* + * The checksum code doesn't include WiredTiger configuration or include files. + * This means the HAVE_NO_CRC32_HARDWARE #define isn't configurable as part of + * standalone WiredTiger configuration, there's no way to turn off the checksum + * hardware. + */ +#if !defined(HAVE_NO_CRC32_HARDWARE) #if (defined(__amd64) || defined(__x86_64)) /* * __wt_checksum_hw -- @@ -116,17 +123,23 @@ __wt_checksum_hw(const void *chunk, size_t len) return (~crc); } #endif -#endif /* HAVE_CRC32_HARDWARE */ +#endif + +extern uint32_t __wt_checksum_sw(const void *chunk, size_t len); +#if defined(__GNUC__) +extern uint32_t (*wiredtiger_crc32c_func(void))(const void *, size_t) + __attribute__((visibility("default"))); +#else +extern uint32_t (*wiredtiger_crc32c_func(void))(const void *, size_t); +#endif /* - * __wt_checksum_init -- - * WiredTiger: detect CRC hardware and set the checksum function. + * wiredtiger_crc32c_func -- + * WiredTiger: detect CRC hardware and return the checksum function. */ -void -__wt_checksum_init(void) - WT_GCC_FUNC_ATTRIBUTE((cold)) +uint32_t (*wiredtiger_crc32c_func(void))(const void *, size_t) { -#if defined(HAVE_CRC32_HARDWARE) +#if !defined(HAVE_NO_CRC32_HARDWARE) #if (defined(__amd64) || defined(__x86_64)) unsigned int eax, ebx, ecx, edx; @@ -137,9 +150,8 @@ __wt_checksum_init(void) #define CPUID_ECX_HAS_SSE42 (1 << 20) if (ecx & CPUID_ECX_HAS_SSE42) - __wt_process.checksum = __wt_checksum_hw; - else - __wt_process.checksum = __wt_checksum_sw; + return (__wt_checksum_hw); + return (__wt_checksum_sw); #elif defined(_M_AMD64) int cpuInfo[4]; @@ -148,14 +160,12 @@ __wt_checksum_init(void) #define CPUID_ECX_HAS_SSE42 (1 << 20) if (cpuInfo[2] & CPUID_ECX_HAS_SSE42) - __wt_process.checksum = __wt_checksum_hw; - else - __wt_process.checksum = __wt_checksum_sw; + return (__wt_checksum_hw); + return (__wt_checksum_sw); #else - __wt_process.checksum = __wt_checksum_sw; + return (__wt_checksum_sw); +#endif +#else + return (__wt_checksum_sw); #endif - -#else /* !HAVE_CRC32_HARDWARE */ - __wt_process.checksum = __wt_checksum_sw; -#endif /* HAVE_CRC32_HARDWARE */ } diff --git a/src/third_party/wiredtiger/src/checksum/zseries/crc32-s390x.c b/src/third_party/wiredtiger/src/checksum/zseries/crc32-s390x.c index ec7adb02cba..9c7ae30bf71 100644 --- a/src/third_party/wiredtiger/src/checksum/zseries/crc32-s390x.c +++ b/src/third_party/wiredtiger/src/checksum/zseries/crc32-s390x.c @@ -6,12 +6,19 @@ * Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com> * */ -#include "wt_internal.h" #include <sys/types.h> #include <endian.h> +#include <inttypes.h> +#include <stddef.h> -#if defined(__linux__) && defined(HAVE_CRC32_HARDWARE) +/* + * The checksum code doesn't include WiredTiger configuration or include files. + * This means the HAVE_NO_CRC32_HARDWARE #define isn't configurable as part of + * standalone WiredTiger configuration, there's no way to turn off the checksum + * hardware. + */ +#if defined(__linux__) && !defined(HAVE_NO_CRC32_HARDWARE) #include <sys/auxv.h> /* RHEL 7 has kernel support, but does not define this constant in the lib c headers. */ @@ -89,26 +96,30 @@ __wt_checksum_hw(const void *chunk, size_t len) { return (~__wt_crc32c_le_vx(0xffffffff, chunk, len)); } +#endif +extern uint32_t __wt_checksum_sw(const void *chunk, size_t len); +#if defined(__GNUC__) +extern uint32_t (*wiredtiger_crc32c_func(void))(const void *, size_t) + __attribute__((visibility("default"))); +#else +extern uint32_t (*wiredtiger_crc32c_func(void))(const void *, size_t); #endif /* - * __wt_checksum_init -- - * WiredTiger: detect CRC hardware and set the checksum function. + * wiredtiger_crc32c_func -- + * WiredTiger: detect CRC hardware and return the checksum function. */ -void -__wt_checksum_init(void) - WT_GCC_FUNC_ATTRIBUTE((cold)) +uint32_t (*wiredtiger_crc32c_func(void))(const void *, size_t) { -#if defined(__linux__) && defined(HAVE_CRC32_HARDWARE) +#if defined(__linux__) && !defined(HAVE_NO_CRC32_HARDWARE) unsigned long caps = getauxval(AT_HWCAP); if (caps & HWCAP_S390_VX) - __wt_process.checksum = __wt_checksum_hw; + return (__wt_checksum_hw); else - __wt_process.checksum = __wt_checksum_sw; - + return (__wt_checksum_sw); #else - __wt_process.checksum = __wt_checksum_sw; + return (__wt_checksum_sw); #endif } diff --git a/src/third_party/wiredtiger/src/checksum/zseries/crc32le-vx.sx b/src/third_party/wiredtiger/src/checksum/zseries/crc32le-vx.sx index 0f1392b0952..358ff51b6c8 100644 --- a/src/third_party/wiredtiger/src/checksum/zseries/crc32le-vx.sx +++ b/src/third_party/wiredtiger/src/checksum/zseries/crc32le-vx.sx @@ -1,4 +1,12 @@ /* + * The checksum code doesn't include WiredTiger configuration or include files. + * This means the HAVE_NO_CRC32_HARDWARE #define isn't configurable as part of + * standalone WiredTiger configuration, there's no way to turn off the checksum + * hardware. + */ +#if !defined(HAVE_NO_CRC32_HARDWARE) + +/* * Hardware-accelerated CRC-32 variants for Linux on z Systems * * Use the z/Architecture Vector Extension Facility to accelerate the @@ -278,3 +286,4 @@ crc32_le_vgfm_generic: #ifndef __clang__ .section .note.GNU-stack,"",@progbits #endif +#endif diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index 0945d768ce2..aebfc72b098 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -190,17 +190,18 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = { { "timing_stress_for_test", "list", NULL, "choices=[\"checkpoint_slow\",\"lookaside_sweep_race\"," "\"split_1\",\"split_2\",\"split_3\",\"split_4\",\"split_5\"," - "\"split_6\",\"split_7\",\"split_8\",\"split_9\"]", + "\"split_6\",\"split_7\",\"split_8\"]", NULL, 0 }, { "verbose", "list", NULL, "choices=[\"api\",\"block\",\"checkpoint\"," - "\"checkpoint_progress\",\"compact\",\"evict\",\"evict_stuck\"," - "\"evictserver\",\"fileops\",\"handleops\",\"log\",\"lookaside\"," - "\"lookaside_activity\",\"lsm\",\"lsm_manager\",\"metadata\"," - "\"mutex\",\"overflow\",\"read\",\"rebalance\",\"reconcile\"," - "\"recovery\",\"recovery_progress\",\"salvage\",\"shared_cache\"," - "\"split\",\"temporary\",\"thread_group\",\"timestamp\"," - "\"transaction\",\"verify\",\"version\",\"write\"]", + "\"checkpoint_progress\",\"compact\",\"error_returns\",\"evict\"," + "\"evict_stuck\",\"evictserver\",\"fileops\",\"handleops\"," + "\"log\",\"lookaside\",\"lookaside_activity\",\"lsm\"," + "\"lsm_manager\",\"metadata\",\"mutex\",\"overflow\",\"read\"," + "\"rebalance\",\"reconcile\",\"recovery\",\"recovery_progress\"," + "\"salvage\",\"shared_cache\",\"split\",\"temporary\"," + "\"thread_group\",\"timestamp\",\"transaction\",\"verify\"," + "\"version\",\"write\"]", NULL, 0 }, { NULL, NULL, NULL, NULL, NULL, 0 } }; @@ -378,6 +379,7 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_create[] = { { "lsm", "category", NULL, NULL, confchk_WT_SESSION_create_lsm_subconfigs, 12 }, + { "memory_page_image_max", "int", NULL, "min=0", NULL, 0 }, { "memory_page_max", "int", NULL, "min=512B,max=10TB", NULL, 0 }, @@ -567,6 +569,7 @@ static const WT_CONFIG_CHECK confchk_file_config[] = { { "log", "category", NULL, NULL, confchk_WT_SESSION_create_log_subconfigs, 1 }, + { "memory_page_image_max", "int", NULL, "min=0", NULL, 0 }, { "memory_page_max", "int", NULL, "min=512B,max=10TB", NULL, 0 }, @@ -634,6 +637,7 @@ static const WT_CONFIG_CHECK confchk_file_meta[] = { { "log", "category", NULL, NULL, confchk_WT_SESSION_create_log_subconfigs, 1 }, + { "memory_page_image_max", "int", NULL, "min=0", NULL, 0 }, { "memory_page_max", "int", NULL, "min=512B,max=10TB", NULL, 0 }, @@ -720,6 +724,7 @@ static const WT_CONFIG_CHECK confchk_lsm_meta[] = { { "lsm", "category", NULL, NULL, confchk_WT_SESSION_create_lsm_subconfigs, 12 }, + { "memory_page_image_max", "int", NULL, "min=0", NULL, 0 }, { "memory_page_max", "int", NULL, "min=512B,max=10TB", NULL, 0 }, @@ -774,7 +779,7 @@ static const WT_CONFIG_CHECK { "path", "string", NULL, NULL, NULL, 0 }, { "prealloc", "boolean", NULL, NULL, NULL, 0 }, { "recover", "string", - NULL, "choices=[\"error\",\"on\"]", + NULL, "choices=[\"error\",\"on\",\"salvage\"]", NULL, 0 }, { "zero_fill", "boolean", NULL, NULL, NULL, 0 }, { NULL, NULL, NULL, NULL, NULL, 0 } @@ -881,7 +886,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { { "timing_stress_for_test", "list", NULL, "choices=[\"checkpoint_slow\",\"lookaside_sweep_race\"," "\"split_1\",\"split_2\",\"split_3\",\"split_4\",\"split_5\"," - "\"split_6\",\"split_7\",\"split_8\",\"split_9\"]", + "\"split_6\",\"split_7\",\"split_8\"]", NULL, 0 }, { "transaction_sync", "category", NULL, NULL, @@ -890,13 +895,14 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { { "use_environment_priv", "boolean", NULL, NULL, NULL, 0 }, { "verbose", "list", NULL, "choices=[\"api\",\"block\",\"checkpoint\"," - "\"checkpoint_progress\",\"compact\",\"evict\",\"evict_stuck\"," - "\"evictserver\",\"fileops\",\"handleops\",\"log\",\"lookaside\"," - "\"lookaside_activity\",\"lsm\",\"lsm_manager\",\"metadata\"," - "\"mutex\",\"overflow\",\"read\",\"rebalance\",\"reconcile\"," - "\"recovery\",\"recovery_progress\",\"salvage\",\"shared_cache\"," - "\"split\",\"temporary\",\"thread_group\",\"timestamp\"," - "\"transaction\",\"verify\",\"version\",\"write\"]", + "\"checkpoint_progress\",\"compact\",\"error_returns\",\"evict\"," + "\"evict_stuck\",\"evictserver\",\"fileops\",\"handleops\"," + "\"log\",\"lookaside\",\"lookaside_activity\",\"lsm\"," + "\"lsm_manager\",\"metadata\",\"mutex\",\"overflow\",\"read\"," + "\"rebalance\",\"reconcile\",\"recovery\",\"recovery_progress\"," + "\"salvage\",\"shared_cache\",\"split\",\"temporary\"," + "\"thread_group\",\"timestamp\",\"transaction\",\"verify\"," + "\"version\",\"write\"]", NULL, 0 }, { "write_through", "list", NULL, "choices=[\"data\",\"log\"]", @@ -985,7 +991,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { { "timing_stress_for_test", "list", NULL, "choices=[\"checkpoint_slow\",\"lookaside_sweep_race\"," "\"split_1\",\"split_2\",\"split_3\",\"split_4\",\"split_5\"," - "\"split_6\",\"split_7\",\"split_8\",\"split_9\"]", + "\"split_6\",\"split_7\",\"split_8\"]", NULL, 0 }, { "transaction_sync", "category", NULL, NULL, @@ -994,13 +1000,14 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { { "use_environment_priv", "boolean", NULL, NULL, NULL, 0 }, { "verbose", "list", NULL, "choices=[\"api\",\"block\",\"checkpoint\"," - "\"checkpoint_progress\",\"compact\",\"evict\",\"evict_stuck\"," - "\"evictserver\",\"fileops\",\"handleops\",\"log\",\"lookaside\"," - "\"lookaside_activity\",\"lsm\",\"lsm_manager\",\"metadata\"," - "\"mutex\",\"overflow\",\"read\",\"rebalance\",\"reconcile\"," - "\"recovery\",\"recovery_progress\",\"salvage\",\"shared_cache\"," - "\"split\",\"temporary\",\"thread_group\",\"timestamp\"," - "\"transaction\",\"verify\",\"version\",\"write\"]", + "\"checkpoint_progress\",\"compact\",\"error_returns\",\"evict\"," + "\"evict_stuck\",\"evictserver\",\"fileops\",\"handleops\"," + "\"log\",\"lookaside\",\"lookaside_activity\",\"lsm\"," + "\"lsm_manager\",\"metadata\",\"mutex\",\"overflow\",\"read\"," + "\"rebalance\",\"reconcile\",\"recovery\",\"recovery_progress\"," + "\"salvage\",\"shared_cache\",\"split\",\"temporary\"," + "\"thread_group\",\"timestamp\",\"transaction\",\"verify\"," + "\"version\",\"write\"]", NULL, 0 }, { "version", "string", NULL, NULL, NULL, 0 }, { "write_through", "list", @@ -1086,20 +1093,21 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { { "timing_stress_for_test", "list", NULL, "choices=[\"checkpoint_slow\",\"lookaside_sweep_race\"," "\"split_1\",\"split_2\",\"split_3\",\"split_4\",\"split_5\"," - "\"split_6\",\"split_7\",\"split_8\",\"split_9\"]", + "\"split_6\",\"split_7\",\"split_8\"]", NULL, 0 }, { "transaction_sync", "category", NULL, NULL, confchk_wiredtiger_open_transaction_sync_subconfigs, 2 }, { "verbose", "list", NULL, "choices=[\"api\",\"block\",\"checkpoint\"," - "\"checkpoint_progress\",\"compact\",\"evict\",\"evict_stuck\"," - "\"evictserver\",\"fileops\",\"handleops\",\"log\",\"lookaside\"," - "\"lookaside_activity\",\"lsm\",\"lsm_manager\",\"metadata\"," - "\"mutex\",\"overflow\",\"read\",\"rebalance\",\"reconcile\"," - "\"recovery\",\"recovery_progress\",\"salvage\",\"shared_cache\"," - "\"split\",\"temporary\",\"thread_group\",\"timestamp\"," - "\"transaction\",\"verify\",\"version\",\"write\"]", + "\"checkpoint_progress\",\"compact\",\"error_returns\",\"evict\"," + "\"evict_stuck\",\"evictserver\",\"fileops\",\"handleops\"," + "\"log\",\"lookaside\",\"lookaside_activity\",\"lsm\"," + "\"lsm_manager\",\"metadata\",\"mutex\",\"overflow\",\"read\"," + "\"rebalance\",\"reconcile\",\"recovery\",\"recovery_progress\"," + "\"salvage\",\"shared_cache\",\"split\",\"temporary\"," + "\"thread_group\",\"timestamp\",\"transaction\",\"verify\"," + "\"version\",\"write\"]", NULL, 0 }, { "version", "string", NULL, NULL, NULL, 0 }, { "write_through", "list", @@ -1185,20 +1193,21 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { { "timing_stress_for_test", "list", NULL, "choices=[\"checkpoint_slow\",\"lookaside_sweep_race\"," "\"split_1\",\"split_2\",\"split_3\",\"split_4\",\"split_5\"," - "\"split_6\",\"split_7\",\"split_8\",\"split_9\"]", + "\"split_6\",\"split_7\",\"split_8\"]", NULL, 0 }, { "transaction_sync", "category", NULL, NULL, confchk_wiredtiger_open_transaction_sync_subconfigs, 2 }, { "verbose", "list", NULL, "choices=[\"api\",\"block\",\"checkpoint\"," - "\"checkpoint_progress\",\"compact\",\"evict\",\"evict_stuck\"," - "\"evictserver\",\"fileops\",\"handleops\",\"log\",\"lookaside\"," - "\"lookaside_activity\",\"lsm\",\"lsm_manager\",\"metadata\"," - "\"mutex\",\"overflow\",\"read\",\"rebalance\",\"reconcile\"," - "\"recovery\",\"recovery_progress\",\"salvage\",\"shared_cache\"," - "\"split\",\"temporary\",\"thread_group\",\"timestamp\"," - "\"transaction\",\"verify\",\"version\",\"write\"]", + "\"checkpoint_progress\",\"compact\",\"error_returns\",\"evict\"," + "\"evict_stuck\",\"evictserver\",\"fileops\",\"handleops\"," + "\"log\",\"lookaside\",\"lookaside_activity\",\"lsm\"," + "\"lsm_manager\",\"metadata\",\"mutex\",\"overflow\",\"read\"," + "\"rebalance\",\"reconcile\",\"recovery\",\"recovery_progress\"," + "\"salvage\",\"shared_cache\",\"split\",\"temporary\"," + "\"thread_group\",\"timestamp\",\"transaction\",\"verify\"," + "\"version\",\"write\"]", NULL, 0 }, { "write_through", "list", NULL, "choices=[\"data\",\"log\"]", @@ -1334,11 +1343,12 @@ static const WT_CONFIG_ENTRY config_entries[] = { "bloom=true,bloom_bit_count=16,bloom_config=,bloom_hash_count=8," "bloom_oldest=false,chunk_count_limit=0,chunk_max=5GB," "chunk_size=10MB,merge_custom=(prefix=,start_generation=0," - "suffix=),merge_max=15,merge_min=0),memory_page_max=5MB," - "os_cache_dirty_max=0,os_cache_max=0,prefix_compression=false," - "prefix_compression_min=4,source=,split_deepen_min_child=0," - "split_deepen_per_child=0,split_pct=90,type=file,value_format=u", - confchk_WT_SESSION_create, 43 + "suffix=),merge_max=15,merge_min=0),memory_page_image_max=0," + "memory_page_max=5MB,os_cache_dirty_max=0,os_cache_max=0," + "prefix_compression=false,prefix_compression_min=4,source=," + "split_deepen_min_child=0,split_deepen_per_child=0,split_pct=90," + "type=file,value_format=u", + confchk_WT_SESSION_create, 44 }, { "WT_SESSION.drop", "checkpoint_wait=true,force=false,lock_wait=true," @@ -1438,11 +1448,12 @@ static const WT_CONFIG_ENTRY config_entries[] = { "internal_item_max=0,internal_key_max=0," "internal_key_truncate=true,internal_page_max=4KB,key_format=u," "key_gap=10,leaf_item_max=0,leaf_key_max=0,leaf_page_max=32KB," - "leaf_value_max=0,log=(enabled=true),memory_page_max=5MB," - "os_cache_dirty_max=0,os_cache_max=0,prefix_compression=false," - "prefix_compression_min=4,split_deepen_min_child=0," - "split_deepen_per_child=0,split_pct=90,value_format=u", - confchk_file_config, 36 + "leaf_value_max=0,log=(enabled=true),memory_page_image_max=0," + "memory_page_max=5MB,os_cache_dirty_max=0,os_cache_max=0," + "prefix_compression=false,prefix_compression_min=4," + "split_deepen_min_child=0,split_deepen_per_child=0,split_pct=90," + "value_format=u", + confchk_file_config, 37 }, { "file.meta", "access_pattern_hint=none,allocation_size=4KB,app_metadata=," @@ -1455,11 +1466,12 @@ static const WT_CONFIG_ENTRY config_entries[] = { "internal_key_max=0,internal_key_truncate=true," "internal_page_max=4KB,key_format=u,key_gap=10,leaf_item_max=0," "leaf_key_max=0,leaf_page_max=32KB,leaf_value_max=0," - "log=(enabled=true),memory_page_max=5MB,os_cache_dirty_max=0," - "os_cache_max=0,prefix_compression=false,prefix_compression_min=4" - ",split_deepen_min_child=0,split_deepen_per_child=0,split_pct=90," - "value_format=u,version=(major=0,minor=0)", - confchk_file_meta, 40 + "log=(enabled=true),memory_page_image_max=0,memory_page_max=5MB," + "os_cache_dirty_max=0,os_cache_max=0,prefix_compression=false," + "prefix_compression_min=4,split_deepen_min_child=0," + "split_deepen_per_child=0,split_pct=90,value_format=u," + "version=(major=0,minor=0)", + confchk_file_meta, 41 }, { "index.meta", "app_metadata=,collator=,columns=,extractor=,immutable=false," @@ -1481,11 +1493,12 @@ static const WT_CONFIG_ENTRY config_entries[] = { "bloom_config=,bloom_hash_count=8,bloom_oldest=false," "chunk_count_limit=0,chunk_max=5GB,chunk_size=10MB," "merge_custom=(prefix=,start_generation=0,suffix=),merge_max=15," - "merge_min=0),memory_page_max=5MB,old_chunks=," - "os_cache_dirty_max=0,os_cache_max=0,prefix_compression=false," - "prefix_compression_min=4,split_deepen_min_child=0," - "split_deepen_per_child=0,split_pct=90,value_format=u", - confchk_lsm_meta, 40 + "merge_min=0),memory_page_image_max=0,memory_page_max=5MB," + "old_chunks=,os_cache_dirty_max=0,os_cache_max=0," + "prefix_compression=false,prefix_compression_min=4," + "split_deepen_min_child=0,split_deepen_per_child=0,split_pct=90," + "value_format=u", + confchk_lsm_meta, 41 }, { "table.meta", "app_metadata=,colgroups=,collator=,columns=,key_format=u," diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index 589560acc88..47ab622a7f4 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -1832,6 +1832,7 @@ __wt_verbose_config(WT_SESSION_IMPL *session, const char *cfg[]) { "checkpoint", WT_VERB_CHECKPOINT }, { "checkpoint_progress",WT_VERB_CHECKPOINT_PROGRESS }, { "compact", WT_VERB_COMPACT }, + { "error_returns", WT_VERB_ERROR_RETURNS }, { "evict", WT_VERB_EVICT }, { "evict_stuck", WT_VERB_EVICT_STUCK }, { "evictserver", WT_VERB_EVICTSERVER }, @@ -2025,7 +2026,6 @@ __wt_timing_stress_config(WT_SESSION_IMPL *session, const char *cfg[]) { "split_6", WT_TIMING_STRESS_SPLIT_6 }, { "split_7", WT_TIMING_STRESS_SPLIT_7 }, { "split_8", WT_TIMING_STRESS_SPLIT_8 }, - { "split_9", WT_TIMING_STRESS_SPLIT_9 }, { NULL, 0 } }; WT_CONFIG_ITEM cval, sval; @@ -2750,15 +2750,3 @@ err: /* Discard the scratch buffers. */ return (ret); } - -/* - * wiredtiger_checksum_crc32c -- - * CRC32C checksum function entry point. - */ -uint32_t -wiredtiger_checksum_crc32c(const void *buffer, size_t len) -{ - if (__wt_process.checksum == NULL) - __wt_checksum_init(); - return (__wt_process.checksum(buffer, len)); -} diff --git a/src/third_party/wiredtiger/src/conn/conn_cache_pool.c b/src/third_party/wiredtiger/src/conn/conn_cache_pool.c index f1043ee7546..63fd9486823 100644 --- a/src/third_party/wiredtiger/src/conn/conn_cache_pool.c +++ b/src/third_party/wiredtiger/src/conn/conn_cache_pool.c @@ -342,7 +342,7 @@ __wt_conn_cache_pool_destroy(WT_SESSION_IMPL *session) FLD_CLR(cache->pool_flags, WT_CACHE_POOL_RUN); __wt_cond_signal(session, cp->cache_pool_cond); - WT_TRET(__wt_thread_join(session, cache->cp_tid)); + WT_TRET(__wt_thread_join(session, &cache->cp_tid)); wt_session = &cache->cp_session->iface; WT_TRET(wt_session->close(wt_session, NULL)); diff --git a/src/third_party/wiredtiger/src/conn/conn_ckpt.c b/src/third_party/wiredtiger/src/conn/conn_ckpt.c index 8691a72cc47..99bfdca0331 100644 --- a/src/third_party/wiredtiger/src/conn/conn_ckpt.c +++ b/src/third_party/wiredtiger/src/conn/conn_ckpt.c @@ -219,7 +219,7 @@ __wt_checkpoint_server_destroy(WT_SESSION_IMPL *session) F_CLR(conn, WT_CONN_SERVER_CHECKPOINT); if (conn->ckpt_tid_set) { __wt_cond_signal(session, conn->ckpt_cond); - WT_TRET(__wt_thread_join(session, conn->ckpt_tid)); + WT_TRET(__wt_thread_join(session, &conn->ckpt_tid)); conn->ckpt_tid_set = false; } __wt_cond_destroy(session, &conn->ckpt_cond); diff --git a/src/third_party/wiredtiger/src/conn/conn_dhandle.c b/src/third_party/wiredtiger/src/conn/conn_dhandle.c index 7c24f3c126f..eeaa71683f1 100644 --- a/src/third_party/wiredtiger/src/conn/conn_dhandle.c +++ b/src/third_party/wiredtiger/src/conn/conn_dhandle.c @@ -47,7 +47,7 @@ __conn_dhandle_config_set(WT_SESSION_IMPL *session) if ((ret = __wt_metadata_search(session, dhandle->name, &metaconf)) != 0) { if (ret == WT_NOTFOUND) - ret = ENOENT; + ret = __wt_set_return(session, ENOENT); WT_RET(ret); } @@ -140,10 +140,11 @@ __wt_conn_dhandle_alloc( dhandle->type = WT_DHANDLE_TYPE_BTREE; } else if (WT_PREFIX_MATCH(uri, "table:")) { WT_RET(__wt_calloc_one(session, &table)); - dhandle = &table->iface; + dhandle = (WT_DATA_HANDLE *)table; dhandle->type = WT_DHANDLE_TYPE_TABLE; } else - return (__wt_illegal_value(session, NULL)); + WT_PANIC_RET(session, EINVAL, + "illegal handle allocation URI %s", uri); /* Btree handles keep their data separate from the interface. */ if (dhandle->type == WT_DHANDLE_TYPE_BTREE) { @@ -703,7 +704,7 @@ __conn_dhandle_remove(WT_SESSION_IMPL *session, bool final) /* Check if the handle was reacquired by a session while we waited. */ if (!final && (dhandle->session_inuse != 0 || dhandle->session_ref != 0)) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); WT_CONN_DHANDLE_REMOVE(conn, dhandle, bucket); return (0); diff --git a/src/third_party/wiredtiger/src/conn/conn_log.c b/src/third_party/wiredtiger/src/conn/conn_log.c index d8eb095d6d2..d42789c809e 100644 --- a/src/third_party/wiredtiger/src/conn/conn_log.c +++ b/src/third_party/wiredtiger/src/conn/conn_log.c @@ -299,6 +299,13 @@ __logmgr_config( session, cfg, "log.recover", 0, &cval)); if (WT_STRING_MATCH("error", cval.str, cval.len)) FLD_SET(conn->log_flags, WT_CONN_LOG_RECOVER_ERR); + else if (WT_STRING_MATCH("salvage", cval.str, cval.len)) { + if (F_ISSET(conn, WT_CONN_READONLY)) + WT_RET_MSG(session, EINVAL, + "Readonly configuration incompatible with " + "log=(recover=salvage)"); + FLD_SET(conn->log_flags, WT_CONN_LOG_RECOVER_SALVAGE); + } } WT_RET(__wt_config_gets(session, cfg, "log.zero_fill", &cval)); @@ -1171,12 +1178,12 @@ __wt_logmgr_destroy(WT_SESSION_IMPL *session) } if (conn->log_tid_set) { __wt_cond_signal(session, conn->log_cond); - WT_TRET(__wt_thread_join(session, conn->log_tid)); + WT_TRET(__wt_thread_join(session, &conn->log_tid)); conn->log_tid_set = false; } if (conn->log_file_tid_set) { __wt_cond_signal(session, conn->log_file_cond); - WT_TRET(__wt_thread_join(session, conn->log_file_tid)); + WT_TRET(__wt_thread_join(session, &conn->log_file_tid)); conn->log_file_tid_set = false; } if (conn->log_file_session != NULL) { @@ -1186,7 +1193,7 @@ __wt_logmgr_destroy(WT_SESSION_IMPL *session) } if (conn->log_wrlsn_tid_set) { __wt_cond_signal(session, conn->log_wrlsn_cond); - WT_TRET(__wt_thread_join(session, conn->log_wrlsn_tid)); + WT_TRET(__wt_thread_join(session, &conn->log_wrlsn_tid)); conn->log_wrlsn_tid_set = false; } if (conn->log_wrlsn_session != NULL) { diff --git a/src/third_party/wiredtiger/src/conn/conn_stat.c b/src/third_party/wiredtiger/src/conn/conn_stat.c index 14a1570c138..ffbc1caf2bb 100644 --- a/src/third_party/wiredtiger/src/conn/conn_stat.c +++ b/src/third_party/wiredtiger/src/conn/conn_stat.c @@ -501,17 +501,17 @@ static int __statlog_log_one(WT_SESSION_IMPL *session, WT_ITEM *path, WT_ITEM *tmp) { struct timespec ts; - struct tm *tm, _tm; + struct tm localt; WT_CONNECTION_IMPL *conn; conn = S2C(session); /* Get the current local time of day. */ __wt_epoch(session, &ts); - tm = localtime_r(&ts.tv_sec, &_tm); + WT_RET(__wt_localtime(session, &ts.tv_sec, &localt)); /* Create the logging path name for this time of day. */ - if (strftime(tmp->mem, tmp->memsize, conn->stat_path, tm) == 0) + if (strftime(tmp->mem, tmp->memsize, conn->stat_path, &localt) == 0) WT_RET_MSG(session, ENOMEM, "strftime path conversion"); /* If the path has changed, cycle the log file. */ @@ -527,7 +527,7 @@ __statlog_log_one(WT_SESSION_IMPL *session, WT_ITEM *path, WT_ITEM *tmp) } /* Create the entry prefix for this time of day. */ - if (strftime(tmp->mem, tmp->memsize, conn->stat_format, tm) == 0) + if (strftime(tmp->mem, tmp->memsize, conn->stat_format, &localt) == 0) WT_RET_MSG(session, ENOMEM, "strftime timestamp conversion"); conn->stat_stamp = tmp->mem; WT_RET(__statlog_print_header(session)); @@ -742,7 +742,7 @@ __wt_statlog_destroy(WT_SESSION_IMPL *session, bool is_close) F_CLR(conn, WT_CONN_SERVER_STATISTICS); if (conn->stat_tid_set) { __wt_cond_signal(session, conn->stat_cond); - WT_TRET(__wt_thread_join(session, conn->stat_tid)); + WT_TRET(__wt_thread_join(session, &conn->stat_tid)); conn->stat_tid_set = false; } __wt_cond_destroy(session, &conn->stat_cond); diff --git a/src/third_party/wiredtiger/src/conn/conn_sweep.c b/src/third_party/wiredtiger/src/conn/conn_sweep.c index 33342fb4873..734c455d854 100644 --- a/src/third_party/wiredtiger/src/conn/conn_sweep.c +++ b/src/third_party/wiredtiger/src/conn/conn_sweep.c @@ -451,7 +451,7 @@ __wt_sweep_destroy(WT_SESSION_IMPL *session) F_CLR(conn, WT_CONN_SERVER_SWEEP); if (conn->sweep_tid_set) { __wt_cond_signal(session, conn->sweep_cond); - WT_TRET(__wt_thread_join(session, conn->sweep_tid)); + WT_TRET(__wt_thread_join(session, &conn->sweep_tid)); conn->sweep_tid_set = 0; } __wt_cond_destroy(session, &conn->sweep_cond); diff --git a/src/third_party/wiredtiger/src/cursor/cur_backup.c b/src/third_party/wiredtiger/src/cursor/cur_backup.c index a9e08cfa4d8..b6da199f731 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_backup.c +++ b/src/third_party/wiredtiger/src/cursor/cur_backup.c @@ -77,8 +77,8 @@ __curbackup_close(WT_CURSOR *cursor) WT_SESSION_IMPL *session; cb = (WT_CURSOR_BACKUP *)cursor; - CURSOR_API_CALL_PREPARE_ALLOWED(cursor, session, close, NULL); +err: /* * When starting a hot backup, we serialize hot backup cursors and set @@ -92,10 +92,10 @@ __curbackup_close(WT_CURSOR *cursor) if (F_ISSET(cb, WT_CURBACKUP_LOCKER)) WT_TRET(__backup_stop(session, cb)); - WT_TRET(__wt_cursor_close(cursor)); + __wt_cursor_close(cursor); session->bkp_cursor = NULL; -err: API_END_RET(session, ret); + API_END_RET(session, ret); } /* @@ -133,17 +133,15 @@ __wt_curbackup_open(WT_SESSION_IMPL *session, WT_STATIC_ASSERT(offsetof(WT_CURSOR_BACKUP, iface) == 0); - cb = NULL; - WT_RET(__wt_calloc_one(session, &cb)); - cursor = &cb->iface; + cursor = (WT_CURSOR *)cb; *cursor = iface; - cursor->session = &session->iface; - session->bkp_cursor = cb; - + cursor->session = (WT_SESSION *)session; cursor->key_format = "S"; /* Return the file names as the key. */ cursor->value_format = ""; /* No value. */ + session->bkp_cursor = cb; + /* * Start the backup and fill in the cursor's list. Acquire the schema * lock, we need a consistent view when creating a copy. diff --git a/src/third_party/wiredtiger/src/cursor/cur_config.c b/src/third_party/wiredtiger/src/cursor/cur_config.c index 98c59392161..e4d1f3e94e0 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_config.c +++ b/src/third_party/wiredtiger/src/cursor/cur_config.c @@ -19,9 +19,11 @@ __curconfig_close(WT_CURSOR *cursor) WT_SESSION_IMPL *session; CURSOR_API_CALL_PREPARE_ALLOWED(cursor, session, close, NULL); - WT_TRET(__wt_cursor_close(cursor)); +err: -err: API_END_RET(session, ret); + __wt_cursor_close(cursor); + + API_END_RET(session, ret); } /* @@ -60,10 +62,9 @@ __wt_curconfig_open(WT_SESSION_IMPL *session, WT_STATIC_ASSERT(offsetof(WT_CURSOR_CONFIG, iface) == 0); WT_RET(__wt_calloc_one(session, &cconfig)); - - cursor = &cconfig->iface; + cursor = (WT_CURSOR *)cconfig; *cursor = iface; - cursor->session = &session->iface; + cursor->session = (WT_SESSION *)session; cursor->key_format = cursor->value_format = "S"; WT_ERR(__wt_cursor_init(cursor, uri, NULL, cfg, cursorp)); diff --git a/src/third_party/wiredtiger/src/cursor/cur_ds.c b/src/third_party/wiredtiger/src/cursor/cur_ds.c index 1eb778ed0c9..d82fa934ddf 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_ds.c +++ b/src/third_party/wiredtiger/src/cursor/cur_ds.c @@ -443,11 +443,11 @@ __curds_close(WT_CURSOR *cursor) WT_SESSION_IMPL *session; cds = (WT_CURSOR_DATA_SOURCE *)cursor; - CURSOR_API_CALL_PREPARE_ALLOWED(cursor, session, close, NULL); +err: if (cds->source != NULL) - ret = cds->source->close(cds->source); + WT_TRET(cds->source->close(cds->source)); if (cds->collator_owned) { if (cds->collator->terminate != NULL) @@ -464,9 +464,9 @@ __curds_close(WT_CURSOR *cursor) __wt_free(session, cursor->key_format); __wt_free(session, cursor->value_format); - WT_TRET(__wt_cursor_close(cursor)); + __wt_cursor_close(cursor); -err: API_END_RET(session, ret); + API_END_RET(session, ret); } /* @@ -507,13 +507,12 @@ __wt_curds_open( WT_STATIC_ASSERT(offsetof(WT_CURSOR_DATA_SOURCE, iface) == 0); - data_source = NULL; metaconf = NULL; WT_RET(__wt_calloc_one(session, &data_source)); - cursor = &data_source->iface; + cursor = (WT_CURSOR *)data_source; *cursor = iface; - cursor->session = &session->iface; + cursor->session = (WT_SESSION *)session; /* * XXX diff --git a/src/third_party/wiredtiger/src/cursor/cur_dump.c b/src/third_party/wiredtiger/src/cursor/cur_dump.c index 8853e6f30d6..2ac8823ddb9 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_dump.c +++ b/src/third_party/wiredtiger/src/cursor/cur_dump.c @@ -337,16 +337,17 @@ __curdump_close(WT_CURSOR *cursor) cdump = (WT_CURSOR_DUMP *)cursor; child = cdump->child; - CURSOR_API_CALL_PREPARE_ALLOWED(cursor, session, close, NULL); +err: + if (child != NULL) WT_TRET(child->close(child)); /* We shared the child's URI. */ cursor->internal_uri = NULL; __wt_json_close(session, cursor); - WT_TRET(__wt_cursor_close(cursor)); + __wt_cursor_close(cursor); -err: API_END_RET(session, ret); + API_END_RET(session, ret); } /* @@ -389,7 +390,7 @@ __wt_curdump_create(WT_CURSOR *child, WT_CURSOR *owner, WT_CURSOR **cursorp) session = (WT_SESSION_IMPL *)child->session; WT_RET(__wt_calloc_one(session, &cdump)); - cursor = &cdump->iface; + cursor = (WT_CURSOR *)cdump; *cursor = iface; cursor->session = child->session; cursor->internal_uri = child->internal_uri; diff --git a/src/third_party/wiredtiger/src/cursor/cur_file.c b/src/third_party/wiredtiger/src/cursor/cur_file.c index 4f81cc0c10b..1c3fcc29492 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_file.c +++ b/src/third_party/wiredtiger/src/cursor/cur_file.c @@ -469,15 +469,18 @@ __curfile_close(WT_CURSOR *cursor) cbt = (WT_CURSOR_BTREE *)cursor; CURSOR_API_CALL_PREPARE_ALLOWED(cursor, session, close, cbt->btree); - released = false; +err: - /* - * If releasing the cursor fails in any way, it will be left - * in a state that allows it to be normally closed. - */ - WT_TRET(__wt_cursor_cache_release(session, cursor, &released)); - if (released) - goto done; + /* Only try to cache the cursor if there's no error. */ + if (ret == 0) { + /* + * If releasing the cursor fails in any way, it will be left in + * a state that allows it to be normally closed. + */ + ret = __wt_cursor_cache_release(session, cursor, &released); + if (released) + goto done; + } dead = F_ISSET(cursor, WT_CURSTD_DEAD); if (F_ISSET(cursor, WT_CURSTD_BULK)) { @@ -494,7 +497,7 @@ __curfile_close(WT_CURSOR *cursor) WT_ASSERT(session, session->dhandle == NULL || session->dhandle->session_inuse > 0); - WT_TRET(__wt_cursor_close(cursor)); + __wt_cursor_close(cursor); /* * Note: release the data handle last so that cursor statistics are @@ -513,8 +516,7 @@ __curfile_close(WT_CURSOR *cursor) WT_TRET(__wt_session_release_dhandle(session)); } -done: -err: API_END_RET(session, ret); +done: API_END_RET(session, ret); } /* @@ -632,18 +634,16 @@ __curfile_create(WT_SESSION_IMPL *session, WT_STATIC_ASSERT(offsetof(WT_CURSOR_BTREE, iface) == 0); - cbt = NULL; - cacheable = F_ISSET(session, WT_SESSION_CACHE_CURSORS) && !bulk; - btree = S2BT(session); WT_ASSERT(session, btree != NULL); csize = bulk ? sizeof(WT_CURSOR_BULK) : sizeof(WT_CURSOR_BTREE); - WT_RET(__wt_calloc(session, 1, csize, &cbt)); + cacheable = F_ISSET(session, WT_SESSION_CACHE_CURSORS) && !bulk; - cursor = &cbt->iface; + WT_RET(__wt_calloc(session, 1, csize, &cbt)); + cursor = (WT_CURSOR *)cbt; *cursor = iface; - cursor->session = &session->iface; + cursor->session = (WT_SESSION *)session; cursor->internal_uri = btree->dhandle->name; cursor->key_format = btree->key_format; cursor->value_format = btree->value_format; @@ -724,8 +724,7 @@ err: /* * Our caller expects to release the data handle if we fail. * Disconnect it from the cursor before closing. */ - if (session->dhandle != NULL) - __wt_cursor_dhandle_decr_use(session); + __wt_cursor_dhandle_decr_use(session); cbt->btree = NULL; WT_TRET(__curfile_close(cursor)); *cursorp = NULL; diff --git a/src/third_party/wiredtiger/src/cursor/cur_index.c b/src/third_party/wiredtiger/src/cursor/cur_index.c index 9e75442a243..627bfbe2f44 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_index.c +++ b/src/third_party/wiredtiger/src/cursor/cur_index.c @@ -359,8 +359,8 @@ __curindex_close(WT_CURSOR *cursor) cindex = (WT_CURSOR_INDEX *)cursor; idx = cindex->index; - JOINABLE_CURSOR_API_CALL_PREPARE_ALLOWED(cursor, session, close, NULL); +err: if ((cp = cindex->cg_cursors) != NULL) for (i = 0, cp = cindex->cg_cursors; @@ -385,9 +385,9 @@ __curindex_close(WT_CURSOR *cursor) WT_TRET(__wt_schema_release_table(session, cindex->table)); /* The URI is owned by the index. */ cursor->internal_uri = NULL; - WT_TRET(__wt_cursor_close(cursor)); + __wt_cursor_close(cursor); -err: API_END_RET(session, ret); + API_END_RET(session, ret); } /* @@ -494,9 +494,9 @@ __wt_curindex_open(WT_SESSION_IMPL *session, } WT_RET(__wt_calloc_one(session, &cindex)); - cursor = &cindex->iface; + cursor = (WT_CURSOR *)cindex; *cursor = iface; - cursor->session = &session->iface; + cursor->session = (WT_SESSION *)session; cindex->table = table; cindex->index = idx; diff --git a/src/third_party/wiredtiger/src/cursor/cur_join.c b/src/third_party/wiredtiger/src/cursor/cur_join.c index 8d0abbeccbf..1a23f4a51fc 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_join.c +++ b/src/third_party/wiredtiger/src/cursor/cur_join.c @@ -324,8 +324,8 @@ __curjoin_close(WT_CURSOR *cursor) u_int i; cjoin = (WT_CURSOR_JOIN *)cursor; - JOINABLE_CURSOR_API_CALL_PREPARE_ALLOWED(cursor, session, close, NULL); +err: WT_TRET(__wt_schema_release_table(session, cjoin->table)); @@ -362,9 +362,9 @@ __curjoin_close(WT_CURSOR *cursor) WT_TRET(cjoin->main->close(cjoin->main)); __wt_free(session, cjoin->entries); - WT_TRET(__wt_cursor_close(cursor)); + __wt_cursor_close(cursor); -err: API_END_RET(session, ret); + API_END_RET(session, ret); } /* @@ -500,7 +500,7 @@ __curjoin_entry_in_range(WT_SESSION_IMPL *session, WT_CURSOR_JOIN_ENTRY *entry, passed = (cmp < 0); break; - WT_ILLEGAL_VALUE(session); + WT_ILLEGAL_VALUE(session, WT_CURJOIN_END_RANGE(end)); } if (!passed) { @@ -651,15 +651,17 @@ __curjoin_entry_member(WT_SESSION_IMPL *session, WT_CURSOR_JOIN_ENTRY *entry, if (iter != NULL && entry == iter->entry) WT_ITEM_SET(v, iter->idxkey); else { - memset(&v, 0, sizeof(v)); /* Keep lint quiet. */ + memset(&v, 0, sizeof(v)); /* Keep lint quiet. */ c = entry->main; c->set_key(c, key); entry->stats.main_access++; if ((ret = c->search(c)) == 0) ret = c->get_value(c, &v); - else if (ret == WT_NOTFOUND) - WT_ERR_MSG(session, WT_ERROR, + else if (ret == WT_NOTFOUND) { + __wt_err(session, ret, "main table for join is missing entry"); + ret = WT_ERROR; + } WT_TRET(c->reset(c)); WT_ERR(ret); } @@ -801,7 +803,8 @@ __curjoin_init_bloom(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, goto done; WT_ERR(ret); } else - WT_ERR(__wt_illegal_value(session, NULL)); + WT_PANIC_ERR(session, EINVAL, + "fatal error in join cursor position state"); } collator = (entry->index == NULL) ? NULL : entry->index->collator; while (ret == 0) { @@ -1336,11 +1339,12 @@ __wt_curjoin_open(WT_SESSION_IMPL *session, session, tablename, size, false, 0, &table)); WT_RET(__wt_calloc_one(session, &cjoin)); - cursor = &cjoin->iface; + cursor = (WT_CURSOR *)cjoin; *cursor = iface; - cursor->session = &session->iface; + cursor->session = (WT_SESSION *)session; cursor->key_format = table->key_format; cursor->value_format = table->value_format; + cjoin->table = table; /* Handle projections. */ diff --git a/src/third_party/wiredtiger/src/cursor/cur_json.c b/src/third_party/wiredtiger/src/cursor/cur_json.c index 87f8899d9c8..d4847d5a2ee 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_json.c +++ b/src/third_party/wiredtiger/src/cursor/cur_json.c @@ -53,7 +53,7 @@ static int __json_pack_size(WT_SESSION_IMPL *, const char *, WT_CONFIG_ITEM *, (pv).type = 'K'; \ break; \ /* User format strings have already been validated. */ \ - WT_ILLEGAL_VALUE(session); \ + WT_ILLEGAL_VALUE(session, (pv).type); \ } \ } while (0) @@ -922,7 +922,7 @@ __wt_json_strncpy(WT_SESSION *wt_session, case '\\': *dst++ = ch; break; - WT_ILLEGAL_VALUE(session); + WT_ILLEGAL_VALUE(session, ch); } else *dst++ = ch; diff --git a/src/third_party/wiredtiger/src/cursor/cur_log.c b/src/third_party/wiredtiger/src/cursor/cur_log.c index 5c2fbd325f6..ca2163b2818 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_log.c +++ b/src/third_party/wiredtiger/src/cursor/cur_log.c @@ -317,10 +317,11 @@ __curlog_close(WT_CURSOR *cursor) WT_DECL_RET; WT_SESSION_IMPL *session; - CURSOR_API_CALL_PREPARE_ALLOWED(cursor, session, close, NULL); cl = (WT_CURSOR_LOG *)cursor; - conn = S2C(session); + CURSOR_API_CALL_PREPARE_ALLOWED(cursor, session, close, NULL); +err: + conn = S2C(session); if (F_ISSET(cl, WT_CURLOG_ARCHIVE_LOCK)) { (void)__wt_atomic_sub32(&conn->log_cursors, 1); __wt_readunlock(session, &conn->log->log_archive_lock); @@ -334,9 +335,9 @@ __curlog_close(WT_CURSOR *cursor) __wt_free(session, cl->packed_key); __wt_free(session, cl->packed_value); - WT_TRET(__wt_cursor_close(cursor)); + __wt_cursor_close(cursor); -err: API_END_RET(session, ret); + API_END_RET(session, ret); } /* @@ -375,22 +376,22 @@ __wt_curlog_open(WT_SESSION_IMPL *session, WT_LOG *log; WT_STATIC_ASSERT(offsetof(WT_CURSOR_LOG, iface) == 0); - conn = S2C(session); + conn = S2C(session); log = conn->log; - cl = NULL; + WT_RET(__wt_calloc_one(session, &cl)); - cursor = &cl->iface; + cursor = (WT_CURSOR *)cl; *cursor = iface; - cursor->session = &session->iface; + cursor->session = (WT_SESSION *)session; + cursor->key_format = WT_LOGC_KEY_FORMAT; + cursor->value_format = WT_LOGC_VALUE_FORMAT; + WT_ERR(__wt_calloc_one(session, &cl->cur_lsn)); WT_ERR(__wt_calloc_one(session, &cl->next_lsn)); WT_ERR(__wt_scr_alloc(session, 0, &cl->logrec)); WT_ERR(__wt_scr_alloc(session, 0, &cl->opkey)); WT_ERR(__wt_scr_alloc(session, 0, &cl->opvalue)); - cursor->key_format = WT_LOGC_KEY_FORMAT; - cursor->value_format = WT_LOGC_VALUE_FORMAT; - WT_INIT_LSN(cl->cur_lsn); WT_INIT_LSN(cl->next_lsn); diff --git a/src/third_party/wiredtiger/src/cursor/cur_metadata.c b/src/third_party/wiredtiger/src/cursor/cur_metadata.c index 63c005ceeef..031001bbf80 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_metadata.c +++ b/src/third_party/wiredtiger/src/cursor/cur_metadata.c @@ -553,16 +553,17 @@ __curmetadata_close(WT_CURSOR *cursor) mdc = (WT_CURSOR_METADATA *)cursor; c = mdc->file_cursor; - CURSOR_API_CALL_PREPARE_ALLOWED(cursor, session, close, c == NULL ? - NULL : ((WT_CURSOR_BTREE *)c)->btree); + CURSOR_API_CALL_PREPARE_ALLOWED(cursor, session, close, + c == NULL ? NULL : ((WT_CURSOR_BTREE *)c)->btree); +err: if (c != NULL) - ret = c->close(c); + WT_TRET(c->close(c)); if ((c = mdc->create_cursor) != NULL) WT_TRET(c->close(c)); - WT_TRET(__wt_cursor_close(cursor)); + __wt_cursor_close(cursor); -err: API_END_RET(session, ret); + API_END_RET(session, ret); } /* @@ -606,10 +607,9 @@ __wt_curmetadata_open(WT_SESSION_IMPL *session, WT_CONFIG_ITEM cval; WT_RET(__wt_calloc_one(session, &mdc)); - - cursor = &mdc->iface; + cursor = (WT_CURSOR *)mdc; *cursor = iface; - cursor->session = &session->iface; + cursor->session = (WT_SESSION *)session; cursor->key_format = "S"; cursor->value_format = "S"; diff --git a/src/third_party/wiredtiger/src/cursor/cur_stat.c b/src/third_party/wiredtiger/src/cursor/cur_stat.c index 9d68e2399be..25d4b588d3b 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_stat.c +++ b/src/third_party/wiredtiger/src/cursor/cur_stat.c @@ -321,6 +321,7 @@ __curstat_close(WT_CURSOR *cursor) cst = (WT_CURSOR_STAT *)cursor; CURSOR_API_CALL_PREPARE_ALLOWED(cursor, session, close, NULL); +err: if (cst->cfg != NULL) { for (i = 0; cst->cfg[i] != NULL; ++i) @@ -331,9 +332,9 @@ __curstat_close(WT_CURSOR *cursor) __wt_buf_free(session, &cst->pv); __wt_free(session, cst->desc_buf); - WT_ERR(__wt_cursor_close(cursor)); + __wt_cursor_close(cursor); -err: API_END_RET(session, ret); + API_END_RET(session, ret); } /* @@ -595,9 +596,9 @@ __wt_curstat_open(WT_SESSION_IMPL *session, conn = S2C(session); WT_RET(__wt_calloc_one(session, &cst)); - cursor = &cst->iface; + cursor = (WT_CURSOR *)cst; *cursor = iface; - cursor->session = &session->iface; + cursor->session = (WT_SESSION *)session; /* * Statistics cursor configuration: must match (and defaults to), the diff --git a/src/third_party/wiredtiger/src/cursor/cur_std.c b/src/third_party/wiredtiger/src/cursor/cur_std.c index 5e719599160..479f0d42ae1 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_std.c +++ b/src/third_party/wiredtiger/src/cursor/cur_std.c @@ -835,7 +835,7 @@ __wt_cursor_cache_get(WT_SESSION_IMPL *session, const char *uri, * __wt_cursor_close -- * WT_CURSOR->close default implementation. */ -int +void __wt_cursor_close(WT_CURSOR *cursor) { WT_SESSION_IMPL *session; @@ -854,7 +854,6 @@ __wt_cursor_close(WT_CURSOR *cursor) __wt_free(session, cursor->internal_uri); __wt_free(session, cursor->uri); __wt_overwrite_and_free(session, cursor); - return (0); } /* diff --git a/src/third_party/wiredtiger/src/cursor/cur_table.c b/src/third_party/wiredtiger/src/cursor/cur_table.c index 495209b7f9f..534c13e7831 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_table.c +++ b/src/third_party/wiredtiger/src/cursor/cur_table.c @@ -808,6 +808,7 @@ __curtable_close(WT_CURSOR *cursor) ctable = (WT_CURSOR_TABLE *)cursor; JOINABLE_CURSOR_API_CALL_PREPARE_ALLOWED(cursor, session, close, NULL); +err: if (ctable->cg_cursors != NULL) for (i = 0, cp = ctable->cg_cursors; @@ -841,9 +842,9 @@ __curtable_close(WT_CURSOR *cursor) WT_TRET(__wt_schema_release_table(session, ctable->table)); /* The URI is owned by the table. */ cursor->internal_uri = NULL; - WT_TRET(__wt_cursor_close(cursor)); + __wt_cursor_close(cursor); -err: API_END_RET(session, ret); + API_END_RET(session, ret); } /* @@ -979,8 +980,6 @@ __wt_curtable_open(WT_SESSION_IMPL *session, WT_STATIC_ASSERT(offsetof(WT_CURSOR_TABLE, iface) == 0); - ctable = NULL; - tablename = uri; WT_PREFIX_SKIP_REQUIRED(session, tablename, "table:"); columns = strchr(tablename, '('); @@ -1011,10 +1010,9 @@ __wt_curtable_open(WT_SESSION_IMPL *session, } WT_RET(__wt_calloc_one(session, &ctable)); - - cursor = &ctable->iface; + cursor = (WT_CURSOR *)ctable; *cursor = iface; - cursor->session = &session->iface; + cursor->session = (WT_SESSION *)session; cursor->internal_uri = table->iface.name; cursor->key_format = table->key_format; cursor->value_format = table->value_format; diff --git a/src/third_party/wiredtiger/src/docs/config-strings.dox b/src/third_party/wiredtiger/src/docs/config-strings.dox index a583573214f..d6291d5b4ba 100644 --- a/src/third_party/wiredtiger/src/docs/config-strings.dox +++ b/src/third_party/wiredtiger/src/docs/config-strings.dox @@ -44,7 +44,7 @@ columns in a table, values are nested using parentheses. For example: All types of parentheses are treated equivalently by the parser. -When an integer values is expected, the value may have multiplier characters +When an integer value is expected, the value may have multiplier characters appended, as follows: <table> diff --git a/src/third_party/wiredtiger/src/docs/top/main.dox b/src/third_party/wiredtiger/src/docs/top/main.dox index e4de22ff042..d802443a9d8 100644 --- a/src/third_party/wiredtiger/src/docs/top/main.dox +++ b/src/third_party/wiredtiger/src/docs/top/main.dox @@ -6,12 +6,12 @@ WiredTiger is an high performance, scalable, production quality, NoSQL, @section releases Releases <table> -@row{<b>WiredTiger 3.0.0</b> (current), +@row{<b>WiredTiger 3.1.0</b> (current), + <a href="releases/wiredtiger-3.1.0.tar.bz2"><b>[Release package]</b></a>, + <a href="3.1.0/index.html"><b>[Documentation]</b></a>} +@row{<b>WiredTiger 3.0.0</b> (previous), <a href="releases/wiredtiger-3.0.0.tar.bz2"><b>[Release package]</b></a>, <a href="3.0.0/index.html"><b>[Documentation]</b></a>} -@row{<b>WiredTiger 2.9.3</b> (previous), - <a href="releases/wiredtiger-2.9.3.tar.bz2"><b>[Release package]</b></a>, - <a href="2.9.3/index.html"><b>[Documentation]</b></a>} @row{<b>Development branch</b>, <a href="https://github.com/wiredtiger/wiredtiger"><b>[Source code]</b></a>, <a href="develop/index.html"><b>[Documentation]</b></a>} diff --git a/src/third_party/wiredtiger/src/docs/upgrading.dox b/src/third_party/wiredtiger/src/docs/upgrading.dox index 2e4990e8a33..7e89d23230f 100644 --- a/src/third_party/wiredtiger/src/docs/upgrading.dox +++ b/src/third_party/wiredtiger/src/docs/upgrading.dox @@ -1,5 +1,61 @@ /*! @page upgrading Upgrading WiredTiger applications +@section version_310 Upgrading to Version 3.1.0 +<dl> + +<dt>WiredTiger on-disk log file format change</dt> +<dd> +The WiredTiger on-disk file format for write-ahead log files has changed +as the log file version number was incremented. See +<a href=https://jira.mongodb.org/browse/WT-4029>WT-4029</a> for details. +</dd> + +<dt>::wiredtiger_open compatibility configuration changes</dt> +<dd> +The compatibility setting now takes additional options that can define +the minimum or maximum required version of existing data files. See +<a href=https://jira.mongodb.org/browse/WT-4056>WT-4056</a> and +<a href=https://jira.mongodb.org/browse/WT-4098>WT-4098</a> for details. +</dd> +<dt>::wiredtiger_open cache configuration changes</dt> +<dd> +The cache configuration options \c eviction_checkpoint_target, \c +eviction_dirty_target, \c eviction_dirty_trigger, \c eviction_target and \c +eviction_trigger have changed. The options can now take an absolute size. It would +be a percentage of the cache size if the value is within the range of 0 to 100 +or an absolute size when greater than 100. This API change is compatible with +existing usage. See <a href=https://jira.mongodb.org/browse/WT-3632>WT-3632</a> +for details. +</dd> + +<dt>Changed transaction semantics around schema operations</dt> +<dd> +WiredTiger does not offer fully transactional create and drop operations. +We have made some changes to how create and drop are implemented +if done within the scope of an explicit transaction. If an application +is relying on particular visibility/atomicity guarantees around table +create or drop, care should be taken when upgrading. See +<a href=https://jira.mongodb.org/browse/WT-3964>WT-3964</a> for details. +</dd> + +<dt>On-disk format change for metadata</dt> +<dd> +There was a change to the content stored in the WiredTiger owned metadata +files, which means metadata created or updated by this version of WiredTiger +is not compatible with earlier versions. See +<a href=https://jira.mongodb.org/browse/WT-3905>WT-3905</a> for details. +</dd> + +<dt>Implement a per-session cursor cache</dt> +<dd> +WiredTiger now holds a cache of recently closed cursors in each +session. This improves performance for applications that open and +close cursors frequently, but increases memory overhead. The cache +is enabled by default, but can be disabled. See +<a href=https://jira.mongodb.org/browse/WT-1228>WT-1228</a> for details. +</dd> + +</dl><hr> @section version_300 Upgrading to Version 3.0.0 <dl> @@ -28,16 +84,6 @@ The performance visualization tool \c wtstats has been removed and is no longer supported. </dd> -<dt>::wiredtiger_open cache configuration changes</dt> -<dd> -The cache configuration options \c eviction_checkpoint_target, \c -eviction_dirty_target, \c eviction_dirty_trigger, \c eviction_target and \c -eviction_trigger have changed. The options can now take absolute size. It would -be a percentage of the cache size if the value is within the range of 0 to 100 -or an absolute size when greater than 100. This API change is compatible with -existing usage. -</dd> - </dl><hr> @section version_292 Upgrading to Version 2.9.2 <dl> diff --git a/src/third_party/wiredtiger/src/evict/evict_file.c b/src/third_party/wiredtiger/src/evict/evict_file.c index b6c6305d190..a214c58afc2 100644 --- a/src/third_party/wiredtiger/src/evict/evict_file.c +++ b/src/third_party/wiredtiger/src/evict/evict_file.c @@ -126,7 +126,7 @@ __wt_evict_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) break; case WT_SYNC_CHECKPOINT: case WT_SYNC_WRITE_LEAVES: - WT_ERR(__wt_illegal_value(session, NULL)); + WT_ERR(__wt_illegal_value(session, syncop)); break; } } diff --git a/src/third_party/wiredtiger/src/evict/evict_lru.c b/src/third_party/wiredtiger/src/evict/evict_lru.c index 05397843fc7..13b5dfc4c8d 100644 --- a/src/third_party/wiredtiger/src/evict/evict_lru.c +++ b/src/third_party/wiredtiger/src/evict/evict_lru.c @@ -451,7 +451,7 @@ __evict_server(WT_SESSION_IMPL *session, bool *did_work) "Cache stuck for too long, giving up"); WT_RET(__wt_verbose_dump_txn(session)); WT_RET(__wt_verbose_dump_cache(session)); - return (ETIMEDOUT); + return (__wt_set_return(session, ETIMEDOUT)); #else if (WT_VERBOSE_ISSET(session, WT_VERB_EVICT_STUCK)) { WT_RET(__wt_verbose_dump_txn(session)); diff --git a/src/third_party/wiredtiger/src/evict/evict_page.c b/src/third_party/wiredtiger/src/evict/evict_page.c index 0daccdf5b1c..8379c27617f 100644 --- a/src/third_party/wiredtiger/src/evict/evict_page.c +++ b/src/third_party/wiredtiger/src/evict/evict_page.c @@ -42,7 +42,7 @@ __evict_exclusive(WT_SESSION_IMPL *session, WT_REF *ref) WT_STAT_DATA_INCR(session, cache_eviction_hazard); WT_STAT_CONN_INCR(session, cache_eviction_hazard); - return (EBUSY); + return (__wt_set_return(session, EBUSY)); } /* @@ -299,6 +299,18 @@ __evict_page_clean_update(WT_SESSION_IMPL *session, WT_REF *ref, bool closing) WT_DECL_RET; /* + * Before discarding a page, assert that all updates are globally + * visible unless the tree is closing, dead, or we're evicting with + * history in lookaside. + */ + WT_ASSERT(session, + closing || ref->page->modify == NULL || + F_ISSET(session->dhandle, WT_DHANDLE_DEAD) || + (ref->page_las != NULL && ref->page_las->eviction_to_lookaside) || + __wt_txn_visible_all(session, ref->page->modify->rec_max_txn, + WT_TIMESTAMP_NULL(&ref->page->modify->rec_max_timestamp))); + + /* * Discard the page and update the reference structure. If evicting a * WT_REF_LIMBO page with active history, transition back to * WT_REF_LOOKASIDE. Otherwise, a page with a disk address is an @@ -424,7 +436,7 @@ __evict_page_dirty_update(WT_SESSION_IMPL *session, WT_REF *ref, bool closing) } break; - WT_ILLEGAL_VALUE(session); + WT_ILLEGAL_VALUE(session, mod->rec_result); } return (0); @@ -453,7 +465,7 @@ __evict_child_check(WT_SESSION_IMPL *session, WT_REF *parent) * page. */ if (__wt_page_del_active(session, child, true)) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); break; case WT_REF_LOOKASIDE: /* @@ -461,10 +473,10 @@ __evict_child_check(WT_SESSION_IMPL *session, WT_REF *parent) * can be ignored. */ if (__wt_page_las_active(session, child)) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); break; default: - return (EBUSY); + return (__wt_set_return(session, EBUSY)); } } WT_INTL_FOREACH_END; @@ -528,7 +540,7 @@ __evict_review( * should be uncommon - we don't add clean pages to the queue. */ if (F_ISSET(conn, WT_CONN_IN_MEMORY) && !modified && !closing) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); /* Check if the page can be evicted. */ if (!closing) { @@ -541,7 +553,7 @@ __evict_review( session, WT_TXN_OLDEST_STRICT)); if (!__wt_page_can_evict(session, ref, inmem_splitp)) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); /* * Check for an append-only workload needing an in-memory @@ -563,7 +575,7 @@ __evict_review( * eviction that writes to lookaside), give up. */ if (F_ISSET(session, WT_SESSION_NO_RECONCILE)) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); /* * If the page is dirty, reconcile it to decide if we can evict it. @@ -636,7 +648,7 @@ __evict_review( if (WT_SESSION_IS_CHECKPOINT(session) && !__wt_page_is_modified(page) && !__wt_txn_visible_all(session, page->modify->rec_max_txn, WT_TIMESTAMP_NULL(&page->modify->rec_max_timestamp))) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); /* * If reconciliation fails but reports it might succeed if we use the @@ -663,21 +675,14 @@ __evict_review( */ if (WT_SESSION_IS_CHECKPOINT(session) && page->modify->rec_result == WT_PM_REC_MULTIBLOCK) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); /* - * Success: assert the page is clean or reconciliation was configured - * for update/restore. If the page is clean, assert that reconciliation - * was configured for a lookaside table, or it's not a durable object - * (currently the lookaside table), or all page updates were globally - * visible. + * Success: assert that the page is clean or reconciliation was + * configured to save updates. */ WT_ASSERT(session, !__wt_page_is_modified(page) || LF_ISSET(WT_REC_LOOKASIDE | WT_REC_UPDATE_RESTORE)); - WT_ASSERT(session, - __wt_page_is_modified(page) || - __wt_txn_visible_all(session, page->modify->rec_max_txn, - WT_TIMESTAMP_NULL(&page->modify->rec_max_timestamp))); return (0); } diff --git a/src/third_party/wiredtiger/src/include/api.h b/src/third_party/wiredtiger/src/include/api.h index aabb19c86aa..1c22c99a11c 100644 --- a/src/third_party/wiredtiger/src/include/api.h +++ b/src/third_party/wiredtiger/src/include/api.h @@ -48,7 +48,7 @@ WT_TRACK_OP_INIT(s); \ WT_SINGLE_THREAD_CHECK_START(s); \ WT_ERR(WT_SESSION_CHECK_PANIC(s)); \ - /* Reset wait time if this isn't an API re entry. */ \ + /* Reset wait time if this isn't an API reentry. */ \ if (__oldname == NULL) \ (s)->cache_wait_us = 0; \ __wt_verbose((s), WT_VERB_API, "%s", "CALL: " #h ":" #n) diff --git a/src/third_party/wiredtiger/src/include/btree.h b/src/third_party/wiredtiger/src/include/btree.h index 96f6309aba4..2a5be782b06 100644 --- a/src/third_party/wiredtiger/src/include/btree.h +++ b/src/third_party/wiredtiger/src/include/btree.h @@ -95,6 +95,7 @@ struct __wt_btree { uint32_t maxleafkey; /* Leaf page max key size */ uint32_t maxleafvalue; /* Leaf page max value size */ uint64_t maxmempage; /* In-memory page max size */ + uint32_t maxmempage_image; /* In-memory page image max size */ uint64_t splitmempage; /* In-memory split trigger size */ /* AUTOMATIC FLAG VALUE GENERATION START */ @@ -129,6 +130,16 @@ struct __wt_btree { int split_pct; /* Split page percent */ WT_COMPRESSOR *compressor; /* Page compressor */ + /* + * When doing compression, the pre-compression in-memory byte size is + * optionally adjusted based on previous compression results. + * It's an 8B value because it's updated without a lock. + */ + bool leafpage_compadjust; /* Run-time compression adjustment */ + uint64_t maxleafpage_precomp; /* Leaf page pre-compression size */ + bool intlpage_compadjust; /* Run-time compression adjustment */ + uint64_t maxintlpage_precomp; /* Internal page pre-compression size */ + WT_KEYED_ENCRYPTOR *kencryptor; /* Page encryptor */ WT_RWLOCK ovfl_lock; /* Overflow lock */ diff --git a/src/third_party/wiredtiger/src/include/btree.i b/src/third_party/wiredtiger/src/include/btree.i index 7813f1299fd..ed3480a40d0 100644 --- a/src/third_party/wiredtiger/src/include/btree.i +++ b/src/third_party/wiredtiger/src/include/btree.i @@ -1357,13 +1357,12 @@ __wt_page_evict_retry(WT_SESSION_IMPL *session, WT_PAGE *page) return (true); #ifdef HAVE_TIMESTAMPS - if (!__wt_timestamp_iszero(&mod->last_eviction_timestamp)) { - __wt_txn_pinned_timestamp(session, &pinned_ts); - if (__wt_timestamp_cmp( - &mod->last_eviction_timestamp, - &txn_global->pinned_timestamp) != 0) - return (true); - } + if (__wt_timestamp_iszero(&mod->last_eviction_timestamp)) + return (true); + + __wt_txn_pinned_timestamp(session, &pinned_ts); + if (__wt_timestamp_cmp(&pinned_ts, &mod->last_eviction_timestamp) > 0) + return (true); #endif return (false); diff --git a/src/third_party/wiredtiger/src/include/cache.i b/src/third_party/wiredtiger/src/include/cache.i index 7f12949e162..2e3700f6287 100644 --- a/src/third_party/wiredtiger/src/include/cache.i +++ b/src/third_party/wiredtiger/src/include/cache.i @@ -220,9 +220,9 @@ __wt_cache_update_lookaside_score( global_score = cache->evict_lookaside_score; if (score > global_score && global_score < 100) - __wt_atomic_addi32(&cache->evict_lookaside_score, 1); + (void)__wt_atomic_addi32(&cache->evict_lookaside_score, 1); else if (score < global_score && global_score > 0) - __wt_atomic_subi32(&cache->evict_lookaside_score, 1); + (void)__wt_atomic_subi32(&cache->evict_lookaside_score, 1); } /* diff --git a/src/third_party/wiredtiger/src/include/cell.i b/src/third_party/wiredtiger/src/include/cell.i index c160f84b870..fbdbe241165 100644 --- a/src/third_party/wiredtiger/src/include/cell.i +++ b/src/third_party/wiredtiger/src/include/cell.i @@ -778,7 +778,7 @@ __cell_data_ref(WT_SESSION_IMPL *session, return (0); huffman = btree->huffman_value; break; - WT_ILLEGAL_VALUE(session); + WT_ILLEGAL_VALUE(session, unpack->type); } return (huffman == NULL || store->size == 0 ? 0 : diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h index 22459b0072c..19bb21ce2f1 100644 --- a/src/third_party/wiredtiger/src/include/connection.h +++ b/src/third_party/wiredtiger/src/include/connection.h @@ -322,7 +322,8 @@ struct __wt_connection_impl { #define WT_CONN_LOG_RECOVER_DIRTY 0x020u /* Recovering unclean */ #define WT_CONN_LOG_RECOVER_DONE 0x040u /* Recovery completed */ #define WT_CONN_LOG_RECOVER_ERR 0x080u /* Error if recovery required */ -#define WT_CONN_LOG_ZERO_FILL 0x100u /* Manually zero files */ +#define WT_CONN_LOG_RECOVER_SALVAGE 0x100u /* Salvage log files */ +#define WT_CONN_LOG_ZERO_FILL 0x200u /* Manually zero files */ /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t log_flags; /* Global logging configuration */ WT_CONDVAR *log_cond; /* Log server wait mutex */ @@ -413,34 +414,35 @@ struct __wt_connection_impl { #define WT_VERB_CHECKPOINT 0x000000004u #define WT_VERB_CHECKPOINT_PROGRESS 0x000000008u #define WT_VERB_COMPACT 0x000000010u -#define WT_VERB_EVICT 0x000000020u -#define WT_VERB_EVICTSERVER 0x000000040u -#define WT_VERB_EVICT_STUCK 0x000000080u -#define WT_VERB_FILEOPS 0x000000100u -#define WT_VERB_HANDLEOPS 0x000000200u -#define WT_VERB_LOG 0x000000400u -#define WT_VERB_LOOKASIDE 0x000000800u -#define WT_VERB_LOOKASIDE_ACTIVITY 0x000001000u -#define WT_VERB_LSM 0x000002000u -#define WT_VERB_LSM_MANAGER 0x000004000u -#define WT_VERB_METADATA 0x000008000u -#define WT_VERB_MUTEX 0x000010000u -#define WT_VERB_OVERFLOW 0x000020000u -#define WT_VERB_READ 0x000040000u -#define WT_VERB_REBALANCE 0x000080000u -#define WT_VERB_RECONCILE 0x000100000u -#define WT_VERB_RECOVERY 0x000200000u -#define WT_VERB_RECOVERY_PROGRESS 0x000400000u -#define WT_VERB_SALVAGE 0x000800000u -#define WT_VERB_SHARED_CACHE 0x001000000u -#define WT_VERB_SPLIT 0x002000000u -#define WT_VERB_TEMPORARY 0x004000000u -#define WT_VERB_THREAD_GROUP 0x008000000u -#define WT_VERB_TIMESTAMP 0x010000000u -#define WT_VERB_TRANSACTION 0x020000000u -#define WT_VERB_VERIFY 0x040000000u -#define WT_VERB_VERSION 0x080000000u -#define WT_VERB_WRITE 0x100000000u +#define WT_VERB_ERROR_RETURNS 0x000000020u +#define WT_VERB_EVICT 0x000000040u +#define WT_VERB_EVICTSERVER 0x000000080u +#define WT_VERB_EVICT_STUCK 0x000000100u +#define WT_VERB_FILEOPS 0x000000200u +#define WT_VERB_HANDLEOPS 0x000000400u +#define WT_VERB_LOG 0x000000800u +#define WT_VERB_LOOKASIDE 0x000001000u +#define WT_VERB_LOOKASIDE_ACTIVITY 0x000002000u +#define WT_VERB_LSM 0x000004000u +#define WT_VERB_LSM_MANAGER 0x000008000u +#define WT_VERB_METADATA 0x000010000u +#define WT_VERB_MUTEX 0x000020000u +#define WT_VERB_OVERFLOW 0x000040000u +#define WT_VERB_READ 0x000080000u +#define WT_VERB_REBALANCE 0x000100000u +#define WT_VERB_RECONCILE 0x000200000u +#define WT_VERB_RECOVERY 0x000400000u +#define WT_VERB_RECOVERY_PROGRESS 0x000800000u +#define WT_VERB_SALVAGE 0x001000000u +#define WT_VERB_SHARED_CACHE 0x002000000u +#define WT_VERB_SPLIT 0x004000000u +#define WT_VERB_TEMPORARY 0x008000000u +#define WT_VERB_THREAD_GROUP 0x010000000u +#define WT_VERB_TIMESTAMP 0x020000000u +#define WT_VERB_TRANSACTION 0x040000000u +#define WT_VERB_VERIFY 0x080000000u +#define WT_VERB_VERSION 0x100000000u +#define WT_VERB_WRITE 0x200000000u /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint64_t verbose; @@ -459,7 +461,6 @@ struct __wt_connection_impl { #define WT_TIMING_STRESS_SPLIT_6 0x080u #define WT_TIMING_STRESS_SPLIT_7 0x100u #define WT_TIMING_STRESS_SPLIT_8 0x200u -#define WT_TIMING_STRESS_SPLIT_9 0x400u /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint64_t timing_stress_flags; diff --git a/src/third_party/wiredtiger/src/include/error.h b/src/third_party/wiredtiger/src/include/error.h index f94c3d7b880..6f28dfae2c8 100644 --- a/src/third_party/wiredtiger/src/include/error.h +++ b/src/third_party/wiredtiger/src/include/error.h @@ -19,6 +19,13 @@ #define WT_DIAGNOSTIC_YIELD #endif +#define __wt_err(session, error, ...) \ + __wt_err_func(session, error, __func__, __LINE__, __VA_ARGS__) +#define __wt_errx(session, ...) \ + __wt_errx_func(session, __func__, __LINE__, __VA_ARGS__) +#define __wt_set_return(session, error) \ + __wt_set_return_func(session, __func__, __LINE__, error) + /* Set "ret" and branch-to-err-label tests. */ #define WT_ERR(a) do { \ if ((ret = (a)) != 0) \ @@ -89,18 +96,18 @@ #define WT_TRET_BUSY_OK(a) WT_TRET_ERROR_OK(a, EBUSY) #define WT_TRET_NOTFOUND_OK(a) WT_TRET_ERROR_OK(a, WT_NOTFOUND) +/* Called on unexpected code path: locate the failure. */ +#define __wt_illegal_value(session, v) \ + __wt_illegal_value_func(session, (uintmax_t)v, __func__, __LINE__) + /* Return and branch-to-err-label cases for switch statements. */ -#define WT_ILLEGAL_VALUE(session) \ +#define WT_ILLEGAL_VALUE(session, v) \ default: \ - return (__wt_illegal_value(session, NULL)) -#define WT_ILLEGAL_VALUE_ERR(session) \ + return (__wt_illegal_value(session, v)) +#define WT_ILLEGAL_VALUE_ERR(session, v) \ default: \ - ret = __wt_illegal_value(session, NULL); \ + ret = __wt_illegal_value(session, v); \ goto err -#define WT_ILLEGAL_VALUE_SET(session) \ - default: \ - ret = __wt_illegal_value(session, NULL); \ - break #define WT_PANIC_MSG(session, v, ...) do { \ __wt_err(session, v, __VA_ARGS__); \ @@ -125,8 +132,10 @@ */ #ifdef HAVE_DIAGNOSTIC #define WT_ASSERT(session, exp) do { \ - if (!(exp)) \ - __wt_assert(session, 0, __func__, __LINE__, "%s", #exp);\ + if (!(exp)) { \ + __wt_errx(session, "%s", #exp); \ + __wt_abort(session); \ + } \ } while (0) #else #define WT_ASSERT(session, exp) \ diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index 24e16adefd1..34cee52df99 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -146,7 +146,7 @@ extern bool __wt_btree_immediately_durable(WT_SESSION_IMPL *session) WT_GCC_FUNC extern int __wt_btree_huffman_open(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_btree_huffman_close(WT_SESSION_IMPL *session); extern int __wt_bt_read(WT_SESSION_IMPL *session, WT_ITEM *buf, const uint8_t *addr, size_t addr_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_bt_write(WT_SESSION_IMPL *session, WT_ITEM *buf, uint8_t *addr, size_t *addr_sizep, bool checkpoint, bool checkpoint_io, bool compressed) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_bt_write(WT_SESSION_IMPL *session, WT_ITEM *buf, uint8_t *addr, size_t *addr_sizep, size_t *compressed_sizep, bool checkpoint, bool checkpoint_io, bool compressed) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern const char *__wt_page_type_string(u_int type) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern const char *__wt_cell_type_string(uint8_t type); extern const char *__wt_page_addr_string(WT_SESSION_IMPL *session, WT_REF *ref, WT_ITEM *buf); @@ -215,8 +215,7 @@ extern int __wt_las_cursor_position(WT_CURSOR *cursor, uint64_t pageid) WT_GCC_F extern int __wt_las_remove_block(WT_SESSION_IMPL *session, uint64_t pageid, bool lock_wait) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_las_save_dropped(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_las_sweep(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern uint32_t __wt_checksum_sw(const void *chunk, size_t len) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); -extern void __wt_checksum_init(void) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)); +extern uint32_t __wt_checksum_sw(const void *chunk, size_t len); extern void __wt_config_initn(WT_SESSION_IMPL *session, WT_CONFIG *conf, const char *str, size_t len); extern void __wt_config_init(WT_SESSION_IMPL *session, WT_CONFIG *conf, const char *str); extern void __wt_config_subinit(WT_SESSION_IMPL *session, WT_CONFIG *conf, WT_CONFIG_ITEM *item); @@ -355,7 +354,7 @@ extern int __wt_cursor_cache(WT_CURSOR *cursor, WT_DATA_HANDLE *dhandle) WT_GCC_ extern void __wt_cursor_reopen(WT_CURSOR *cursor, WT_DATA_HANDLE *dhandle); extern int __wt_cursor_cache_release(WT_SESSION_IMPL *session, WT_CURSOR *cursor, bool *released) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_cursor_cache_get(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *to_dup, const char *cfg[], WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_cursor_close(WT_CURSOR *cursor) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_cursor_close(WT_CURSOR *cursor); extern int __wt_cursor_equals(WT_CURSOR *cursor, WT_CURSOR *other, int *equalp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_cursor_reconfigure(WT_CURSOR *cursor, const char *config) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_cursor_dup_position(WT_CURSOR *to_dup, WT_CURSOR *cursor) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -549,7 +548,7 @@ extern int __wt_nfilename(WT_SESSION_IMPL *session, const char *name, size_t nam extern int __wt_filename_construct(WT_SESSION_IMPL *session, const char *path, const char *file_prefix, uintmax_t id_1, uint32_t id_2, WT_ITEM *buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_remove_if_exists(WT_SESSION_IMPL *session, const char *name, bool durable) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_copy_and_sync(WT_SESSION *wt_session, const char *from, const char *to) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern void __wt_abort(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); +extern void __wt_abort(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern int __wt_calloc(WT_SESSION_IMPL *session, size_t number, size_t size, void *retp) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_malloc(WT_SESSION_IMPL *session, size_t bytes_to_allocate, void *retp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_realloc(WT_SESSION_IMPL *session, size_t *bytes_allocated_ret, size_t bytes_to_allocate, void *retp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -599,7 +598,7 @@ extern void __wt_ovfl_reuse_free(WT_SESSION_IMPL *session, WT_PAGE *page); extern int __wt_ovfl_track_wrapup(WT_SESSION_IMPL *session, WT_PAGE *page) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_ovfl_track_wrapup_err(WT_SESSION_IMPL *session, WT_PAGE *page) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_reconcile(WT_SESSION_IMPL *session, WT_REF *ref, WT_SALVAGE_COOKIE *salvage, uint32_t flags, bool *lookaside_retryp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern uint32_t __wt_split_page_size(WT_BTREE *btree, uint32_t maxpagesize); +extern uint32_t __wt_split_page_size(int split_pct, uint32_t maxpagesize, uint32_t allocsize); extern int __wt_bulk_init(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_bulk_wrapup(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_bulk_insert_row(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -676,25 +675,17 @@ extern int __wt_decrypt(WT_SESSION_IMPL *session, WT_ENCRYPTOR *encryptor, size_ extern int __wt_encrypt(WT_SESSION_IMPL *session, WT_KEYED_ENCRYPTOR *kencryptor, size_t skip, WT_ITEM *in, WT_ITEM *out) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_encrypt_size(WT_SESSION_IMPL *session, WT_KEYED_ENCRYPTOR *kencryptor, size_t incoming_size, size_t *sizep); extern void __wt_event_handler_set(WT_SESSION_IMPL *session, WT_EVENT_HANDLER *handler); -extern void __wt_err(WT_SESSION_IMPL *session, int error, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4))) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); -extern void __wt_errx(WT_SESSION_IMPL *session, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 2, 3))); +extern void __wt_err_func(WT_SESSION_IMPL *session, int error, const char *func_name, int line_number, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 5, 6))) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); +extern void __wt_errx_func(WT_SESSION_IMPL *session, const char *func_name, int line_number, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 4, 5))) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); +extern int __wt_set_return_func(WT_SESSION_IMPL *session, const char*func, int line, int err) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_ext_err_printf(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_verbose_worker(WT_SESSION_IMPL *session, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 2, 3))) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)); extern int __wt_msg(WT_SESSION_IMPL *session, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 2, 3))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_ext_msg_printf(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern const char *__wt_ext_strerror(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, int error); extern int __wt_progress(WT_SESSION_IMPL *session, const char *s, uint64_t v) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern void -__wt_assert(WT_SESSION_IMPL *session, - int error, const char *file_name, int line_number, const char *fmt, ...) - WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) - WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 5, 6))) -#ifdef HAVE_DIAGNOSTIC - WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)) -#endif - WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern int __wt_panic(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_illegal_value_func(WT_SESSION_IMPL *session, const char *tag, const char *file, int line) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_illegal_value_func(WT_SESSION_IMPL *session, uintmax_t v, const char *func, int line) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_inmem_unsupported_op(WT_SESSION_IMPL *session, const char *tag) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_object_unsupported(WT_SESSION_IMPL *session, const char *uri) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_bad_object_type(WT_SESSION_IMPL *session, const char *uri) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -724,6 +715,7 @@ extern int __wt_hazard_clear(WT_SESSION_IMPL *session, WT_REF *ref) WT_GCC_FUNC_ extern void __wt_hazard_close(WT_SESSION_IMPL *session); extern WT_HAZARD *__wt_hazard_check(WT_SESSION_IMPL *session, WT_REF *ref); extern u_int __wt_hazard_count(WT_SESSION_IMPL *session, WT_REF *ref); +extern bool __wt_hazard_check_assert(WT_SESSION_IMPL *session, void *ref, bool waitfor) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_fill_hex(const uint8_t *src, size_t src_max, uint8_t *dest, size_t dest_max, size_t *lenp); extern int __wt_raw_to_hex(WT_SESSION_IMPL *session, const uint8_t *from, size_t size, WT_ITEM *to) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_raw_to_esc_hex(WT_SESSION_IMPL *session, const uint8_t *from, size_t size, WT_ITEM *to) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); diff --git a/src/third_party/wiredtiger/src/include/extern_posix.h b/src/third_party/wiredtiger/src/include/extern_posix.h index 8b92d99d4f1..94bc1e78597 100644 --- a/src/third_party/wiredtiger/src/include/extern_posix.h +++ b/src/third_party/wiredtiger/src/include/extern_posix.h @@ -27,9 +27,10 @@ extern void __wt_stream_set_no_buffer(FILE *fp) WT_GCC_FUNC_DECL_ATTRIBUTE((visi extern void __wt_sleep(uint64_t seconds, uint64_t micro_seconds) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern int __wt_vsnprintf_len_incr(char *buf, size_t size, size_t *retsizep, const char *fmt, va_list ap) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_thread_create(WT_SESSION_IMPL *session, wt_thread_t *tidret, WT_THREAD_CALLBACK(*func)(void *), void *arg) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_thread_join(WT_SESSION_IMPL *session, wt_thread_t tid) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_thread_join(WT_SESSION_IMPL *session, wt_thread_t *tid) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_thread_id(uintmax_t *id) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern int __wt_thread_str(char *buf, size_t buflen) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern uintmax_t __wt_process_id(void); extern void __wt_epoch_raw(WT_SESSION_IMPL *session, struct timespec *tsp); +extern int __wt_localtime(WT_SESSION_IMPL *session, const time_t *timep, struct tm *result) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_yield(void) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); diff --git a/src/third_party/wiredtiger/src/include/extern_win.h b/src/third_party/wiredtiger/src/include/extern_win.h index 50808750c56..9356b844be0 100644 --- a/src/third_party/wiredtiger/src/include/extern_win.h +++ b/src/third_party/wiredtiger/src/include/extern_win.h @@ -25,11 +25,12 @@ extern void __wt_stream_set_no_buffer(FILE *fp); extern void __wt_sleep(uint64_t seconds, uint64_t micro_seconds); extern int __wt_vsnprintf_len_incr(char *buf, size_t size, size_t *retsizep, const char *fmt, va_list ap) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_thread_create(WT_SESSION_IMPL *session, wt_thread_t *tidret, WT_THREAD_CALLBACK(*func)(void *), void *arg) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_thread_join(WT_SESSION_IMPL *session, wt_thread_t tid) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_thread_join(WT_SESSION_IMPL *session, wt_thread_t *tid) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_thread_id(uintmax_t *id); extern int __wt_thread_str(char *buf, size_t buflen) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern uintmax_t __wt_process_id(void); extern void __wt_epoch_raw(WT_SESSION_IMPL *session, struct timespec *tsp); +extern int __wt_localtime(WT_SESSION_IMPL *session, const time_t *timep, struct tm *result) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_to_utf16_string(WT_SESSION_IMPL *session, const char *utf8, WT_ITEM **outbuf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_to_utf8_string(WT_SESSION_IMPL *session, const wchar_t *wide, WT_ITEM **outbuf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern DWORD __wt_getlasterror(void); diff --git a/src/third_party/wiredtiger/src/include/log.h b/src/third_party/wiredtiger/src/include/log.h index 0605c458673..2613808e74e 100644 --- a/src/third_party/wiredtiger/src/include/log.h +++ b/src/third_party/wiredtiger/src/include/log.h @@ -317,9 +317,15 @@ struct __wt_log_record { /* * No automatic generation: flag values cannot change, they're written * to disk. + * + * Unused bits in the flags, as well as the 'unused' padding, + * are expected to be zeroed; we check that to help detect file + * corruption. */ #define WT_LOG_RECORD_COMPRESSED 0x01u /* Compressed except hdr */ #define WT_LOG_RECORD_ENCRYPTED 0x02u /* Encrypted except hdr */ +#define WT_LOG_RECORD_ALL_FLAGS \ + (WT_LOG_RECORD_COMPRESSED | WT_LOG_RECORD_ENCRYPTED) uint16_t flags; /* 08-09: Flags */ uint8_t unused[2]; /* 10-11: Padding */ uint32_t mem_len; /* 12-15: Uncompressed len if needed */ diff --git a/src/third_party/wiredtiger/src/include/misc.h b/src/third_party/wiredtiger/src/include/misc.h index 7060e6cea23..1180c641222 100644 --- a/src/third_party/wiredtiger/src/include/misc.h +++ b/src/third_party/wiredtiger/src/include/misc.h @@ -308,10 +308,6 @@ typedef void wt_timestamp_t; __wt_page_swap_func(session, held, want, flags) #endif -/* Called on unexpected code path: locate the failure. */ -#define __wt_illegal_value(session, msg) \ - __wt_illegal_value_func(session, msg, __func__, __LINE__) - /* Random number generator state. */ union __wt_rand_state { uint64_t v; @@ -338,3 +334,50 @@ union __wt_rand_state { continue; \ } #define WT_TAILQ_SAFE_REMOVE_END } + +/* + * WT_VA_ARGS_BUF_FORMAT -- + * Format into a scratch buffer, extending it as necessary. This is a + * macro because we need to repeatedly call va_start/va_end and there's no + * way to do that inside a function call. + */ +#define WT_VA_ARGS_BUF_FORMAT(session, buf, fmt, concatenate) do { \ + size_t __len, __space; \ + va_list __ap; \ + int __ret_xx; /* __ret already used by WT_RET */ \ + char *__p; \ + \ + /* \ + * This macro is used to both initialize and concatenate into a \ + * buffer. If not concatenating, clear the size so we don't use \ + * any existing contents. \ + */ \ + if (!concatenate) \ + (buf)->size = 0; \ + for (;;) { \ + WT_ASSERT(session, (buf)->memsize >= (buf)->size); \ + __p = (char *)((uint8_t *)(buf)->mem + (buf)->size); \ + __space = (buf)->memsize - (buf)->size; \ + \ + /* Format into the buffer. */ \ + va_start(__ap, fmt); \ + __ret_xx = __wt_vsnprintf_len_set( \ + __p, __space, &__len, fmt, __ap); \ + va_end(__ap); \ + WT_RET(__ret_xx); \ + \ + /* Check if there was enough space. */ \ + if (__len < __space) { \ + (buf)->data = (buf)->mem; \ + (buf)->size += __len; \ + break; \ + } \ + \ + /* \ + * If not, double the size of the buffer: we're dealing \ + * with strings, we don't expect the size to get huge. \ + */ \ + WT_RET(__wt_buf_extend( \ + session, buf, (buf)->size + __len + 1)); \ + } \ +} while (0) diff --git a/src/third_party/wiredtiger/src/include/misc.i b/src/third_party/wiredtiger/src/include/misc.i index 2c380e95ade..5c9f95bc08a 100644 --- a/src/third_party/wiredtiger/src/include/misc.i +++ b/src/third_party/wiredtiger/src/include/misc.i @@ -259,15 +259,26 @@ __wt_spin_backoff(uint64_t *yield_count, uint64_t *sleep_usecs) static inline void __wt_timing_stress(WT_SESSION_IMPL *session, u_int flag) { - WT_CONNECTION_IMPL *conn; - uint64_t sleep_usecs; + uint64_t i; - conn = S2C(session); - - /* Only sleep when the specified configuration flag is set. */ - if (!FLD_ISSET(conn->timing_stress_flags, flag)) + /* Optionally only sleep when a specified configuration flag is set. */ + if (flag != 0 && !FLD_ISSET(S2C(session)->timing_stress_flags, flag)) return; - sleep_usecs = __wt_random(&session->rnd) % WT_TIMING_STRESS_MAX_DELAY; - __wt_sleep(0, sleep_usecs); + /* + * We need a fast way to choose a sleep time. We want to sleep a short + * period most of the time, but occasionally wait longer. Divide the + * maximum period of time into 10 buckets (where bucket 0 doesn't sleep + * at all), and roll dice, advancing to the next bucket 50% of the time. + * That means we'll hit the maximum roughly every 1K calls. + */ + for (i = 0;;) + if (__wt_random(&session->rnd) & 0x1 || ++i > 9) + break; + + if (i == 0) + __wt_yield(); + else + /* The default maximum delay is 1/10th of a second. */ + __wt_sleep(0, i * (WT_TIMING_STRESS_MAX_DELAY / 10)); } diff --git a/src/third_party/wiredtiger/src/include/os_fhandle.i b/src/third_party/wiredtiger/src/include/os_fhandle.i index 7c09a83132c..78d01abca4b 100644 --- a/src/third_party/wiredtiger/src/include/os_fhandle.i +++ b/src/third_party/wiredtiger/src/include/os_fhandle.i @@ -72,7 +72,7 @@ __wt_fextend(WT_SESSION_IMPL *session, WT_FH *fh, wt_off_t offset) if (handle->fh_extend != NULL) return (handle->fh_extend( handle, (WT_SESSION *)session, offset)); - return (ENOTSUP); + return (__wt_set_return(session, ENOTSUP)); } /* @@ -157,7 +157,7 @@ __wt_ftruncate(WT_SESSION_IMPL *session, WT_FH *fh, wt_off_t offset) if (handle->fh_truncate != NULL) return (handle->fh_truncate( handle, (WT_SESSION *)session, offset)); - return (ENOTSUP); + return (__wt_set_return(session, ENOTSUP)); } /* diff --git a/src/third_party/wiredtiger/src/include/packing.i b/src/third_party/wiredtiger/src/include/packing.i index 34a1bb62edb..6e5ea92b54d 100644 --- a/src/third_party/wiredtiger/src/include/packing.i +++ b/src/third_party/wiredtiger/src/include/packing.i @@ -245,7 +245,7 @@ next: if (pack->cur == pack->end) (pv).u.u = va_arg(ap, uint64_t); \ break; \ /* User format strings have already been validated. */ \ - WT_ILLEGAL_VALUE(session); \ + WT_ILLEGAL_VALUE(session, (pv).type); \ } \ } while (0) @@ -612,7 +612,7 @@ __unpack_read(WT_SESSION_IMPL *session, *va_arg(ap, uint64_t *) = (pv).u.u; \ break; \ /* User format strings have already been validated. */ \ - WT_ILLEGAL_VALUE(session); \ + WT_ILLEGAL_VALUE(session, (pv).type); \ } \ } while (0) diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index a4ba834d5ef..c80aaef53fb 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -1375,6 +1375,14 @@ struct __wt_session { * value of merge_max is used., an integer no more than 100; default \c * 0.} * @config{ ),,} + * @config{memory_page_image_max, the maximum in-memory page image + * represented by a single storage block. Depending on compression + * efficiency\, compression can create storage blocks which require + * significant resources to re-instantiate in the cache\, penalizing the + * performance of future point updates. The value limits the maximum + * in-memory page image a storage block will need. If set to 0\, a + * default of 4 times \c leaf_page_max is used., an integer greater than + * or equal to 0; default \c 0.} * @config{memory_page_max, the maximum size a page can grow to in * memory before being reconciled to disk. The specified size will be * adjusted to a lower bound of <code>leaf_page_max</code>\, and an @@ -2334,15 +2342,15 @@ struct __wt_connection { * given as a list\, such as * <code>"verbose=[evictserver\,read]"</code>., a list\, with values * chosen from the following options: \c "api"\, \c "block"\, \c - * "checkpoint"\, \c "checkpoint_progress"\, \c "compact"\, \c "evict"\, - * \c "evict_stuck"\, \c "evictserver"\, \c "fileops"\, \c "handleops"\, - * \c "log"\, \c "lookaside"\, \c "lookaside_activity"\, \c "lsm"\, \c - * "lsm_manager"\, \c "metadata"\, \c "mutex"\, \c "overflow"\, \c - * "read"\, \c "rebalance"\, \c "reconcile"\, \c "recovery"\, \c - * "recovery_progress"\, \c "salvage"\, \c "shared_cache"\, \c "split"\, - * \c "temporary"\, \c "thread_group"\, \c "timestamp"\, \c - * "transaction"\, \c "verify"\, \c "version"\, \c "write"; default - * empty.} + * "checkpoint"\, \c "checkpoint_progress"\, \c "compact"\, \c + * "error_returns"\, \c "evict"\, \c "evict_stuck"\, \c "evictserver"\, + * \c "fileops"\, \c "handleops"\, \c "log"\, \c "lookaside"\, \c + * "lookaside_activity"\, \c "lsm"\, \c "lsm_manager"\, \c "metadata"\, + * \c "mutex"\, \c "overflow"\, \c "read"\, \c "rebalance"\, \c + * "reconcile"\, \c "recovery"\, \c "recovery_progress"\, \c "salvage"\, + * \c "shared_cache"\, \c "split"\, \c "temporary"\, \c "thread_group"\, + * \c "timestamp"\, \c "transaction"\, \c "verify"\, \c "version"\, \c + * "write"; default empty.} * @configend * @errors */ @@ -2902,9 +2910,10 @@ struct __wt_connection { * flag; default \c true.} * @config{ recover, run recovery * or error if recovery needs to run after an unclean shutdown., a string\, - * chosen from the following options: \c "error"\, \c "on"; default \c on.} - * @config{ zero_fill, manually write zeroes into log - * files., a boolean flag; default \c false.} + * chosen from the following options: \c "error"\, \c "on"\, \c "salvage"; + * default \c on.} + * @config{ zero_fill, manually write + * zeroes into log files., a boolean flag; default \c false.} * @config{ ),,} * @config{lsm_manager = (, configure database wide options for LSM tree * management. The LSM manager is started automatically the first time an LSM @@ -3017,14 +3026,14 @@ struct __wt_connection { * @config{verbose, enable messages for various events. Options are given as a * list\, such as <code>"verbose=[evictserver\,read]"</code>., a list\, with * values chosen from the following options: \c "api"\, \c "block"\, \c - * "checkpoint"\, \c "checkpoint_progress"\, \c "compact"\, \c "evict"\, \c - * "evict_stuck"\, \c "evictserver"\, \c "fileops"\, \c "handleops"\, \c "log"\, - * \c "lookaside"\, \c "lookaside_activity"\, \c "lsm"\, \c "lsm_manager"\, \c - * "metadata"\, \c "mutex"\, \c "overflow"\, \c "read"\, \c "rebalance"\, \c - * "reconcile"\, \c "recovery"\, \c "recovery_progress"\, \c "salvage"\, \c - * "shared_cache"\, \c "split"\, \c "temporary"\, \c "thread_group"\, \c - * "timestamp"\, \c "transaction"\, \c "verify"\, \c "version"\, \c "write"; - * default empty.} + * "checkpoint"\, \c "checkpoint_progress"\, \c "compact"\, \c "error_returns"\, + * \c "evict"\, \c "evict_stuck"\, \c "evictserver"\, \c "fileops"\, \c + * "handleops"\, \c "log"\, \c "lookaside"\, \c "lookaside_activity"\, \c + * "lsm"\, \c "lsm_manager"\, \c "metadata"\, \c "mutex"\, \c "overflow"\, \c + * "read"\, \c "rebalance"\, \c "reconcile"\, \c "recovery"\, \c + * "recovery_progress"\, \c "salvage"\, \c "shared_cache"\, \c "split"\, \c + * "temporary"\, \c "thread_group"\, \c "timestamp"\, \c "transaction"\, \c + * "verify"\, \c "version"\, \c "write"; default empty.} * @config{write_through, Use \c FILE_FLAG_WRITE_THROUGH on Windows to write to * files. Ignored on non-Windows systems. Options are given as a list\, such * as <code>"write_through=[data]"</code>. Configuring \c write_through requires @@ -3549,18 +3558,17 @@ struct __wt_config_parser { */ /*! - * Return a buffer's CRC32C checksum. + * Return a pointer to a function that calculates a CRC32C checksum. * * The WiredTiger library CRC32C checksum function uses hardware support where * available, else it falls back to a software implementation. * * @snippet ex_all.c Checksum a buffer * - * @param buffer a pointer to a buffer - * @param len the number of valid bytes in the buffer - * @returns the buffer's CRC32C checksum + * @returns a pointer to a function that takes a buffer and length and returns + * the CRC32C checksum */ -uint32_t wiredtiger_checksum_crc32c(const void *buffer, size_t len) +uint32_t (*wiredtiger_crc32c_func(void))(const void *, size_t) WT_ATTRIBUTE_LIBRARY_VISIBLE; /*! @} */ @@ -5290,28 +5298,19 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); /*! lock: checkpoint lock internal thread wait time (usecs) */ #define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_INTERNAL 1168 /*! - * lock: commit timestamp queue lock application thread time waiting for - * the dhandle lock (usecs) + * lock: commit timestamp queue lock application thread time waiting + * (usecs) */ #define WT_STAT_CONN_LOCK_COMMIT_TIMESTAMP_WAIT_APPLICATION 1169 -/*! - * lock: commit timestamp queue lock internal thread time waiting for the - * dhandle lock (usecs) - */ +/*! lock: commit timestamp queue lock internal thread time waiting (usecs) */ #define WT_STAT_CONN_LOCK_COMMIT_TIMESTAMP_WAIT_INTERNAL 1170 /*! lock: commit timestamp queue read lock acquisitions */ #define WT_STAT_CONN_LOCK_COMMIT_TIMESTAMP_READ_COUNT 1171 /*! lock: commit timestamp queue write lock acquisitions */ #define WT_STAT_CONN_LOCK_COMMIT_TIMESTAMP_WRITE_COUNT 1172 -/*! - * lock: dhandle lock application thread time waiting for the dhandle - * lock (usecs) - */ +/*! lock: dhandle lock application thread time waiting (usecs) */ #define WT_STAT_CONN_LOCK_DHANDLE_WAIT_APPLICATION 1173 -/*! - * lock: dhandle lock internal thread time waiting for the dhandle lock - * (usecs) - */ +/*! lock: dhandle lock internal thread time waiting (usecs) */ #define WT_STAT_CONN_LOCK_DHANDLE_WAIT_INTERNAL 1174 /*! lock: dhandle read lock acquisitions */ #define WT_STAT_CONN_LOCK_DHANDLE_READ_COUNT 1175 @@ -5324,14 +5323,11 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); /*! lock: metadata lock internal thread wait time (usecs) */ #define WT_STAT_CONN_LOCK_METADATA_WAIT_INTERNAL 1179 /*! - * lock: read timestamp queue lock application thread time waiting for - * the dhandle lock (usecs) + * lock: read timestamp queue lock application thread time waiting + * (usecs) */ #define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_APPLICATION 1180 -/*! - * lock: read timestamp queue lock internal thread time waiting for the - * dhandle lock (usecs) - */ +/*! lock: read timestamp queue lock internal thread time waiting (usecs) */ #define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_INTERNAL 1181 /*! lock: read timestamp queue read lock acquisitions */ #define WT_STAT_CONN_LOCK_READ_TIMESTAMP_READ_COUNT 1182 @@ -5357,15 +5353,9 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_CONN_LOCK_TABLE_READ_COUNT 1189 /*! lock: table write lock acquisitions */ #define WT_STAT_CONN_LOCK_TABLE_WRITE_COUNT 1190 -/*! - * lock: txn global lock application thread time waiting for the dhandle - * lock (usecs) - */ +/*! lock: txn global lock application thread time waiting (usecs) */ #define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_APPLICATION 1191 -/*! - * lock: txn global lock internal thread time waiting for the dhandle - * lock (usecs) - */ +/*! lock: txn global lock internal thread time waiting (usecs) */ #define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_INTERNAL 1192 /*! lock: txn global read lock acquisitions */ #define WT_STAT_CONN_LOCK_TXN_GLOBAL_READ_COUNT 1193 diff --git a/src/third_party/wiredtiger/src/log/log.c b/src/third_party/wiredtiger/src/log/log.c index 6ce26e03c5d..d8cff7afa10 100644 --- a/src/third_party/wiredtiger/src/log/log.c +++ b/src/third_party/wiredtiger/src/log/log.c @@ -10,6 +10,7 @@ static int __log_newfile(WT_SESSION_IMPL *, bool, bool *); static int __log_openfile(WT_SESSION_IMPL *, uint32_t, uint32_t, WT_FH **); +static int __log_truncate(WT_SESSION_IMPL *, WT_LSN *, bool, bool); static int __log_write_internal( WT_SESSION_IMPL *, WT_ITEM *, WT_LSN *, uint32_t); @@ -531,7 +532,7 @@ __wt_log_extract_lognum( if (id == NULL || name == NULL) WT_RET_MSG(session, EINVAL, - "%s: unexpected usage: no id or no name", __func__); + "unexpected usage: no id or no name"); if ((p = strrchr(name, '.')) == NULL || sscanf(++p, "%" SCNu32, id) != 1) WT_RET_MSG(session, WT_ERROR, "Bad log file name '%s'", name); @@ -716,8 +717,7 @@ __log_decompress(WT_SESSION_IMPL *session, WT_ITEM *in, WT_ITEM *out) compressor = conn->log_compressor; if (compressor == NULL || compressor->decompress == NULL) WT_RET_MSG(session, WT_ERROR, - "%s: Compressed record with no configured compressor", - __func__); + "Compressed record with no configured compressor"); uncompressed_size = logrec->mem_len; WT_RET(__wt_buf_initsize(session, out, uncompressed_size)); memcpy(out->mem, in->mem, skip); @@ -734,7 +734,7 @@ __log_decompress(WT_SESSION_IMPL *session, WT_ITEM *in, WT_ITEM *out) */ if (result_len != uncompressed_size - WT_LOG_COMPRESS_SKIP) WT_RET_MSG(session, WT_ERROR, - "%s: decompression failed with incorrect size", __func__); + "decompression failed with incorrect size"); return (0); } @@ -756,8 +756,7 @@ __log_decrypt(WT_SESSION_IMPL *session, WT_ITEM *in, WT_ITEM *out) (encryptor = kencryptor->encryptor) == NULL || encryptor->decrypt == NULL) WT_RET_MSG(session, WT_ERROR, - "%s: Encrypted record with no configured decrypt method", - __func__); + "Encrypted record with no configured decrypt method"); return (__wt_decrypt(session, encryptor, WT_LOG_ENCRYPT_SKIP, in, out)); } @@ -926,7 +925,7 @@ err: __wt_scr_free(session, &buf); */ static int __log_open_verify(WT_SESSION_IMPL *session, uint32_t id, WT_FH **fhp, - WT_LSN *lsnp, uint16_t *versionp) + WT_LSN *lsnp, uint16_t *versionp, bool *need_salvagep) { WT_CONNECTION_IMPL *conn; WT_DECL_ITEM(buf); @@ -937,11 +936,15 @@ __log_open_verify(WT_SESSION_IMPL *session, uint32_t id, WT_FH **fhp, WT_LOG_RECORD *logrec; uint32_t allocsize, rectype; const uint8_t *end, *p; + bool need_salvage, salvage_mode; conn = S2C(session); + fh = NULL; log = conn->log; + need_salvage = false; WT_RET(__wt_scr_alloc(session, 0, &buf)); - WT_ERR(__log_openfile(session, id, 0, &fh)); + salvage_mode = (need_salvagep != NULL && + FLD_ISSET(conn->log_flags, WT_CONN_LOG_RECOVER_SALVAGE)); if (log == NULL) allocsize = WT_LOG_ALIGN; @@ -953,17 +956,30 @@ __log_open_verify(WT_SESSION_IMPL *session, uint32_t id, WT_FH **fhp, memset(buf->mem, 0, allocsize); /* + * Any operation that fails from here on out indicates corruption + * that could be salvaged. + */ + need_salvage = true; + + /* * Read in the log file header and verify it. */ + WT_ERR(__log_openfile(session, id, 0, &fh)); WT_ERR(__wt_read(session, fh, 0, allocsize, buf->mem)); logrec = (WT_LOG_RECORD *)buf->mem; __wt_log_record_byteswap(logrec); desc = (WT_LOG_DESC *)logrec->record; __wt_log_desc_byteswap(desc); - if (desc->log_magic != WT_LOG_MAGIC) - WT_PANIC_RET(session, WT_ERROR, - "log file %s corrupted: Bad magic number %" PRIu32, - fh->name, desc->log_magic); + if (desc->log_magic != WT_LOG_MAGIC) { + if (salvage_mode) + WT_ERR_MSG(session, WT_ERROR, + "log file %s corrupted: Bad magic number %" PRIu32, + fh->name, desc->log_magic); + else + WT_PANIC_RET(session, WT_ERROR, + "log file %s corrupted: Bad magic number %" PRIu32, + fh->name, desc->log_magic); + } /* * We cannot read future log file formats. */ @@ -1043,13 +1059,74 @@ err: __wt_scr_free(session, &buf); */ if (fhp != NULL && ret == 0) *fhp = fh; - else + else if (ret != 0 && need_salvage && salvage_mode) { + /* Let the caller know this file must be salvaged. */ + ret = 0; + WT_TRET(__wt_close(session, &fh)); + if (fhp != NULL) + *fhp = NULL; + *need_salvagep = true; + } else WT_TRET(__wt_close(session, &fh)); return (ret); } /* + * __log_record_verify -- + * Check that values of the log record header are valid. + * No byteswap of the header has been done at this point. + */ +static int +__log_record_verify(WT_SESSION_IMPL *session, WT_FH *log_fh, uint32_t offset, + WT_LOG_RECORD *logrecp, bool *corrupt) +{ + WT_LOG_RECORD logrec; + size_t i; + + *corrupt = false; + + /* + * Make our own copy of the header so we can get the bytes in the + * proper order. + */ + logrec = *logrecp; + __wt_log_record_byteswap(&logrec); + + if (F_ISSET(&logrec, ~(WT_LOG_RECORD_ALL_FLAGS))) { + WT_RET(__wt_msg(session, + "%s: log record at position %" PRIu32 + " has flag corruption 0x%" PRIx16, log_fh->name, offset, + logrec.flags)); + *corrupt = true; + } + for (i = 0; i < sizeof(logrec.unused); i++) + if (logrec.unused[i] != 0) { + WT_RET(__wt_msg(session, + "%s: log record at position %" PRIu32 + " has unused[%" WT_SIZET_FMT "] corruption 0x%" + PRIx8, log_fh->name, offset, i, logrec.unused[i])); + *corrupt = true; + } + if (logrec.mem_len != 0 && !F_ISSET(&logrec, + WT_LOG_RECORD_COMPRESSED | WT_LOG_RECORD_ENCRYPTED)) { + WT_RET(__wt_msg(session, + "%s: log record at position %" PRIu32 + " has memory len corruption 0x%" PRIx32, log_fh->name, + offset, logrec.mem_len)); + *corrupt = true; + } + if (logrec.len <= offsetof(WT_LOG_RECORD, record)) { + WT_RET(__wt_msg(session, + "%s: log record at position %" PRIu32 + " has record len corruption 0x%" PRIx32, log_fh->name, + offset, logrec.len)); + *corrupt = true; + } + return (0); +} + +/* * __log_alloc_prealloc -- * Look for a pre-allocated log file and rename it to use as the next * real log file. Called locked. @@ -1131,7 +1208,7 @@ __log_newfile(WT_SESSION_IMPL *session, bool conn_open, bool *created) WT_STAT_CONN_INCR(session, log_close_yields); __wt_log_wrlsn(session, NULL); if (++yield_cnt > 10000) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); __wt_yield(); } /* @@ -1200,7 +1277,8 @@ __log_newfile(WT_SESSION_IMPL *session, bool conn_open, bool *created) * we must pass in a local file handle. Otherwise there is a wide * window where another thread could see a NULL log file handle. */ - WT_RET(__log_open_verify(session, log->fileid, &log_fh, NULL, NULL)); + WT_RET(__log_open_verify(session, log->fileid, &log_fh, NULL, NULL, + NULL)); /* * Write the LSN at the end of the last record in the previous log file * as the first record in this log file. @@ -1412,25 +1490,31 @@ __log_truncate_file(WT_SESSION_IMPL *session, WT_FH *log_fh, wt_off_t offset) * it will truncate between the given LSN and the trunc_lsn. That is, * since we pre-allocate log files, it will free that space and allow the * log to be traversed. We use the trunc_lsn because logging has already - * opened the new/next log file before recovery ran. This function assumes - * we are in recovery or other dedicated time and not during live running. + * opened the new/next log file before recovery ran. If salvage_mode is + * set, we verify headers of log files visited and recreate them if they + * are damaged. This function assumes we are in recovery or other + * dedicated time and not during live running. */ static int -__log_truncate(WT_SESSION_IMPL *session, WT_LSN *lsn, bool this_log) +__log_truncate(WT_SESSION_IMPL *session, WT_LSN *lsn, bool this_log, + bool salvage_mode) { WT_CONNECTION_IMPL *conn; WT_DECL_RET; WT_FH *log_fh; WT_LOG *log; - uint32_t lognum; + uint32_t lognum, salvage_first, salvage_last; u_int i, logcount; char **logfiles; + bool need_salvage, opened; conn = S2C(session); log = conn->log; log_fh = NULL; logcount = 0; logfiles = NULL; + salvage_first = salvage_last = 0; + need_salvage = false; /* * Truncate the log file to the given LSN. @@ -1446,6 +1530,10 @@ __log_truncate(WT_SESSION_IMPL *session, WT_LSN *lsn, bool this_log) WT_ERR(__wt_fsync(session, log_fh, true)); WT_ERR(__wt_close(session, &log_fh)); + if (salvage_mode) + WT_ERR(__wt_msg(session, + "salvage: log file %" PRIu32 " truncated", lsn->l.file)); + /* * If we just want to truncate the current log, return and skip * looking for intervening logs. @@ -1456,7 +1544,32 @@ __log_truncate(WT_SESSION_IMPL *session, WT_LSN *lsn, bool this_log) for (i = 0; i < logcount; i++) { WT_ERR(__wt_log_extract_lognum(session, logfiles[i], &lognum)); if (lognum > lsn->l.file && lognum < log->trunc_lsn.l.file) { - WT_ERR(__log_openfile(session, lognum, 0, &log_fh)); + opened = false; + if (salvage_mode) { + /* + * When salvaging, we verify that the + * header of the log file is valid. + * If not, create a new, empty one. + */ + need_salvage = false; + WT_ERR(__log_open_verify(session, lognum, + &log_fh, NULL, NULL, &need_salvage)); + if (need_salvage) { + WT_ASSERT(session, log_fh == NULL); + WT_ERR(__wt_log_remove(session, + WT_LOG_FILENAME, lognum)); + WT_ERR(__wt_log_allocfile(session, + lognum, WT_LOG_FILENAME)); + } else + opened = true; + + if (salvage_first == 0) + salvage_first = lognum; + salvage_last = lognum; + } + if (!opened) + WT_ERR(__log_openfile(session, lognum, 0, + &log_fh)); /* * If there are intervening files pre-allocated, * truncate them to the end of the log file header. @@ -1469,6 +1582,17 @@ __log_truncate(WT_SESSION_IMPL *session, WT_LSN *lsn, bool this_log) } err: WT_TRET(__wt_close(session, &log_fh)); WT_TRET(__wt_fs_directory_list_free(session, &logfiles, logcount)); + if (salvage_first != 0) { + if (salvage_last > salvage_first) + WT_TRET(__wt_msg(session, + "salvage: log files %" PRIu32 "-%" PRIu32 + " truncated at beginning", salvage_first, + salvage_last)); + else + WT_TRET(__wt_msg(session, + "salvage: log file %" PRIu32 + " truncated at beginning", salvage_first)); + } return (ret); } @@ -1566,13 +1690,12 @@ __wt_log_open(WT_SESSION_IMPL *session) uint16_t version; u_int i, logcount; char **logfiles; + bool need_salvage; conn = S2C(session); log = conn->log; logfiles = NULL; logcount = 0; - lastlog = 0; - firstlog = UINT32_MAX; /* * Open up a file handle to the log directory if we haven't. @@ -1587,9 +1710,14 @@ __wt_log_open(WT_SESSION_IMPL *session) if (!F_ISSET(conn, WT_CONN_READONLY)) WT_ERR(__log_prealloc_remove(session)); +again: /* * Now look at the log files and set our LSNs. */ + lastlog = 0; + firstlog = UINT32_MAX; + need_salvage = false; + WT_ERR(__log_get_files(session, WT_LOG_FILENAME, &logfiles, &logcount)); for (i = 0; i < logcount; i++) { WT_ERR(__wt_log_extract_lognum(session, logfiles[i], &lognum)); @@ -1610,8 +1738,23 @@ __wt_log_open(WT_SESSION_IMPL *session) * we create a new log file so that we can detect an unsupported * version before modifying the file space. */ - WT_ERR(__log_open_verify(session, - lastlog, NULL, NULL, &version)); + WT_ERR(__log_open_verify(session, lastlog, NULL, NULL, + &version, &need_salvage)); + + /* + * If we were asked to salvage and the last log file was + * indeed corrupt, remove it and try all over again. + */ + if (need_salvage) { + WT_ERR(__wt_log_remove( + session, WT_LOG_FILENAME, lastlog)); + WT_ERR(__wt_msg(session, + "salvage: log file %" PRIu32 " removed", lastlog)); + WT_ERR(__wt_fs_directory_list_free(session, &logfiles, + logcount)); + logfiles = NULL; + goto again; + } } /* @@ -1641,7 +1784,7 @@ __wt_log_open(WT_SESSION_IMPL *session) * have to close the file. */ WT_ERR(__log_open_verify(session, - lognum, NULL, NULL, &version)); + lognum, NULL, NULL, &version, NULL)); /* * If we find any log file at the wrong version * set the flag and we're done. @@ -1711,17 +1854,20 @@ __wt_log_close(WT_SESSION_IMPL *session) * file is zeroes. */ static int -__log_has_hole(WT_SESSION_IMPL *session, - WT_FH *fh, wt_off_t log_size, wt_off_t offset, bool *hole) +__log_has_hole(WT_SESSION_IMPL *session, WT_FH *fh, wt_off_t log_size, + wt_off_t offset, wt_off_t *error_offset, bool *hole) { WT_CONNECTION_IMPL *conn; WT_DECL_RET; WT_LOG *log; + WT_LOG_RECORD *logrec; wt_off_t off, remainder; - size_t bufsz, rdlen; - char *buf, *zerobuf; + size_t buf_left, bufsz, rdlen; + char *buf, *p, *zerobuf; + bool corrupt; - *hole = false; + *error_offset = 0; + corrupt = *hole = false; conn = S2C(session); log = conn->log; @@ -1753,6 +1899,31 @@ __log_has_hole(WT_SESSION_IMPL *session, rdlen = WT_MIN(bufsz, (size_t)remainder); WT_ERR(__wt_read(session, fh, off, rdlen, buf)); if (memcmp(buf, zerobuf, rdlen) != 0) { + /* + * Find where the next log record starts after the + * hole. + */ + for (p = buf, buf_left = rdlen; buf_left > 0; + buf_left -= rdlen, p += rdlen) { + rdlen = WT_MIN(log->allocsize, buf_left); + if (memcmp(p, zerobuf, rdlen) != 0) + break; + } + /* + * A presumed log record begins here where the buffer + * becomes non-zero. If we have enough of a log record + * present in the buffer, we either have a valid header + * or corruption. Verify the header of this record to + * determine whether it is just a hole or corruption. + */ + logrec = (WT_LOG_RECORD *)p; + if (buf_left >= sizeof(WT_LOG_RECORD)) { + off += p - buf; + WT_ERR(__log_record_verify(session, fh, + (uint32_t)off, logrec, &corrupt)); + if (corrupt) + *error_offset = off; + } *hole = true; break; } @@ -1764,6 +1935,42 @@ err: __wt_free(session, buf); } /* + * __log_check_partial_write -- + * Determine if the log record may be a partial write. If that's + * possible, return true, otherwise false. + * + * Since the log file is initially zeroed up to a predetermined size, + * any record that falls within that boundary that ends in one or + * more zeroes may be partial (or the initial record may have been + * padded with zeroes before writing). The only way we have any certainty + * is if the last byte is non-zero, when that happens, we know that + * the write cannot be partial. + * + * When we have a checksum mismatch, it is important to know that whether + * it may be 1) the result of a partial write or 2) the result of + * corruption. The former can happen in normal operations, and we + * will silently truncate the log when it occurs. The latter will + * result in an error during recovery, and requires salvage to fix. + */ +static bool +__log_check_partial_write(WT_SESSION_IMPL *session, WT_ITEM *buf, + uint32_t reclen) +{ + uint8_t *rec; + + WT_UNUSED(session); + + /* + * We only check the final byte since that's the only way have any + * certainty. Even if the second to last byte is non-zero and the + * last byte is zero, that could still technically be the result of + * a partial write, however unlikely it may be. + */ + rec = buf->mem; + return (reclen > 0 && rec[reclen - 1] == 0); +} + +/* * __wt_log_release -- * Release a log slot. */ @@ -1932,6 +2139,22 @@ err: if (locked) } /* + * __log_salvageable_error -- + * Show error messages consistently for a salvageable error. + */ +static int +__log_salvageable_error(WT_SESSION_IMPL *session, const char *log_name, + const char *extra_msg, wt_off_t offset) +{ + if (!FLD_ISSET(S2C(session)->log_flags, WT_CONN_LOG_RECOVER_SALVAGE)) + WT_PANIC_RET(session, WT_ERROR, + "log file %s corrupted%s at position %" PRIuMAX + ", use salvage to fix", log_name, extra_msg, + (uintmax_t)offset); + return (WT_ERROR); +} + +/* * __wt_log_scan -- * Scan the logs, calling a function on each record found. */ @@ -1951,21 +2174,22 @@ __wt_log_scan(WT_SESSION_IMPL *session, WT_LSN *lsnp, uint32_t flags, WT_LOG *log; WT_LOG_RECORD *logrec; WT_LSN end_lsn, next_lsn, prev_eof, prev_lsn, rd_lsn, start_lsn; - wt_off_t log_size; + wt_off_t bad_offset, log_size; uint32_t allocsize, firstlog, lastlog, lognum, rdup_len, reclen; uint16_t version; u_int i, logcount; int firstrecord; char **logfiles; - bool eol, partial_record; + bool corrupt, eol, need_salvage, partial_record; conn = S2C(session); log = conn->log; log_fh = NULL; logcount = 0; logfiles = NULL; - eol = false; + corrupt = eol = false; firstrecord = 1; + need_salvage = false; /* * If the caller did not give us a callback function there is nothing @@ -1992,7 +2216,7 @@ __wt_log_scan(WT_SESSION_IMPL *session, WT_LSN *lsnp, uint32_t flags, start_lsn = log->ckpt_lsn; else if (!LF_ISSET(WT_LOGSCAN_FIRST)) WT_RET_MSG(session, WT_ERROR, - "%s: WT_LOGSCAN_FIRST not set", __func__); + "WT_LOGSCAN_FIRST not set"); } lastlog = log->fileid; } else { @@ -2064,8 +2288,10 @@ __wt_log_scan(WT_SESSION_IMPL *session, WT_LSN *lsnp, uint32_t flags, if (!WT_IS_INIT_LSN(lsnp)) start_lsn = *lsnp; } - WT_ERR(__log_open_verify(session, - start_lsn.l.file, &log_fh, &prev_lsn, NULL)); + WT_ERR(__log_open_verify(session, start_lsn.l.file, &log_fh, &prev_lsn, + NULL, &need_salvage)); + if (need_salvage) + WT_ERR_MSG(session, WT_ERROR, "log file requires salvage"); WT_ERR(__wt_filesize(session, log_fh, &log_size)); rd_lsn = start_lsn; if (LF_ISSET(WT_LOGSCAN_RECOVER)) @@ -2081,14 +2307,21 @@ __wt_log_scan(WT_SESSION_IMPL *session, WT_LSN *lsnp, uint32_t flags, advance: if (rd_lsn.l.offset == log_size) partial_record = false; - else + else { /* * See if there is anything non-zero at the * end of this log file. */ WT_ERR(__log_has_hole( session, log_fh, log_size, - rd_lsn.l.offset, &partial_record)); + rd_lsn.l.offset, &bad_offset, + &partial_record)); + if (bad_offset != 0) { + need_salvage = true; + WT_ERR(__log_salvageable_error(session, + log_fh->name, "", bad_offset)); + } + } /* * If we read the last record, go to the next file. */ @@ -2103,7 +2336,8 @@ advance: __wt_verbose(session, WT_VERB_LOG, "Truncate end of log %" PRIu32 "/%" PRIu32, rd_lsn.l.file, rd_lsn.l.offset); - WT_ERR(__log_truncate(session, &rd_lsn, true)); + WT_ERR(__log_truncate(session, &rd_lsn, true, + false)); } /* * If we had a partial record, we'll want to break @@ -2128,7 +2362,11 @@ advance: " through %" PRIu32, rd_lsn.l.file, end_lsn.l.file); WT_ERR(__log_open_verify(session, - rd_lsn.l.file, &log_fh, &prev_lsn, &version)); + rd_lsn.l.file, &log_fh, &prev_lsn, &version, + &need_salvage)); + if (need_salvage) + WT_ERR_MSG(session, WT_ERROR, + "log file requires salvage"); /* * Opening the log file reads with verify sets up the * previous LSN from the first record. This detects @@ -2161,10 +2399,15 @@ advance: } /* * Read the minimum allocation size a record could be. + * Conditionally set the need_salvage flag so that if the + * read fails, we know this is an situation we can salvage. */ WT_ASSERT(session, buf->memsize >= allocsize); + need_salvage = FLD_ISSET(conn->log_flags, + WT_CONN_LOG_RECOVER_SALVAGE); WT_ERR(__wt_read(session, log_fh, rd_lsn.l.offset, (size_t)allocsize, buf->mem)); + need_salvage = false; /* * See if we need to read more than the allocation size. We * expect that we rarely will have to read more. Most log @@ -2187,7 +2430,13 @@ advance: */ if (reclen == 0) { WT_ERR(__log_has_hole( - session, log_fh, log_size, rd_lsn.l.offset, &eol)); + session, log_fh, log_size, rd_lsn.l.offset, + &bad_offset, &eol)); + if (bad_offset != 0) { + need_salvage = true; + WT_ERR(__log_salvageable_error(session, + log_fh->name, "", bad_offset)); + } if (eol) /* Found a hole. This LSN is the end. */ break; @@ -2215,14 +2464,9 @@ advance: WT_STAT_CONN_INCR(session, log_scan_rereads); } /* - * We read in the record, verify checksum. - * - * Handle little- and big-endian objects. Objects are written - * in little-endian format: save the header checksum, and - * calculate the checksum for the header in its little-endian - * form. Then, restore the header's checksum, and byte-swap - * the whole thing as necessary, leaving us with a calculated - * checksum that should match the checksum in the header. + * We read in the record, now verify the checksum. A failed + * checksum does not imply corruption, it may be the result + * of a partial write. */ buf->size = reclen; logrec = (WT_LOG_RECORD *)buf->mem; @@ -2236,6 +2480,31 @@ advance: */ if (log != NULL) log->trunc_lsn = rd_lsn; + /* Make a check to see if it may be a partial write. */ + if (!__log_check_partial_write(session, buf, reclen)) { + /* + * It's not a partial write, and we have a bad + * checksum. We treat it as a corruption that + * must be salvaged. + */ + need_salvage = true; + WT_ERR(__log_salvageable_error(session, + log_fh->name, ", bad checksum", + rd_lsn.l.offset)); + } else { + /* + * It may be a partial write, or it's possible + * that the header is corrupt. Make a sanity + * check of the log record header. + */ + WT_ERR(__log_record_verify(session, log_fh, + rd_lsn.l.offset, logrec, &corrupt)); + if (corrupt) { + need_salvage = true; + WT_ERR(__log_salvageable_error(session, + log_fh->name, "", rd_lsn.l.offset)); + } + } /* * If the user asked for a specific LSN and it is not * a valid LSN, return WT_NOTFOUND. @@ -2291,11 +2560,22 @@ advance: __wt_verbose(session, WT_VERB_LOG, "End of recovery truncate end of log %" PRIu32 "/%" PRIu32, rd_lsn.l.file, rd_lsn.l.offset); - WT_ERR(__log_truncate(session, &rd_lsn, false)); + WT_ERR(__log_truncate(session, &rd_lsn, false, false)); } err: WT_STAT_CONN_INCR(session, log_scans); /* + * If we are salvaging and failed a salvageable operation, then + * truncate the log at the fail point. + */ + if (ret != 0 && ret != WT_PANIC && need_salvage) { + WT_TRET(__wt_close(session, &log_fh)); + log_fh = NULL; + WT_TRET(__log_truncate(session, &rd_lsn, false, true)); + ret = 0; + } + + /* * If the first attempt to read a log record results in * an error recovery is likely going to fail. Try to provide * a helpful failure message. @@ -2491,7 +2771,7 @@ __log_write_internal(WT_SESSION_IMPL *session, WT_ITEM *record, WT_LSN *lsnp, WT_LSN lsn; WT_MYSLOT myslot; int64_t release_size; - uint32_t force, rdup_len; + uint32_t fill_size, force, rdup_len; bool free_slot; conn = S2C(session); @@ -2520,10 +2800,39 @@ __log_write_internal(WT_SESSION_IMPL *session, WT_ITEM *record, WT_LSN *lsnp, /* * If the caller's record only partially fills the necessary * space, we need to zero-fill the remainder. + * + * The cast is safe, we've already checked to make sure it's in range. */ - if (record->size != rdup_len) { - memset((uint8_t *)record->mem + record->size, 0, - rdup_len - record->size); + fill_size = rdup_len - (uint32_t)record->size; + if (fill_size != 0) { + memset((uint8_t *)record->mem + record->size, 0, fill_size); + /* + * Set the last byte of the log record to a non-zero value, + * that allows us, on the input side, to tell that a log + * record was completely written; there couldn't have been + * a partial write. That means that any checksum mismatch + * in those conditions is a log corruption. + * + * Without this changed byte, when we see a zeroed last byte, + * we must always treat a checksum error as a possible partial + * write. Since partial writes can happen as a result of an + * interrupted process (for example, a shutdown), we must + * treat a checksum error as a normal occurrence, and merely + * the place where the log must be truncated. So any real + * corruption within log records is hard to detect as such. + * + * However, we can only make this modification if there is + * more than one byte being filled, as the first zero byte + * past the actual record is needed to terminate the loop + * in txn_commit_apply. + * + * This is not a log format change, as we only are changing a + * byte in the padding portion of a record, and no logging code + * has ever checked that it is any particular value up to now. + */ + if (fill_size > 1) + *((uint8_t *)record->mem + rdup_len - 1) = + WT_DEBUG_BYTE; record->size = rdup_len; } /* diff --git a/src/third_party/wiredtiger/src/log/log_auto.c b/src/third_party/wiredtiger/src/log/log_auto.c index 703a87b09d4..3a6aabf32c0 100644 --- a/src/third_party/wiredtiger/src/log/log_auto.c +++ b/src/third_party/wiredtiger/src/log/log_auto.c @@ -852,7 +852,7 @@ __wt_txn_op_printlog(WT_SESSION_IMPL *session, WT_RET(__wt_logop_prev_lsn_print(session, pp, end, flags)); break; - WT_ILLEGAL_VALUE(session); + WT_ILLEGAL_VALUE(session, optype); } return (0); diff --git a/src/third_party/wiredtiger/src/log/log_slot.c b/src/third_party/wiredtiger/src/log/log_slot.c index 8deda5e242f..c75181d0687 100644 --- a/src/third_party/wiredtiger/src/log/log_slot.c +++ b/src/third_party/wiredtiger/src/log/log_slot.c @@ -119,7 +119,7 @@ retry: * decide if retrying is necessary or not. */ if (forced && WT_LOG_SLOT_INPROGRESS(old_state)) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); /* * If someone else is switching out this slot we lost. Nothing to * do but return. Return WT_NOTFOUND anytime the given slot was diff --git a/src/third_party/wiredtiger/src/lsm/lsm_cursor.c b/src/third_party/wiredtiger/src/lsm/lsm_cursor.c index 13d7d857a04..5dd3122d41c 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_cursor.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_cursor.c @@ -1752,6 +1752,8 @@ __wt_clsm_close(WT_CURSOR *cursor) */ clsm = (WT_CURSOR_LSM *)cursor; CURSOR_API_CALL_PREPARE_ALLOWED(cursor, session, close, NULL); +err: + WT_TRET(__clsm_close_cursors(session, clsm, 0, clsm->nchunks)); __clsm_free_chunks(session, clsm); @@ -1760,9 +1762,9 @@ __wt_clsm_close(WT_CURSOR *cursor) if (clsm->lsm_tree != NULL) __wt_lsm_tree_release(session, clsm->lsm_tree); - WT_TRET(__wt_cursor_close(cursor)); + __wt_cursor_close(cursor); -err: API_END_RET(session, ret); + API_END_RET(session, ret); } /* @@ -1837,10 +1839,9 @@ __wt_clsm_open(WT_SESSION_IMPL *session, WT_ASSERT(session, !bulk || lsm_tree->excl_session != NULL); WT_ERR(__wt_calloc_one(session, &clsm)); - - cursor = &clsm->iface; + cursor = (WT_CURSOR *)clsm; *cursor = iface; - cursor->session = &session->iface; + cursor->session = (WT_SESSION *)session; WT_ERR(__wt_strdup(session, lsm_tree->name, &cursor->uri)); cursor->key_format = lsm_tree->key_format; cursor->value_format = lsm_tree->value_format; diff --git a/src/third_party/wiredtiger/src/lsm/lsm_manager.c b/src/third_party/wiredtiger/src/lsm/lsm_manager.c index 1a5c60344bc..7a354403ad2 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_manager.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_manager.c @@ -311,7 +311,7 @@ __wt_lsm_manager_destroy(WT_SESSION_IMPL *session) ret = __wt_lsm_tree_close_all(session); WT_TRET(__wt_thread_join( - session, manager->lsm_worker_cookies[0].tid)); + session, &manager->lsm_worker_cookies[0].tid)); /* Release memory from any operations left on the queue. */ while ((current = TAILQ_FIRST(&manager->switchqh)) != NULL) { diff --git a/src/third_party/wiredtiger/src/lsm/lsm_tree.c b/src/third_party/wiredtiger/src/lsm/lsm_tree.c index 16b28a1aecc..03feafe3c8c 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_tree.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_tree.c @@ -419,7 +419,8 @@ __lsm_tree_find(WT_SESSION_IMPL *session, */ if (!__wt_atomic_cas_ptr( &lsm_tree->excl_session, NULL, session)) - return (EBUSY); + return (__wt_set_return( + session, EBUSY)); /* * Drain the work queue before checking for @@ -431,7 +432,8 @@ __lsm_tree_find(WT_SESSION_IMPL *session, if (lsm_tree->refcnt != 1) { __wt_lsm_tree_release( session, lsm_tree); - return (EBUSY); + return (__wt_set_return( + session, EBUSY)); } } else { (void)__wt_atomic_add32(&lsm_tree->refcnt, 1); @@ -445,7 +447,8 @@ __lsm_tree_find(WT_SESSION_IMPL *session, lsm_tree->refcnt > 0); __wt_lsm_tree_release( session, lsm_tree); - return (EBUSY); + return (__wt_set_return( + session, EBUSY)); } } diff --git a/src/third_party/wiredtiger/src/lsm/lsm_worker.c b/src/third_party/wiredtiger/src/lsm/lsm_worker.c index 8588737f6c3..3579207e0c7 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_worker.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_worker.c @@ -37,7 +37,7 @@ __wt_lsm_worker_stop(WT_SESSION_IMPL *session, WT_LSM_WORKER_ARGS *args) { args->running = false; args->tid_set = false; - return (__wt_thread_join(session, args->tid)); + return (__wt_thread_join(session, &args->tid)); } /* diff --git a/src/third_party/wiredtiger/src/meta/meta_track.c b/src/third_party/wiredtiger/src/meta/meta_track.c index a8289b91ffa..6640af0491c 100644 --- a/src/third_party/wiredtiger/src/meta/meta_track.c +++ b/src/third_party/wiredtiger/src/meta/meta_track.c @@ -8,6 +8,7 @@ #include "wt_internal.h" +#undef WT_ENABLE_SCHEMA_TXN /* * WT_META_TRACK -- A tracked metadata operation: a non-transactional log, * maintained to make it easy to unroll simple metadata and filesystem @@ -118,6 +119,7 @@ __wt_meta_track_on(WT_SESSION_IMPL *session) if (!F_ISSET(&session->txn, WT_TXN_RUNNING)) { #ifdef WT_ENABLE_SCHEMA_TXN WT_RET(__wt_txn_begin(session, NULL)); + __wt_errx(session, "TRACK: Using internal schema txn"); #endif F_SET(session, WT_SESSION_SCHEMA_TXN); } @@ -279,6 +281,7 @@ __wt_meta_track_off(WT_SESSION_IMPL *session, bool need_sync, bool unroll) F_CLR(session, WT_SESSION_SCHEMA_TXN); #ifdef WT_ENABLE_SCHEMA_TXN WT_ERR(__wt_txn_commit(session, NULL)); + __wt_errx(session, "TRACK: Commit internal schema txn"); #endif } @@ -339,6 +342,7 @@ err: /* WT_ASSERT(session, unroll || saved_ret != 0 || session->txn.mod_count == 0); #ifdef WT_ENABLE_SCHEMA_TXN + __wt_errx(session, "TRACK: Abort internal schema txn"); WT_TRET(__wt_txn_rollback(session, NULL)); #endif } diff --git a/src/third_party/wiredtiger/src/meta/meta_turtle.c b/src/third_party/wiredtiger/src/meta/meta_turtle.c index 2c83167c28f..af5c753b70a 100644 --- a/src/third_party/wiredtiger/src/meta/meta_turtle.c +++ b/src/third_party/wiredtiger/src/meta/meta_turtle.c @@ -82,7 +82,8 @@ __metadata_load_hot_backup(WT_SESSION_IMPL *session) break; WT_ERR(__wt_getline(session, fs, value)); if (value->size == 0) - WT_ERR(__wt_illegal_value(session, WT_METADATA_BACKUP)); + WT_PANIC_ERR(session, EINVAL, + "%s: zero-length value", WT_METADATA_BACKUP); WT_ERR(__wt_metadata_update(session, key->data, value->data)); } @@ -329,8 +330,10 @@ err: WT_TRET(__wt_fclose(session, &fs)); * something has gone horribly wrong, except for the compatibility * setting which is optional. */ - return (ret == 0 || strcmp(key, WT_METADATA_COMPAT) == 0 ? ret : - __wt_illegal_value(session, WT_METADATA_TURTLE)); + if (ret == 0 || strcmp(key, WT_METADATA_COMPAT) == 0) + return (ret); + WT_PANIC_RET(session, ret, + "%s: fatal turtle file read error", WT_METADATA_TURTLE); } /* @@ -388,5 +391,8 @@ err: WT_TRET(__wt_fclose(session, &fs)); * An error updating the turtle file means something has gone horribly * wrong -- we're done. */ - return (ret == 0 ? 0 : __wt_illegal_value(session, WT_METADATA_TURTLE)); + if (ret == 0) + return (ret); + WT_PANIC_RET(session, ret, + "%s: fatal turtle file update error", WT_METADATA_TURTLE); } diff --git a/src/third_party/wiredtiger/src/optrack/optrack.c b/src/third_party/wiredtiger/src/optrack/optrack.c index ccec13d433b..3a2ac879122 100644 --- a/src/third_party/wiredtiger/src/optrack/optrack.c +++ b/src/third_party/wiredtiger/src/optrack/optrack.c @@ -21,12 +21,15 @@ __wt_optrack_record_funcid( WT_DECL_ITEM(tmp); WT_DECL_RET; wt_off_t fsize; + bool locked; conn = S2C(session); + locked = false; WT_ERR(__wt_scr_alloc(session, strlen(func) + 32, &tmp)); __wt_spin_lock(session, &conn->optrack_map_spinlock); + locked = true; if (*func_idp == 0) { *func_idp = ++optrack_uid; @@ -38,10 +41,12 @@ __wt_optrack_record_funcid( } if (0) { -err: WT_PANIC_MSG(session, ret, "%s", __func__); +err: WT_PANIC_MSG(session, ret, + "operation tracking initialization failure"); } - __wt_spin_unlock(session, &conn->optrack_map_spinlock); + if (locked) + __wt_spin_unlock(session, &conn->optrack_map_spinlock); __wt_scr_free(session, &tmp); } @@ -61,8 +66,7 @@ __optrack_open_file(WT_SESSION_IMPL *session) conn = S2C(session); if (!F_ISSET(conn, WT_CONN_OPTRACK)) - WT_RET_MSG(session, WT_ERROR, - "%s: WT_CONN_OPTRACK not set", __func__); + WT_RET_MSG(session, WT_ERROR, "WT_CONN_OPTRACK not set"); WT_RET(__wt_scr_alloc(session, 0, &buf)); WT_ERR(__wt_filename_construct(session, conn->optrack_path, diff --git a/src/third_party/wiredtiger/src/os_common/os_abort.c b/src/third_party/wiredtiger/src/os_common/os_abort.c index 85dcc741855..54cae3e61aa 100644 --- a/src/third_party/wiredtiger/src/os_common/os_abort.c +++ b/src/third_party/wiredtiger/src/os_common/os_abort.c @@ -15,6 +15,7 @@ void __wt_abort(WT_SESSION_IMPL *session) WT_GCC_FUNC_ATTRIBUTE((noreturn)) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { #ifdef HAVE_ATTACH u_int i; diff --git a/src/third_party/wiredtiger/src/os_common/os_errno.c b/src/third_party/wiredtiger/src/os_common/os_errno.c index 0f57658ab23..8dccab0373e 100644 --- a/src/third_party/wiredtiger/src/os_common/os_errno.c +++ b/src/third_party/wiredtiger/src/os_common/os_errno.c @@ -38,16 +38,23 @@ __wt_strerror(WT_SESSION_IMPL *session, int error, char *errbuf, size_t errlen) return (p); /* - * When called from wiredtiger_strerror, write a passed-in buffer. - * When called from WT_SESSION.strerror, write the session's buffer. + * !!! + * This function MUST handle a NULL WT_SESSION_IMPL handle. + * + * When called with a passed-in buffer, write the buffer. + * When called with a valid session handle, write the session's buffer. + * There's no way the session's buffer should be NULL if buffer format + * succeeded, but Coverity is unconvinced; regardless, a test for NULL + * isn't a bad idea given future code changes in the underlying code. * * Fallback to a generic message. */ - if (session == NULL && + if (errbuf != NULL && __wt_snprintf(errbuf, errlen, "error return: %d", error) == 0) return (errbuf); if (session != NULL && __wt_buf_fmt( - session, &session->err, "error return: %d", error) == 0) + session, &session->err, "error return: %d", error) == 0 && + session->err.data != NULL) return (session->err.data); /* Defeated. */ diff --git a/src/third_party/wiredtiger/src/os_common/os_fs_inmemory.c b/src/third_party/wiredtiger/src/os_common/os_fs_inmemory.c index cb7a05e05d9..182f89cc5e1 100644 --- a/src/third_party/wiredtiger/src/os_common/os_fs_inmemory.c +++ b/src/third_party/wiredtiger/src/os_common/os_fs_inmemory.c @@ -63,7 +63,7 @@ __im_handle_remove(WT_SESSION_IMPL *session, if (im_fh->ref != 0) { __wt_err(session, EBUSY, "%s: file-remove", im_fh->iface.name); if (!force) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); } bucket = im_fh->name_hash % WT_HASH_ARRAY_SIZE; @@ -272,7 +272,7 @@ __im_fs_size(WT_FILE_SYSTEM *file_system, /* Search for the handle, then get its size. */ if ((im_fh = __im_handle_search(file_system, name)) == NULL) - ret = ENOENT; + ret = __wt_set_return(session, ENOENT); else *sizep = (wt_off_t)im_fh->buf.size; diff --git a/src/third_party/wiredtiger/src/os_posix/os_dir.c b/src/third_party/wiredtiger/src/os_posix/os_dir.c index 2c2cb084a91..54614d67649 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_dir.c +++ b/src/third_party/wiredtiger/src/os_posix/os_dir.c @@ -41,11 +41,12 @@ __directory_list_worker(WT_FILE_SYSTEM *file_system, * but various static analysis programs remain unconvinced, check both. */ WT_SYSCALL_RETRY(((dirp = opendir(directory)) == NULL ? -1 : 0), ret); - if (dirp == NULL && ret == 0) - ret = EINVAL; - if (ret != 0) + if (dirp == NULL || ret != 0) { + if (ret == 0) + ret = EINVAL; WT_RET_MSG(session, ret, "%s: directory-list: opendir", directory); + } for (count = 0; (dp = readdir(dirp)) != NULL;) { /* diff --git a/src/third_party/wiredtiger/src/os_posix/os_fallocate.c b/src/third_party/wiredtiger/src/os_posix/os_fallocate.c index 6f6e6a6bdc2..cde6adfe780 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_fallocate.c +++ b/src/third_party/wiredtiger/src/os_posix/os_fallocate.c @@ -33,9 +33,9 @@ __posix_std_fallocate( return (ret); #else WT_UNUSED(file_handle); - WT_UNUSED(wt_session); WT_UNUSED(offset); - return (ENOTSUP); + + return (__wt_set_return((WT_SESSION_IMPL *)wt_session, ENOTSUP)); #endif } @@ -66,9 +66,9 @@ __posix_sys_fallocate( return (ret); #else WT_UNUSED(file_handle); - WT_UNUSED(wt_session); WT_UNUSED(offset); - return (ENOTSUP); + + return (__wt_set_return((WT_SESSION_IMPL *)wt_session, ENOTSUP)); #endif } @@ -92,9 +92,9 @@ __posix_posix_fallocate( return (ret); #else WT_UNUSED(file_handle); - WT_UNUSED(wt_session); WT_UNUSED(offset); - return (ENOTSUP); + + return (__wt_set_return((WT_SESSION_IMPL *)wt_session, ENOTSUP)); #endif } diff --git a/src/third_party/wiredtiger/src/os_posix/os_fs.c b/src/third_party/wiredtiger/src/os_posix/os_fs.c index 0af67ad38c5..42e6e411440 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_fs.c +++ b/src/third_party/wiredtiger/src/os_posix/os_fs.c @@ -338,7 +338,7 @@ __posix_file_advise(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session, */ if (ret == EINVAL) { file_handle->fh_advise = NULL; - return (ENOTSUP); + return (__wt_set_return(session, ENOTSUP)); } WT_RET_MSG(session, ret, diff --git a/src/third_party/wiredtiger/src/os_posix/os_map.c b/src/third_party/wiredtiger/src/os_posix/os_map.c index b9ec284e124..f04a966c468 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_map.c +++ b/src/third_party/wiredtiger/src/os_posix/os_map.c @@ -33,7 +33,7 @@ __wt_posix_map(WT_FILE_HANDLE *fh, WT_SESSION *wt_session, * mmap(2) of files with direct I/O to the same files. */ if (pfh->direct_io) - return (ENOTSUP); + return (__wt_set_return(session, ENOTSUP)); /* * There's no locking here to prevent the underlying file from changing diff --git a/src/third_party/wiredtiger/src/os_posix/os_thread.c b/src/third_party/wiredtiger/src/os_posix/os_thread.c index 7dd803e1b00..3c972b2991d 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_thread.c +++ b/src/third_party/wiredtiger/src/os_posix/os_thread.c @@ -40,14 +40,15 @@ __wt_thread_create(WT_SESSION_IMPL *session, * Wait for a thread of control to exit. */ int -__wt_thread_join(WT_SESSION_IMPL *session, wt_thread_t tid) +__wt_thread_join(WT_SESSION_IMPL *session, wt_thread_t *tid) WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { WT_DECL_RET; /* Only attempt to join if thread was created successfully */ - if (!tid.created) + if (!tid->created) return (0); + tid->created = false; /* * Joining a thread isn't a memory barrier, but WiredTiger commonly @@ -56,11 +57,9 @@ __wt_thread_join(WT_SESSION_IMPL *session, wt_thread_t tid) */ WT_FULL_BARRIER(); - WT_SYSCALL(pthread_join(tid.id, NULL), ret); - if (ret == 0) { - tid.created = false; + WT_SYSCALL(pthread_join(tid->id, NULL), ret); + if (ret == 0) return (0); - } WT_RET_MSG(session, ret, "pthread_join"); } diff --git a/src/third_party/wiredtiger/src/os_posix/os_time.c b/src/third_party/wiredtiger/src/os_posix/os_time.c index 7db4522a3e6..22b2b2effad 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_time.c +++ b/src/third_party/wiredtiger/src/os_posix/os_time.c @@ -47,3 +47,17 @@ __wt_epoch_raw(WT_SESSION_IMPL *session, struct timespec *tsp) NO TIME-OF-DAY IMPLEMENTATION: see src/os_posix/os_time.c #endif } + +/* + * __wt_localtime -- + * Return the current local broken-down time. + */ +int +__wt_localtime(WT_SESSION_IMPL *session, const time_t *timep, struct tm *result) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) +{ + if (localtime_r(timep, result) != NULL) + return (0); + + WT_RET_MSG(session, __wt_errno(), "localtime_r"); +} diff --git a/src/third_party/wiredtiger/src/os_win/os_fs.c b/src/third_party/wiredtiger/src/os_win/os_fs.c index 66f4de87299..c75e8365116 100644 --- a/src/third_party/wiredtiger/src/os_win/os_fs.c +++ b/src/third_party/wiredtiger/src/os_win/os_fs.c @@ -393,7 +393,7 @@ __win_file_set_end( if (SetEndOfFile(win_fh->filehandle_secondary) == FALSE) { if (GetLastError() == ERROR_USER_MAPPED_FILE) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); windows_error = __wt_getlasterror(); ret = __wt_map_windows_error(windows_error); __wt_err(session, ret, diff --git a/src/third_party/wiredtiger/src/os_win/os_thread.c b/src/third_party/wiredtiger/src/os_win/os_thread.c index f27ea84181e..c553bbffef6 100644 --- a/src/third_party/wiredtiger/src/os_win/os_thread.c +++ b/src/third_party/wiredtiger/src/os_win/os_thread.c @@ -38,13 +38,14 @@ __wt_thread_create(WT_SESSION_IMPL *session, * Wait for a thread of control to exit. */ int -__wt_thread_join(WT_SESSION_IMPL *session, wt_thread_t tid) +__wt_thread_join(WT_SESSION_IMPL *session, wt_thread_t *tid) { DWORD windows_error; /* Only attempt to join if thread was created successfully */ - if (!tid.created) + if (!tid->created) return (0); + tid->created = false; /* * Joining a thread isn't a memory barrier, but WiredTiger commonly @@ -54,7 +55,7 @@ __wt_thread_join(WT_SESSION_IMPL *session, wt_thread_t tid) WT_FULL_BARRIER(); if ((windows_error = - WaitForSingleObject(tid.id, INFINITE)) != WAIT_OBJECT_0) { + WaitForSingleObject(tid->id, INFINITE)) != WAIT_OBJECT_0) { if (windows_error == WAIT_FAILED) windows_error = __wt_getlasterror(); __wt_errx(session, "thread join: WaitForSingleObject: %s", @@ -64,14 +65,13 @@ __wt_thread_join(WT_SESSION_IMPL *session, wt_thread_t tid) return (WT_PANIC); } - if (CloseHandle(tid.id) == 0) { + if (CloseHandle(tid->id) == 0) { windows_error = __wt_getlasterror(); __wt_errx(session, "thread join: CloseHandle: %s", __wt_formatmessage(session, windows_error)); return (__wt_map_windows_error(windows_error)); } - tid.created = false; return (0); } diff --git a/src/third_party/wiredtiger/src/os_win/os_time.c b/src/third_party/wiredtiger/src/os_win/os_time.c index 284d2a63931..f6cda08b03d 100644 --- a/src/third_party/wiredtiger/src/os_win/os_time.c +++ b/src/third_party/wiredtiger/src/os_win/os_time.c @@ -29,19 +29,16 @@ __wt_epoch_raw(WT_SESSION_IMPL *session, struct timespec *tsp) } /* - * localtime_r -- - * Return the current local time. + * __wt_localtime -- + * Return the current local broken-down time. */ -struct tm * -localtime_r(const time_t *timer, struct tm *result) +int +__wt_localtime(WT_SESSION_IMPL *session, const time_t *timep, struct tm *result) { errno_t err; - err = localtime_s(result, timer); - if (err != 0) { - __wt_err(NULL, err, "localtime_s"); - return (NULL); - } + if ((err = localtime_s(result, timep)) == 0) + return (0); - return (result); + WT_RET_MSG(session, err, "localtime_s"); } diff --git a/src/third_party/wiredtiger/src/packing/pack_stream.c b/src/third_party/wiredtiger/src/packing/pack_stream.c index 80dfe906bdf..fababff7dea 100644 --- a/src/third_party/wiredtiger/src/packing/pack_stream.c +++ b/src/third_party/wiredtiger/src/packing/pack_stream.c @@ -95,7 +95,7 @@ wiredtiger_pack_item(WT_PACK_STREAM *ps, WT_ITEM *item) WT_RET(__pack_write( session, &pv, &ps->p, (size_t)(ps->end - ps->p))); break; - WT_ILLEGAL_VALUE(session); + WT_ILLEGAL_VALUE(session, pv.type); } return (0); @@ -128,7 +128,7 @@ wiredtiger_pack_int(WT_PACK_STREAM *ps, int64_t i) WT_RET(__pack_write( session, &pv, &ps->p, (size_t)(ps->end - ps->p))); break; - WT_ILLEGAL_VALUE(session); + WT_ILLEGAL_VALUE(session, pv.type); } return (0); @@ -158,7 +158,7 @@ wiredtiger_pack_str(WT_PACK_STREAM *ps, const char *s) WT_RET(__pack_write( session, &pv, &ps->p, (size_t)(ps->end - ps->p))); break; - WT_ILLEGAL_VALUE(session); + WT_ILLEGAL_VALUE(session, pv.type); } return (0); @@ -194,7 +194,7 @@ wiredtiger_pack_uint(WT_PACK_STREAM *ps, uint64_t u) WT_RET(__pack_write( session, &pv, &ps->p, (size_t)(ps->end - ps->p))); break; - WT_ILLEGAL_VALUE(session); + WT_ILLEGAL_VALUE(session, pv.type); } return (0); @@ -225,7 +225,7 @@ wiredtiger_unpack_item(WT_PACK_STREAM *ps, WT_ITEM *item) item->data = pv.u.item.data; item->size = pv.u.item.size; break; - WT_ILLEGAL_VALUE(session); + WT_ILLEGAL_VALUE(session, pv.type); } return (0); @@ -258,7 +258,7 @@ wiredtiger_unpack_int(WT_PACK_STREAM *ps, int64_t *ip) &pv, (const uint8_t **)&ps->p, (size_t)(ps->end - ps->p))); *ip = pv.u.i; break; - WT_ILLEGAL_VALUE(session); + WT_ILLEGAL_VALUE(session, pv.type); } return (0); } @@ -287,7 +287,7 @@ wiredtiger_unpack_str(WT_PACK_STREAM *ps, const char **sp) &pv, (const uint8_t **)&ps->p, (size_t)(ps->end - ps->p))); *sp = pv.u.s; break; - WT_ILLEGAL_VALUE(session); + WT_ILLEGAL_VALUE(session, pv.type); } return (0); } @@ -322,7 +322,7 @@ wiredtiger_unpack_uint(WT_PACK_STREAM *ps, uint64_t *up) &pv, (const uint8_t **)&ps->p, (size_t)(ps->end - ps->p))); *up = pv.u.u; break; - WT_ILLEGAL_VALUE(session); + WT_ILLEGAL_VALUE(session, pv.type); } return (0); } diff --git a/src/third_party/wiredtiger/src/reconcile/rec_write.c b/src/third_party/wiredtiger/src/reconcile/rec_write.c index 2ec28e31201..a38b2ed0f02 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_write.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_write.c @@ -322,7 +322,6 @@ static int __rec_init(WT_SESSION_IMPL *, WT_REF *, uint32_t, WT_SALVAGE_COOKIE *, void *); static int __rec_las_wrapup(WT_SESSION_IMPL *, WT_RECONCILE *); static int __rec_las_wrapup_err(WT_SESSION_IMPL *, WT_RECONCILE *); -static uint32_t __rec_min_split_page_size(WT_BTREE *, uint32_t); static int __rec_root_write(WT_SESSION_IMPL *, WT_PAGE *, uint32_t); static int __rec_row_int(WT_SESSION_IMPL *, WT_RECONCILE *, WT_PAGE *); static int __rec_row_leaf(WT_SESSION_IMPL *, @@ -420,7 +419,7 @@ __wt_reconcile(WT_SESSION_IMPL *session, WT_REF *ref, if (LF_ISSET(WT_REC_EVICT) && !__wt_page_can_evict(session, ref, NULL)) { WT_PAGE_UNLOCK(session, page); - return (EBUSY); + return (__wt_set_return(session, EBUSY)); } /* Initialize the reconciliation structure for each new run. */ @@ -442,8 +441,9 @@ __wt_reconcile(WT_SESSION_IMPL *session, WT_REF *ref, if (LF_ISSET(WT_REC_EVICT)) { mod->last_eviction_id = oldest_id; #ifdef HAVE_TIMESTAMPS - __wt_txn_pinned_timestamp( - session, &mod->last_eviction_timestamp); + if (S2C(session)->txn_global.has_pinned_timestamp) + __wt_txn_pinned_timestamp( + session, &mod->last_eviction_timestamp); #endif mod->last_evict_pass_gen = S2C(session)->cache->evict_pass_gen; } @@ -480,7 +480,9 @@ __wt_reconcile(WT_SESSION_IMPL *session, WT_REF *ref, case WT_PAGE_ROW_LEAF: ret = __rec_row_leaf(session, r, page, salvage); break; - WT_ILLEGAL_VALUE_SET(session); + default: + ret = __wt_illegal_value(session, page->type); + break; } /* @@ -651,7 +653,7 @@ __rec_write_check_complete( return (0); *lookaside_retryp = true; - return (EBUSY); + return (__wt_set_return(session, EBUSY)); } /* @@ -767,7 +769,7 @@ __rec_root_write(WT_SESSION_IMPL *session, WT_PAGE *page, uint32_t flags) return (0); case WT_PM_REC_MULTIBLOCK: /* Multiple blocks */ break; - WT_ILLEGAL_VALUE(session); + WT_ILLEGAL_VALUE(session, mod->rec_result); } __wt_verbose(session, WT_VERB_SPLIT, @@ -1397,7 +1399,7 @@ __rec_txn_read(WT_SESSION_IMPL *session, WT_RECONCILE *r, if (F_ISSET(r, WT_REC_UPDATE_RESTORE) && *updp != NULL && uncommitted) { r->leave_dirty = true; - return (EBUSY); + return (__wt_set_return(session, EBUSY)); } if (upd->type == WT_UPDATE_BIRTHMARK) @@ -1517,9 +1519,9 @@ __rec_txn_read(WT_SESSION_IMPL *session, WT_RECONCILE *r, * the WT_REC_LOOKASIDE flag. */ if (!F_ISSET(r, WT_REC_LOOKASIDE | WT_REC_UPDATE_RESTORE)) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); if (uncommitted && !F_ISSET(r, WT_REC_UPDATE_RESTORE)) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); WT_ASSERT(session, r->max_txn != WT_TXN_NONE); @@ -1704,7 +1706,7 @@ __rec_child_deleted(WT_SESSION_IMPL *session, * the original page and which should see the deleted page). */ if (F_ISSET(r, WT_REC_EVICT)) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); /* * If there are deleted child pages we can't discard immediately, keep @@ -1789,7 +1791,7 @@ __rec_child_modify(WT_SESSION_IMPL *session, */ WT_ASSERT(session, !F_ISSET(r, WT_REC_EVICT)); if (F_ISSET(r, WT_REC_EVICT)) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); /* * If called during checkpoint, the child is being @@ -1817,7 +1819,7 @@ __rec_child_modify(WT_SESSION_IMPL *session, if (F_ISSET(r, WT_REC_EVICT) && __wt_page_las_active(session, ref)) { WT_ASSERT(session, false); - return (EBUSY); + return (__wt_set_return(session, EBUSY)); } /* @@ -1842,7 +1844,7 @@ __rec_child_modify(WT_SESSION_IMPL *session, */ WT_ASSERT(session, !F_ISSET(r, WT_REC_EVICT)); if (F_ISSET(r, WT_REC_EVICT)) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); /* * If called during checkpoint, acquire a hazard pointer @@ -1877,7 +1879,7 @@ __rec_child_modify(WT_SESSION_IMPL *session, */ WT_ASSERT(session, !F_ISSET(r, WT_REC_EVICT)); if (F_ISSET(r, WT_REC_EVICT)) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); goto done; case WT_REF_SPLIT: @@ -1894,9 +1896,9 @@ __rec_child_modify(WT_SESSION_IMPL *session, * for checkpoint. */ WT_ASSERT(session, WT_REF_SPLIT != WT_REF_SPLIT); - return (EBUSY); + return (__wt_set_return(session, EBUSY)); - WT_ILLEGAL_VALUE(session); + WT_ILLEGAL_VALUE(session, r->tested_ref_state); } WT_STAT_CONN_INCR(session, child_modify_blocked_page); } @@ -2203,12 +2205,12 @@ __rec_need_split(WT_RECONCILE *r, size_t len) } /* - * __rec_split_page_size_from_pct -- + * __wt_split_page_size -- * Given a split percentage, calculate split page size in bytes. */ -static uint32_t -__rec_split_page_size_from_pct( - int split_pct, uint32_t maxpagesize, uint32_t allocsize) { +uint32_t +__wt_split_page_size(int split_pct, uint32_t maxpagesize, uint32_t allocsize) +{ uintmax_t a; uint32_t split_size; @@ -2236,32 +2238,6 @@ __rec_split_page_size_from_pct( } /* - * __wt_split_page_size -- - * Split page size calculation: we don't want to repeatedly split every - * time a new entry is added, so we split to a smaller-than-maximum page - * size. - */ -uint32_t -__wt_split_page_size(WT_BTREE *btree, uint32_t maxpagesize) -{ - return (__rec_split_page_size_from_pct( - btree->split_pct, maxpagesize, btree->allocsize)); -} - -/* - * __rec_min_split_page_size -- - * Minimum split size boundary calculation: To track a boundary at the - * minimum split size that we could have split at instead of splitting at - * the split page size. - */ -static uint32_t -__rec_min_split_page_size(WT_BTREE *btree, uint32_t maxpagesize) -{ - return (__rec_split_page_size_from_pct( - WT_BTREE_MIN_SPLIT_PCT, maxpagesize, btree->allocsize)); -} - -/* * __rec_split_chunk_init -- * Initialize a single chunk structure. */ @@ -2306,7 +2282,7 @@ __rec_split_chunk_init( */ static int __rec_split_init(WT_SESSION_IMPL *session, - WT_RECONCILE *r, WT_PAGE *page, uint64_t recno, uint32_t max) + WT_RECONCILE *r, WT_PAGE *page, uint64_t recno, uint64_t max) { WT_BM *bm; WT_BTREE *btree; @@ -2342,7 +2318,7 @@ __rec_split_init(WT_SESSION_IMPL *session, * records, in those cases we split the pages once they have crossed * the maximum size for a page with raw compression. */ - r->page_size = r->page_size_orig = max; + r->page_size = r->page_size_orig = (uint32_t)max; if (r->raw_compression) r->max_raw_page_size = r->page_size = (uint32_t)WT_MIN((uint64_t)r->page_size * 10, @@ -2392,11 +2368,12 @@ __rec_split_init(WT_SESSION_IMPL *session, r->space_avail = r->split_size - WT_PAGE_HEADER_BYTE_SIZE(btree); } else { - r->split_size = __wt_split_page_size(btree, r->page_size); + r->split_size = __wt_split_page_size( + btree->split_pct, r->page_size, btree->allocsize); r->space_avail = r->split_size - WT_PAGE_HEADER_BYTE_SIZE(btree); - r->min_split_size = - __rec_min_split_page_size(btree, r->page_size); + r->min_split_size = __wt_split_page_size( + WT_BTREE_MIN_SPLIT_PCT, r->page_size, btree->allocsize); r->min_space_avail = r->min_split_size - WT_PAGE_HEADER_BYTE_SIZE(btree); } @@ -2405,13 +2382,17 @@ __rec_split_init(WT_SESSION_IMPL *session, * Ensure the disk image buffer is large enough for the max object, as * corrected by the underlying block manager. * - * Since we want to support split_size more than the page size (to allow - * for adjustments based on the compression), this buffer should be the - * greater of split_size and page_size. + * Since we want to support split_size values larger than the page size + * (to allow for adjustments based on the compression), this buffer + * should be the greater of split_size and page_size, then aligned to + * the next allocation size boundary. The latter shouldn't be an issue, + * but it's a possible scenario if, for example, the compression engine + * is expected to give us 5x compression and gives us nothing at all. */ corrected_page_size = r->page_size; WT_RET(bm->write_size(bm, session, &corrected_page_size)); - disk_img_buf_size = WT_MAX(corrected_page_size, r->split_size); + disk_img_buf_size = WT_ALIGN( + WT_MAX(corrected_page_size, r->split_size), btree->allocsize); /* Initialize the first split chunk. */ WT_RET( @@ -2919,7 +2900,7 @@ __rec_split_raw(WT_SESSION_IMPL *session, } r->raw_entries[slots] = entry; continue; - WT_ILLEGAL_VALUE(session); + WT_ILLEGAL_VALUE(session, unpack->type); } /* @@ -3597,6 +3578,85 @@ __rec_split_write_reuse(WT_SESSION_IMPL *session, } /* + * __rec_compression_adjust -- + * Adjust the pre-compression page size based on compression results. + */ +static inline void +__rec_compression_adjust(WT_SESSION_IMPL *session, + uint32_t max, size_t compressed_size, bool last_block, uint64_t *adjustp) +{ + WT_BTREE *btree; + uint64_t adjust, current, new; + u_int ten_percent; + + btree = S2BT(session); + ten_percent = max / 10; + + /* + * Changing the pre-compression size updates a shared memory location + * and it's not uncommon to be pushing out large numbers of pages from + * the same file. If compression creates a page larger than the target + * size, decrease the pre-compression size. If compression creates a + * page smaller than the target size, increase the pre-compression size. + * Once we get under the target size, try and stay there to minimize + * shared memory updates, but don't go over the target size, that means + * we're writing bad page sizes. + * Writing a shared memory location without a lock and letting it + * race, minor trickiness so we only read and write the value once. + */ + WT_ORDERED_READ(current, *adjustp); + WT_ASSERT(session, current >= max); + + if (compressed_size > max) { + /* + * The compressed size is GT the page maximum. + * Check if the pre-compression size is larger than the maximum. + * If 10% of the page size larger than the maximum, decrease it + * by that amount. Else if it's not already at the page maximum, + * set it there. + * + * Note we're using 10% of the maximum page size as our test for + * when to adjust the pre-compression size as well as the amount + * by which we adjust it. Not updating the value when it's close + * to the page size keeps us from constantly updating a shared + * memory location, and 10% of the page size is an OK step value + * as well, so we use it in both cases. + */ + adjust = current - max; + if (adjust > ten_percent) + new = current - ten_percent; + else if (adjust != 0) + new = max; + else + return; + } else { + /* + * The compressed size is LTE the page maximum. + * + * Don't increase the pre-compressed size on the last block, the + * last block might be tiny. + * + * If the compressed size is less than the page maximum by 10%, + * increase the pre-compression size by 10% of the page, or up + * to the maximum in-memory image size. + * + * Note we're using 10% of the maximum page size... see above. + */ + if (last_block || compressed_size > max - ten_percent) + return; + + adjust = current + ten_percent; + if (adjust < btree->maxmempage_image) + new = adjust; + else if (current != btree->maxmempage_image) + new = btree->maxmempage_image; + else + return; + } + *adjustp = new; +} + +/* * __rec_split_write -- * Write a disk block out for the split helper functions. */ @@ -3607,7 +3667,7 @@ __rec_split_write(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_BTREE *btree; WT_MULTI *multi; WT_PAGE *page; - size_t addr_size; + size_t addr_size, compressed_size; uint8_t addr[WT_BTREE_MAX_ADDR_COOKIE]; #ifdef HAVE_DIAGNOSTIC bool verify_image; @@ -3638,7 +3698,7 @@ __rec_split_write(WT_SESSION_IMPL *session, WT_RECONCILE *r, case WT_PAGE_ROW_INT: multi->addr.type = WT_ADDR_INT; break; - WT_ILLEGAL_VALUE(session); + WT_ILLEGAL_VALUE(session, page->type); } multi->size = WT_STORE_SIZE(chunk->image.size); multi->checksum = 0; @@ -3704,7 +3764,7 @@ __rec_split_write(WT_SESSION_IMPL *session, WT_RECONCILE *r, * allocate a zero-length array. */ if (r->page->type != WT_PAGE_ROW_LEAF && chunk->entries == 0) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); if (F_ISSET(r, WT_REC_LOOKASIDE)) { r->cache_write_lookaside = true; @@ -3740,14 +3800,24 @@ __rec_split_write(WT_SESSION_IMPL *session, WT_RECONCILE *r, /* Write the disk image and get an address. */ WT_RET(__wt_bt_write(session, compressed_image == NULL ? &chunk->image : compressed_image, - addr, &addr_size, false, F_ISSET(r, WT_REC_CHECKPOINT), - compressed_image != NULL)); + addr, &addr_size, &compressed_size, + false, F_ISSET(r, WT_REC_CHECKPOINT), compressed_image != NULL)); #ifdef HAVE_DIAGNOSTIC verify_image = false; #endif WT_RET(__wt_memdup(session, addr, addr_size, &multi->addr.addr)); multi->addr.size = (uint8_t)addr_size; + /* Adjust the pre-compression page size based on compression results. */ + if (WT_PAGE_IS_INTERNAL(page) && + compressed_size != 0 && btree->intlpage_compadjust) + __rec_compression_adjust(session, btree->maxintlpage, + compressed_size, last_block, &btree->maxintlpage_precomp); + if (!WT_PAGE_IS_INTERNAL(page) && + compressed_size != 0 && btree->leafpage_compadjust) + __rec_compression_adjust(session, btree->maxleafpage, + compressed_size, last_block, &btree->maxleafpage_precomp); + copy_image: #ifdef HAVE_DIAGNOSTIC /* @@ -3808,8 +3878,8 @@ __wt_bulk_init(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk) recno = btree->type == BTREE_ROW ? WT_RECNO_OOB : 1; - return (__rec_split_init( - session, r, cbulk->leaf, recno, btree->maxleafpage)); + return (__rec_split_init(session, + r, cbulk->leaf, recno, btree->maxleafpage_precomp)); } /* @@ -4107,8 +4177,8 @@ __rec_col_int(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_REF *pageref) val = &r->v; vpack = &_vpack; - WT_RET(__rec_split_init( - session, r, page, pageref->ref_recno, btree->maxintlpage)); + WT_RET(__rec_split_init(session, + r, page, pageref->ref_recno, btree->maxintlpage_precomp)); /* For each entry in the in-memory page... */ WT_INTL_FOREACH_BEGIN(session, page, ref) { @@ -4152,7 +4222,8 @@ __rec_col_int(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_REF *pageref) case WT_PM_REC_REPLACE: addr = &child->modify->mod_replace; break; - WT_ILLEGAL_VALUE_ERR(session); + WT_ILLEGAL_VALUE_ERR( + session, child->modify->rec_result); } break; case WT_CHILD_ORIGINAL: @@ -4160,11 +4231,10 @@ __rec_col_int(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_REF *pageref) break; case WT_CHILD_PROXY: /* - * Deleted child where we write a proxy cell, not - * yet supported for column-store. + * Deleted child where we write a proxy cell, not yet + * supported for column-store. */ - ret = __wt_illegal_value(session, NULL); - goto err; + WT_ERR(__wt_illegal_value(session, state)); } /* @@ -4563,8 +4633,8 @@ __rec_col_var(WT_SESSION_IMPL *session, vpack = &_vpack; cbt = &r->update_modify_cbt; - WT_RET(__rec_split_init( - session, r, page, pageref->ref_recno, btree->maxleafpage)); + WT_RET(__rec_split_init(session, + r, page, pageref->ref_recno, btree->maxleafpage_precomp)); WT_RET(__wt_scr_alloc(session, 0, &orig)); data = NULL; @@ -4703,7 +4773,7 @@ record_loop: /* case WT_UPDATE_TOMBSTONE: deleted = true; break; - WT_ILLEGAL_VALUE_ERR(session); + WT_ILLEGAL_VALUE_ERR(session, upd->type); } } else if (vpack->raw == WT_CELL_VALUE_OVFL_RM) { /* @@ -4948,7 +5018,7 @@ compare: /* case WT_UPDATE_TOMBSTONE: deleted = true; break; - WT_ILLEGAL_VALUE_ERR(session); + WT_ILLEGAL_VALUE_ERR(session, upd->type); } /* @@ -5056,7 +5126,8 @@ __rec_row_int(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) cell = NULL; key_onpage_ovfl = false; - WT_RET(__rec_split_init(session, r, page, 0, btree->maxintlpage)); + WT_RET(__rec_split_init( + session, r, page, 0, btree->maxintlpage_precomp)); /* * Ideally, we'd never store the 0th key on row-store internal pages @@ -5166,7 +5237,8 @@ __rec_row_int(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) */ addr = &child->modify->mod_replace; break; - WT_ILLEGAL_VALUE_ERR(session); + WT_ILLEGAL_VALUE_ERR( + session, child->modify->rec_result); } break; case WT_CHILD_ORIGINAL: @@ -5341,7 +5413,8 @@ __rec_row_leaf(WT_SESSION_IMPL *session, val = &r->v; vpack = &_vpack; - WT_RET(__rec_split_init(session, r, page, 0, btree->maxleafpage)); + WT_RET(__rec_split_init( + session, r, page, 0, btree->maxleafpage_precomp)); /* * Write any K/V pairs inserted into the page before the first from-disk @@ -5551,7 +5624,7 @@ __rec_row_leaf(WT_SESSION_IMPL *session, /* Proceed with appended key/value pairs. */ goto leaf_insert; - WT_ILLEGAL_VALUE_ERR(session); + WT_ILLEGAL_VALUE_ERR(session, upd->type); } } @@ -5761,7 +5834,7 @@ __rec_row_leaf_insert(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins) break; case WT_UPDATE_TOMBSTONE: continue; - WT_ILLEGAL_VALUE(session); + WT_ILLEGAL_VALUE(session, upd->type); } /* Build key cell. */ @@ -5971,7 +6044,7 @@ __rec_write_wrapup(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) mod->mod_replace.size = 0; __wt_free(session, mod->mod_disk_image); break; - WT_ILLEGAL_VALUE(session); + WT_ILLEGAL_VALUE(session, mod->rec_result); } /* Reset the reconciliation state. */ @@ -6048,7 +6121,8 @@ __rec_write_wrapup(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) mod->mod_page_las = r->multi->page_las; } else WT_RET(__wt_bt_write(session, r->wrapup_checkpoint, - NULL, NULL, true, F_ISSET(r, WT_REC_CHECKPOINT), + NULL, NULL, NULL, + true, F_ISSET(r, WT_REC_CHECKPOINT), r->wrapup_checkpoint_compressed)); mod->rec_result = WT_PM_REC_REPLACE; @@ -6470,8 +6544,8 @@ __rec_cell_build_ovfl(WT_SESSION_IMPL *session, /* Write the buffer. */ addr = buf; - WT_ERR(__wt_bt_write(session, tmp, - addr, &size, false, F_ISSET(r, WT_REC_CHECKPOINT), false)); + WT_ERR(__wt_bt_write(session, tmp, addr, &size, NULL, + false, F_ISSET(r, WT_REC_CHECKPOINT), false)); /* * Track the overflow record (unless it's a bulk load, which diff --git a/src/third_party/wiredtiger/src/schema/schema_alter.c b/src/third_party/wiredtiger/src/schema/schema_alter.c index e880cb415c8..aba708f0e0b 100644 --- a/src/third_party/wiredtiger/src/schema/schema_alter.c +++ b/src/third_party/wiredtiger/src/schema/schema_alter.c @@ -53,7 +53,7 @@ err: __wt_free(session, config); * there was no metadata entry. */ if (ret == WT_NOTFOUND) - ret = ENOENT; + ret = __wt_set_return(session, ENOENT); return (ret); } diff --git a/src/third_party/wiredtiger/src/schema/schema_list.c b/src/third_party/wiredtiger/src/schema/schema_list.c index 0f8806dd462..990f2636bf9 100644 --- a/src/third_party/wiredtiger/src/schema/schema_list.c +++ b/src/third_party/wiredtiger/src/schema/schema_list.c @@ -28,7 +28,7 @@ __wt_schema_get_table_uri(WT_SESSION_IMPL *session, WT_ERR(__wt_session_get_dhandle(session, uri, NULL, NULL, flags)); table = (WT_TABLE *)session->dhandle; if (!ok_incomplete && !table->cg_complete) { - ret = EINVAL; + ret = __wt_set_return(session, EINVAL); WT_TRET(__wt_session_release_dhandle(session)); WT_ERR_MSG(session, ret, "'%s' cannot be used " "until all column groups are created", diff --git a/src/third_party/wiredtiger/src/schema/schema_plan.c b/src/third_party/wiredtiger/src/schema/schema_plan.c index abe76013e12..7fb2a391784 100644 --- a/src/third_party/wiredtiger/src/schema/schema_plan.c +++ b/src/third_party/wiredtiger/src/schema/schema_plan.c @@ -298,7 +298,7 @@ __find_column_format(WT_SESSION_IMPL *session, WT_TABLE *table, if (k.len == colname->len && strncmp(colname->str, k.str, k.len) == 0) { if (value_only && inkey) - return (EINVAL); + return (__wt_set_return(session, EINVAL)); return (0); } } diff --git a/src/third_party/wiredtiger/src/schema/schema_util.c b/src/third_party/wiredtiger/src/schema/schema_util.c index ceec6db6cb5..a281ec3fe12 100644 --- a/src/third_party/wiredtiger/src/schema/schema_util.c +++ b/src/third_party/wiredtiger/src/schema/schema_util.c @@ -39,7 +39,7 @@ __wt_schema_backup_check(WT_SESSION_IMPL *session, const char *name) } for (i = 0; backup_list[i] != NULL; ++i) { if (strcmp(backup_list[i], name) == 0) { - ret = EBUSY; + ret = __wt_set_return(session, EBUSY); break; } } diff --git a/src/third_party/wiredtiger/src/session/session_dhandle.c b/src/third_party/wiredtiger/src/session/session_dhandle.c index caa775686cf..30399cafd22 100644 --- a/src/third_party/wiredtiger/src/session/session_dhandle.c +++ b/src/third_party/wiredtiger/src/session/session_dhandle.c @@ -140,7 +140,7 @@ __wt_session_lock_dhandle( if (!LF_ISSET(WT_DHANDLE_LOCK_ONLY) && (!F_ISSET(dhandle, WT_DHANDLE_OPEN) || (btree != NULL && F_ISSET(btree, WT_BTREE_SPECIAL_FLAGS)))) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); ++dhandle->excl_ref; return (0); } @@ -167,7 +167,7 @@ __wt_session_lock_dhandle( * give up. */ if (btree != NULL && F_ISSET(btree, WT_BTREE_SPECIAL_FLAGS)) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); /* * If the handle is open, get a read lock and recheck. diff --git a/src/third_party/wiredtiger/src/support/err.c b/src/third_party/wiredtiger/src/support/err.c index 0569d0545e6..1c6cd8e50ae 100644 --- a/src/third_party/wiredtiger/src/support/err.c +++ b/src/third_party/wiredtiger/src/support/err.c @@ -174,7 +174,7 @@ __wt_event_handler_set(WT_SESSION_IMPL *session, WT_EVENT_HANDLER *handler) */ static int __eventv(WT_SESSION_IMPL *session, bool msg_event, int error, - const char *file_name, int line_number, const char *fmt, va_list ap) + const char *func_name, int line_number, const char *fmt, va_list ap) WT_GCC_FUNC_ATTRIBUTE((cold)) { struct timespec ts; @@ -231,8 +231,8 @@ __eventv(WT_SESSION_IMPL *session, bool msg_event, int error, WT_ERROR_APPEND(p, remain, ", %s", prefix); WT_ERROR_APPEND(p, remain, ": "); - if (file_name != NULL) - WT_ERROR_APPEND(p, remain, "%s, %d: ", file_name, line_number); + if (func_name != NULL) + WT_ERROR_APPEND(p, remain, "%s, %d: ", func_name, line_number); WT_ERROR_APPEND_AP(p, remain, fmt, ap); @@ -309,13 +309,14 @@ err: if (fprintf(stderr, } /* - * __wt_err -- + * __wt_err_func -- * Report an error. */ void -__wt_err(WT_SESSION_IMPL *session, int error, const char *fmt, ...) +__wt_err_func(WT_SESSION_IMPL *session, + int error, const char *func_name, int line_number, const char *fmt, ...) WT_GCC_FUNC_ATTRIBUTE((cold)) - WT_GCC_FUNC_ATTRIBUTE((format (printf, 3, 4))) + WT_GCC_FUNC_ATTRIBUTE((format (printf, 5, 6))) WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { va_list ap; @@ -325,18 +326,21 @@ __wt_err(WT_SESSION_IMPL *session, int error, const char *fmt, ...) * an error value to return. */ va_start(ap, fmt); - WT_IGNORE_RET(__eventv(session, false, error, NULL, 0, fmt, ap)); + WT_IGNORE_RET(__eventv(session, + false, error, func_name, line_number, fmt, ap)); va_end(ap); } /* - * __wt_errx -- + * __wt_errx_func -- * Report an error with no error code. */ void -__wt_errx(WT_SESSION_IMPL *session, const char *fmt, ...) +__wt_errx_func(WT_SESSION_IMPL *session, + const char *func_name, int line_number, const char *fmt, ...) WT_GCC_FUNC_ATTRIBUTE((cold)) - WT_GCC_FUNC_ATTRIBUTE((format (printf, 2, 3))) + WT_GCC_FUNC_ATTRIBUTE((format (printf, 4, 5))) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { va_list ap; @@ -345,11 +349,25 @@ __wt_errx(WT_SESSION_IMPL *session, const char *fmt, ...) * an error value to return. */ va_start(ap, fmt); - WT_IGNORE_RET(__eventv(session, false, 0, NULL, 0, fmt, ap)); + WT_IGNORE_RET(__eventv(session, + false, 0, func_name, line_number, fmt, ap)); va_end(ap); } /* + * __wt_set_return_func -- + * Conditionally log the source of an error code and return the error. + */ +int +__wt_set_return_func( + WT_SESSION_IMPL *session, const char* func, int line, int err) +{ + __wt_verbose(session, + WT_VERB_ERROR_RETURNS, "%s: %d Error: %d", func, line, err); + return (err); +} + +/* * __wt_ext_err_printf -- * Extension API call to print to the error stream. */ @@ -388,44 +406,28 @@ __wt_verbose_worker(WT_SESSION_IMPL *session, const char *fmt, ...) } /* - * info_msg -- + * __wt_msg -- * Informational message. */ -static int -info_msg(WT_SESSION_IMPL *session, const char *fmt, va_list ap) +int +__wt_msg(WT_SESSION_IMPL *session, const char *fmt, ...) + WT_GCC_FUNC_ATTRIBUTE((cold)) + WT_GCC_FUNC_ATTRIBUTE((format (printf, 2, 3))) { + WT_DECL_ITEM(buf); + WT_DECL_RET; WT_EVENT_HANDLER *handler; WT_SESSION *wt_session; - /* - * !!! - * SECURITY: - * Buffer placed at the end of the stack in case snprintf overflows. - */ - char s[2048]; + WT_RET(__wt_scr_alloc(session, 0, &buf)); - WT_RET(__wt_vsnprintf(s, sizeof(s), fmt, ap)); + WT_VA_ARGS_BUF_FORMAT(session, buf, fmt, false); wt_session = (WT_SESSION *)session; handler = session->event_handler; - return (handler->handle_message(handler, wt_session, s)); -} - -/* - * __wt_msg -- - * Informational message. - */ -int -__wt_msg(WT_SESSION_IMPL *session, const char *fmt, ...) - WT_GCC_FUNC_ATTRIBUTE((cold)) - WT_GCC_FUNC_ATTRIBUTE((format (printf, 2, 3))) -{ - WT_DECL_RET; - va_list ap; + ret = handler->handle_message(handler, wt_session, buf->data); - va_start(ap, fmt); - ret = info_msg(session, fmt, ap); - va_end(ap); + __wt_scr_free(session, &buf); return (ret); } @@ -439,16 +441,24 @@ __wt_ext_msg_printf( WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *fmt, ...) WT_GCC_FUNC_ATTRIBUTE((format (printf, 3, 4))) { + WT_DECL_ITEM(buf); WT_DECL_RET; + WT_EVENT_HANDLER *handler; WT_SESSION_IMPL *session; - va_list ap; if ((session = (WT_SESSION_IMPL *)wt_session) == NULL) session = ((WT_CONNECTION_IMPL *)wt_api->conn)->default_session; - va_start(ap, fmt); - ret = info_msg(session, fmt, ap); - va_end(ap); + WT_RET(__wt_scr_alloc(session, 0, &buf)); + + WT_VA_ARGS_BUF_FORMAT(session, buf, fmt, false); + + wt_session = (WT_SESSION *)session; + handler = session->event_handler; + ret = handler->handle_message(handler, wt_session, buf->data); + + __wt_scr_free(session, &buf); + return (ret); } @@ -487,34 +497,6 @@ __wt_progress(WT_SESSION_IMPL *session, const char *s, uint64_t v) } /* - * __wt_assert -- - * Assert and other unexpected failures, includes file/line information - * for debugging. - */ -void -__wt_assert(WT_SESSION_IMPL *session, - int error, const char *file_name, int line_number, const char *fmt, ...) - WT_GCC_FUNC_ATTRIBUTE((cold)) - WT_GCC_FUNC_ATTRIBUTE((format (printf, 5, 6))) -#ifdef HAVE_DIAGNOSTIC - WT_GCC_FUNC_ATTRIBUTE((noreturn)) -#endif - WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) -{ - va_list ap; - - va_start(ap, fmt); - WT_IGNORE_RET(__eventv( - session, false, error, file_name, line_number, fmt, ap)); - va_end(ap); - -#ifdef HAVE_DIAGNOSTIC - __wt_abort(session); /* Drop core if testing. */ - /* NOTREACHED */ -#endif -} - -/* * __wt_panic -- * A standard error message when we panic. */ @@ -565,16 +547,13 @@ __wt_panic(WT_SESSION_IMPL *session) */ int __wt_illegal_value_func( - WT_SESSION_IMPL *session, const char *tag, const char *file, int line) + WT_SESSION_IMPL *session, uintmax_t v, const char *func, int line) WT_GCC_FUNC_ATTRIBUTE((cold)) WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { - __wt_errx(session, "%s%s%s: (%s, %d)", - tag == NULL ? "" : tag, - tag == NULL ? "" : ": ", - "encountered an illegal file format or internal value", - file, line); - + __wt_err_func(session, EINVAL, + func, line, "%s: 0x%" PRIxMAX, + "encountered an illegal file format or internal value", v); return (__wt_panic(session)); } diff --git a/src/third_party/wiredtiger/src/support/global.c b/src/third_party/wiredtiger/src/support/global.c index d1271e0d427..f71f91a4daa 100644 --- a/src/third_party/wiredtiger/src/support/global.c +++ b/src/third_party/wiredtiger/src/support/global.c @@ -117,7 +117,7 @@ __wt_global_once(void) return; } - __wt_checksum_init(); + __wt_process.checksum = wiredtiger_crc32c_func(); __global_calibrate_ticks(); TAILQ_INIT(&__wt_process.connqh); diff --git a/src/third_party/wiredtiger/src/support/hazard.c b/src/third_party/wiredtiger/src/support/hazard.c index 815c876f444..4116638e31c 100644 --- a/src/third_party/wiredtiger/src/support/hazard.c +++ b/src/third_party/wiredtiger/src/support/hazard.c @@ -401,6 +401,29 @@ __wt_hazard_count(WT_SESSION_IMPL *session, WT_REF *ref) #ifdef HAVE_DIAGNOSTIC /* + * __wt_hazard_check_assert -- + * Assert there's no hazard pointer to the page. + */ +bool +__wt_hazard_check_assert(WT_SESSION_IMPL *session, void *ref, bool waitfor) +{ + WT_HAZARD *hp; + int i; + + for (i = 0;;) { + if ((hp = __wt_hazard_check(session, ref)) == NULL) + return (true); + if (!waitfor || ++i > 100) + break; + __wt_sleep(0, 10000); + } + __wt_errx(session, + "hazard pointer reference to discarded object: (%p: %s, line %d)", + (void *)hp->ref, hp->file, hp->line); + return (false); +} + +/* * __hazard_dump -- * Display the list of hazard pointers. */ diff --git a/src/third_party/wiredtiger/src/support/mtx_rw.c b/src/third_party/wiredtiger/src/support/mtx_rw.c index fd66a1a40bb..959405dee50 100644 --- a/src/third_party/wiredtiger/src/support/mtx_rw.c +++ b/src/third_party/wiredtiger/src/support/mtx_rw.c @@ -137,7 +137,7 @@ __wt_try_readlock(WT_SESSION_IMPL *session, WT_RWLOCK *l) /* This read lock can only be granted if there are no active writers. */ if (old.u.s.current != old.u.s.next) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); /* * The replacement lock value is a result of adding an active reader. @@ -146,7 +146,7 @@ __wt_try_readlock(WT_SESSION_IMPL *session, WT_RWLOCK *l) */ new.u.v = old.u.v; if (++new.u.s.readers_active == 0) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); /* We rely on this atomic operation to provide a barrier. */ return (__wt_atomic_casv64(&l->u.v, old.u.v, new.u.v) ? 0 : EBUSY); @@ -331,7 +331,7 @@ __wt_try_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *l) */ old.u.v = l->u.v; if (old.u.s.current != old.u.s.next || old.u.s.readers_active != 0) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); /* * We've checked above that there is no writer active (since diff --git a/src/third_party/wiredtiger/src/support/scratch.c b/src/third_party/wiredtiger/src/support/scratch.c index 2ead79a1c1c..21c83146a31 100644 --- a/src/third_party/wiredtiger/src/support/scratch.c +++ b/src/third_party/wiredtiger/src/support/scratch.c @@ -71,30 +71,9 @@ __wt_buf_fmt(WT_SESSION_IMPL *session, WT_ITEM *buf, const char *fmt, ...) WT_GCC_FUNC_ATTRIBUTE((format (printf, 3, 4))) WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { - WT_DECL_RET; - size_t len; - va_list ap; - - for (;;) { - va_start(ap, fmt); - ret = __wt_vsnprintf_len_set( - buf->mem, buf->memsize, &len, fmt, ap); - va_end(ap); - WT_RET(ret); - - /* Check if there was enough space. */ - if (len < buf->memsize) { - buf->data = buf->mem; - buf->size = len; - return (0); - } + WT_VA_ARGS_BUF_FORMAT(session, buf, fmt, false); - /* - * If not, double the size of the buffer: we're dealing with - * strings, and we don't expect these numbers to get huge. - */ - WT_RET(__wt_buf_extend(session, buf, len + 1)); - } + return (0); } /* @@ -106,11 +85,6 @@ __wt_buf_catfmt(WT_SESSION_IMPL *session, WT_ITEM *buf, const char *fmt, ...) WT_GCC_FUNC_ATTRIBUTE((format (printf, 3, 4))) WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { - WT_DECL_RET; - size_t len, space; - char *p; - va_list ap; - /* * If we're appending data to an existing buffer, any data field should * point into the allocated memory. (It wouldn't be insane to copy any @@ -119,27 +93,9 @@ __wt_buf_catfmt(WT_SESSION_IMPL *session, WT_ITEM *buf, const char *fmt, ...) */ WT_ASSERT(session, buf->data == NULL || WT_DATA_IN_ITEM(buf)); - for (;;) { - va_start(ap, fmt); - p = (char *)((uint8_t *)buf->mem + buf->size); - WT_ASSERT(session, buf->memsize >= buf->size); - space = buf->memsize - buf->size; - ret = __wt_vsnprintf_len_set(p, space, &len, fmt, ap); - va_end(ap); - WT_RET(ret); - - /* Check if there was enough space. */ - if (len < space) { - buf->size += len; - return (0); - } + WT_VA_ARGS_BUF_FORMAT(session, buf, fmt, true); - /* - * If not, double the size of the buffer: we're dealing with - * strings, and we don't expect these numbers to get huge. - */ - WT_RET(__wt_buf_extend(session, buf, buf->size + len + 1)); - } + return (0); } /* @@ -223,7 +179,7 @@ __wt_buf_set_printable_format(WT_SESSION_IMPL *session, session, buf, "%s%" PRIu64, sep, pv.u.u)); sep = ","; break; - WT_ILLEGAL_VALUE_ERR(session); + WT_ILLEGAL_VALUE_ERR(session, pv.type); } } WT_ERR_NOTFOUND_OK(ret); @@ -233,7 +189,7 @@ err: __wt_scr_free(session, &tmp); return ((const char *)buf->data); retp = "failed to create printable output"; - __wt_err(session, ret, "%s: %s", __func__, retp); + __wt_err(session, ret, "%s", retp); return (retp); } @@ -400,16 +356,19 @@ __wt_scr_discard(WT_SESSION_IMPL *session) if (*bufp == NULL) continue; if (F_ISSET(*bufp, WT_ITEM_INUSE)) +#ifdef HAVE_DIAGNOSTIC __wt_errx(session, "scratch buffer allocated and never discarded" -#ifdef HAVE_DIAGNOSTIC ": %s: %d", session-> scratch_track[bufp - session->scratch].file, session-> scratch_track[bufp - session->scratch].line -#endif ); +#else + __wt_errx(session, + "scratch buffer allocated and never discarded"); +#endif __wt_buf_free(session, *bufp); __wt_free(session, *bufp); diff --git a/src/third_party/wiredtiger/src/support/stat.c b/src/third_party/wiredtiger/src/support/stat.c index 0d39a5b682e..ca2ae861863 100644 --- a/src/third_party/wiredtiger/src/support/stat.c +++ b/src/third_party/wiredtiger/src/support/stat.c @@ -916,19 +916,19 @@ static const char * const __stats_connection_desc[] = { "lock: checkpoint lock acquisitions", "lock: checkpoint lock application thread wait time (usecs)", "lock: checkpoint lock internal thread wait time (usecs)", - "lock: commit timestamp queue lock application thread time waiting for the dhandle lock (usecs)", - "lock: commit timestamp queue lock internal thread time waiting for the dhandle lock (usecs)", + "lock: commit timestamp queue lock application thread time waiting (usecs)", + "lock: commit timestamp queue lock internal thread time waiting (usecs)", "lock: commit timestamp queue read lock acquisitions", "lock: commit timestamp queue write lock acquisitions", - "lock: dhandle lock application thread time waiting for the dhandle lock (usecs)", - "lock: dhandle lock internal thread time waiting for the dhandle lock (usecs)", + "lock: dhandle lock application thread time waiting (usecs)", + "lock: dhandle lock internal thread time waiting (usecs)", "lock: dhandle read lock acquisitions", "lock: dhandle write lock acquisitions", "lock: metadata lock acquisitions", "lock: metadata lock application thread wait time (usecs)", "lock: metadata lock internal thread wait time (usecs)", - "lock: read timestamp queue lock application thread time waiting for the dhandle lock (usecs)", - "lock: read timestamp queue lock internal thread time waiting for the dhandle lock (usecs)", + "lock: read timestamp queue lock application thread time waiting (usecs)", + "lock: read timestamp queue lock internal thread time waiting (usecs)", "lock: read timestamp queue read lock acquisitions", "lock: read timestamp queue write lock acquisitions", "lock: schema lock acquisitions", @@ -938,8 +938,8 @@ static const char * const __stats_connection_desc[] = { "lock: table lock internal thread time waiting for the table lock (usecs)", "lock: table read lock acquisitions", "lock: table write lock acquisitions", - "lock: txn global lock application thread time waiting for the dhandle lock (usecs)", - "lock: txn global lock internal thread time waiting for the dhandle lock (usecs)", + "lock: txn global lock application thread time waiting (usecs)", + "lock: txn global lock internal thread time waiting (usecs)", "lock: txn global read lock acquisitions", "lock: txn global write lock acquisitions", "log: busy returns attempting to switch slots", diff --git a/src/third_party/wiredtiger/src/support/thread_group.c b/src/third_party/wiredtiger/src/support/thread_group.c index 4597d26496d..9df1e3d636f 100644 --- a/src/third_party/wiredtiger/src/support/thread_group.c +++ b/src/third_party/wiredtiger/src/support/thread_group.c @@ -103,7 +103,7 @@ __thread_group_shrink( if (thread == NULL) continue; - WT_TRET(__wt_thread_join(session, thread->tid)); + WT_TRET(__wt_thread_join(session, &thread->tid)); __wt_cond_destroy(session, &thread->pause_cond); } __wt_writelock(session, &group->lock); diff --git a/src/third_party/wiredtiger/src/txn/txn_ckpt.c b/src/third_party/wiredtiger/src/txn/txn_ckpt.c index ad8351923a0..59d9fa2e7c1 100644 --- a/src/third_party/wiredtiger/src/txn/txn_ckpt.c +++ b/src/third_party/wiredtiger/src/txn/txn_ckpt.c @@ -1896,7 +1896,7 @@ __wt_checkpoint_close(WT_SESSION_IMPL *session, bool final) if (btree->modified && !bulk && S2C(session)->txn_global.has_stable_timestamp && !__wt_btree_immediately_durable(session)) - return (EBUSY); + return (__wt_set_return(session, EBUSY)); /* * Turn on metadata tracking if: diff --git a/src/third_party/wiredtiger/src/txn/txn_log.c b/src/third_party/wiredtiger/src/txn/txn_log.c index 81968c940f7..96a97203ecd 100644 --- a/src/third_party/wiredtiger/src/txn/txn_log.c +++ b/src/third_party/wiredtiger/src/txn/txn_log.c @@ -99,7 +99,7 @@ __txn_op_log(WT_SESSION_IMPL *session, WT_RET(__wt_logop_row_remove_pack( session, logrec, op->fileid, &cursor->key)); break; - WT_ILLEGAL_VALUE(session); + WT_ILLEGAL_VALUE(session, upd->type); } } else { recno = WT_INSERT_RECNO(cbt->ins); @@ -118,7 +118,7 @@ __txn_op_log(WT_SESSION_IMPL *session, WT_RET(__wt_logop_col_remove_pack( session, logrec, op->fileid, recno)); break; - WT_ILLEGAL_VALUE(session); + WT_ILLEGAL_VALUE(session, upd->type); } } @@ -498,7 +498,7 @@ __wt_txn_checkpoint_log( __wt_scr_free(session, &txn->ckpt_snapshot); txn->full_ckpt = false; break; - WT_ILLEGAL_VALUE_ERR(session); + WT_ILLEGAL_VALUE_ERR(session, flags); } err: __wt_logrec_free(session, &logrec); diff --git a/src/third_party/wiredtiger/src/txn/txn_recover.c b/src/third_party/wiredtiger/src/txn/txn_recover.c index ac656047e62..34ac57e3c92 100644 --- a/src/third_party/wiredtiger/src/txn/txn_recover.c +++ b/src/third_party/wiredtiger/src/txn/txn_recover.c @@ -247,7 +247,7 @@ __txn_op_apply( stop = cursor; break; - WT_ILLEGAL_VALUE_ERR(session); + WT_ILLEGAL_VALUE_ERR(session, mode); } /* Set the keys. */ @@ -264,7 +264,7 @@ __txn_op_apply( WT_ERR(ret); break; - WT_ILLEGAL_VALUE_ERR(session); + WT_ILLEGAL_VALUE_ERR(session, optype); } /* Reset the cursor so it doesn't block eviction. */ @@ -630,7 +630,7 @@ __wt_txn_recover(WT_SESSION_IMPL *session) * Clear this out. We no longer need it and it could have been * re-allocated when scanning the files. */ - metafile = NULL; + WT_NOT_READ(metafile, NULL); /* * We no longer need the metadata cursor: close it to avoid pinning any @@ -659,7 +659,7 @@ __wt_txn_recover(WT_SESSION_IMPL *session) if (F_ISSET(conn, WT_CONN_READONLY)) WT_ERR_MSG(session, WT_RUN_RECOVERY, "Read-only database needs recovery"); - WT_ERR(WT_RUN_RECOVERY); + WT_ERR_MSG(session, WT_RUN_RECOVERY, "Database needs recovery"); } if (F_ISSET(conn, WT_CONN_READONLY)) { diff --git a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c index e01db53fda9..fde2d63b36b 100644 --- a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c +++ b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c @@ -252,7 +252,7 @@ __txn_abort_newer_updates( case WT_PAGE_ROW_LEAF: __txn_abort_newer_row_leaf(session, page, rollback_timestamp); break; - WT_ILLEGAL_VALUE(session); + WT_ILLEGAL_VALUE(session, page->type); } return (0); @@ -466,7 +466,8 @@ __wt_txn_rollback_to_stable(WT_SESSION_IMPL *session, const char *cfg[]) * trees in cache populates a list that is used to check which * lookaside records should be removed. */ - WT_ERR(__txn_rollback_to_stable_lookaside_fixup(session)); + if (!F_ISSET(conn, WT_CONN_IN_MEMORY)) + WT_ERR(__txn_rollback_to_stable_lookaside_fixup(session)); err: F_CLR(conn, WT_CONN_EVICTION_NO_LOOKASIDE); __wt_free(session, conn->stable_rollback_bitstring); diff --git a/src/third_party/wiredtiger/src/utilities/util_alter.c b/src/third_party/wiredtiger/src/utilities/util_alter.c index b57c2d7b090..152dd1b8c03 100644 --- a/src/third_party/wiredtiger/src/utilities/util_alter.c +++ b/src/third_party/wiredtiger/src/utilities/util_alter.c @@ -31,8 +31,7 @@ util_alter(WT_SESSION *session, int argc, char *argv[]) if (argc % 2 != 0) return (usage()); - for (configp = argv; - configp != NULL && *configp != NULL; configp += 2) + for (configp = argv; *configp != NULL; configp += 2) if ((ret = session->alter( session, configp[0], configp[1])) != 0) { (void)util_err(session, ret, diff --git a/src/third_party/wiredtiger/src/utilities/util_load.c b/src/third_party/wiredtiger/src/utilities/util_load.c index 7e0f816436e..ea816675dee 100644 --- a/src/third_party/wiredtiger/src/utilities/util_load.c +++ b/src/third_party/wiredtiger/src/utilities/util_load.c @@ -237,27 +237,33 @@ config_read(WT_SESSION *session, char ***listp, bool *hexp) memset(&l, 0, sizeof(l)); /* Header line #1: "WiredTiger Dump" and a WiredTiger version. */ - if (util_read_line(session, &l, false, &eof)) - return (1); + if ((ret = util_read_line(session, &l, false, &eof)) != 0) + goto err; s = "WiredTiger Dump "; - if (strncmp(l.mem, s, strlen(s)) != 0) - return (format(session)); + if (strncmp(l.mem, s, strlen(s)) != 0) { + ret = format(session); + goto err; + } /* Header line #2: "Format={hex,print}". */ - if (util_read_line(session, &l, false, &eof)) - return (1); + if ((ret = util_read_line(session, &l, false, &eof)) != 0) + goto err; if (strcmp(l.mem, "Format=print") == 0) *hexp = false; else if (strcmp(l.mem, "Format=hex") == 0) *hexp = true; - else - return (format(session)); + else { + ret = format(session); + goto err; + } /* Header line #3: "Header". */ - if (util_read_line(session, &l, false, &eof)) - return (1); - if (strcmp(l.mem, "Header") != 0) - return (format(session)); + if ((ret = util_read_line(session, &l, false, &eof)) != 0) + goto err; + if (strcmp(l.mem, "Header") != 0) { + ret = format(session); + goto err; + } /* Now, read in lines until we get to the end of the headers. */ for (entry = max_entry = 0, list = NULL;; ++entry) { @@ -297,6 +303,8 @@ config_read(WT_SESSION *session, char ***listp, bool *hexp) goto err; } *listp = list; + + free(l.mem); return (0); err: if (list != NULL) { @@ -304,6 +312,7 @@ err: if (list != NULL) { free(*tlist); free(list); } + free(l.mem); return (ret); } @@ -542,20 +551,21 @@ insert(WT_CURSOR *cursor, const char *name) * and ignore it (a dump with "append" set), or not read it at * all (flat-text load). */ - if (util_read_line(session, &key, true, &eof)) - return (1); + if ((ret = util_read_line(session, &key, true, &eof)) != 0) + goto err; if (eof) break; if (!append) cursor->set_key(cursor, key.mem); - if (util_read_line(session, &value, false, &eof)) - return (1); + if ((ret = util_read_line(session, &value, false, &eof)) != 0) + goto err; cursor->set_value(cursor, value.mem); - if ((ret = cursor->insert(cursor)) != 0) - return ( - util_err(session, ret, "%s: cursor.insert", name)); + if ((ret = cursor->insert(cursor)) != 0) { + ret = util_err(session, ret, "%s: cursor.insert", name); + goto err; + } /* Report on progress every 100 inserts. */ if (verbose && ++insert_count % 100 == 0) { @@ -567,7 +577,10 @@ insert(WT_CURSOR *cursor, const char *name) if (verbose) printf("\r\t%s: %" PRIu64 "\n", name, insert_count); - return (0); +err: free(key.mem); + free(value.mem); + + return (ret); } static int diff --git a/src/third_party/wiredtiger/src/utilities/util_load_json.c b/src/third_party/wiredtiger/src/utilities/util_load_json.c index 47c56a84064..b6e63fef784 100644 --- a/src/third_party/wiredtiger/src/utilities/util_load_json.c +++ b/src/third_party/wiredtiger/src/utilities/util_load_json.c @@ -599,16 +599,16 @@ util_load_json(WT_SESSION *session, const char *filename, uint32_t flags) memset(&instate, 0, sizeof(instate)); instate.session = session; - if (util_read_line(session, &instate.line, false, &instate.ateof)) - return (1); - instate.p = (const char *)instate.line.mem; - instate.linenum = 1; - instate.filename = filename; + if ((ret = util_read_line( + session, &instate.line, false, &instate.ateof)) == 0) { + instate.p = (const char *)instate.line.mem; + instate.linenum = 1; + instate.filename = filename; - if ((ret = json_top_level(session, &instate, flags)) != 0) - goto err; + ret = json_top_level(session, &instate, flags); + } -err: free(instate.line.mem); + free(instate.line.mem); free(instate.kvraw); return (ret); } diff --git a/src/third_party/wiredtiger/src/utilities/util_loadtext.c b/src/third_party/wiredtiger/src/utilities/util_loadtext.c index 3e57e3ea0e4..7271fbedf34 100644 --- a/src/third_party/wiredtiger/src/utilities/util_loadtext.c +++ b/src/third_party/wiredtiger/src/utilities/util_loadtext.c @@ -150,6 +150,8 @@ insert(WT_CURSOR *cursor, const char *name, bool readkey) fflush(stdout); } } + free(key.mem); + free(value.mem); if (verbose) printf("\r\t%s: %" PRIu64 "\n", name, insert_count); diff --git a/src/third_party/wiredtiger/src/utilities/util_main.c b/src/third_party/wiredtiger/src/utilities/util_main.c index 2d08c4c5274..7042736e2a2 100644 --- a/src/third_party/wiredtiger/src/utilities/util_main.c +++ b/src/third_party/wiredtiger/src/utilities/util_main.c @@ -11,7 +11,7 @@ const char *home = "."; /* Home directory */ const char *progname; /* Program name */ /* Global arguments */ -const char *usage_prefix = "[-LRVv] [-C config] [-E secretkey] [-h home]"; +const char *usage_prefix = "[-LRSVv] [-C config] [-E secretkey] [-h home]"; bool verbose = false; /* Verbose flag */ static const char *command; /* Command name */ @@ -19,6 +19,7 @@ static const char *command; /* Command name */ #define REC_ERROR "log=(recover=error)" #define REC_LOGOFF "log=(enabled=false)" #define REC_RECOVER "log=(recover=on)" +#define REC_SALVAGE "log=(recover=salvage)" static void usage(void) @@ -70,7 +71,7 @@ main(int argc, char *argv[]) int ch, major_v, minor_v, tret, (*func)(WT_SESSION *, int, char *[]); const char *cmd_config, *config, *p1, *p2, *p3, *rec_config; char *p, *secretkey; - bool logoff, needconn, recover; + bool logoff, needconn, recover, salvage; conn = NULL; p = NULL; @@ -105,9 +106,9 @@ main(int argc, char *argv[]) * needed, the user can specify -R to run recovery. */ rec_config = REC_ERROR; - logoff = recover = false; + logoff = recover = salvage = false; /* Check for standard options. */ - while ((ch = __wt_getopt(progname, argc, argv, "C:E:h:LRVv")) != EOF) + while ((ch = __wt_getopt(progname, argc, argv, "C:E:h:LRSVv")) != EOF) switch (ch) { case 'C': /* wiredtiger_open config */ cmd_config = __wt_optarg; @@ -131,6 +132,10 @@ main(int argc, char *argv[]) rec_config = REC_RECOVER; recover = true; break; + case 'S': /* salvage */ + rec_config = REC_SALVAGE; + salvage = true; + break; case 'V': /* version */ printf("%s\n", wiredtiger_version(NULL, NULL, NULL)); goto done; @@ -142,8 +147,9 @@ main(int argc, char *argv[]) usage(); goto err; } - if (logoff && recover) { - fprintf(stderr, "Only one of -L and -R is allowed.\n"); + if ((logoff && recover) || (logoff && salvage) || + (recover && salvage)) { + fprintf(stderr, "Only one of -L, -R, and -S is allowed.\n"); goto err; } argc -= __wt_optind; diff --git a/src/third_party/wiredtiger/test/bloom/test_bloom.c b/src/third_party/wiredtiger/test/bloom/test_bloom.c index dcc7ab372a9..f3072b1860a 100644 --- a/src/third_party/wiredtiger/test/bloom/test_bloom.c +++ b/src/third_party/wiredtiger/test/bloom/test_bloom.c @@ -89,7 +89,8 @@ main(int argc, char *argv[]) } argc -= __wt_optind; - argv += __wt_optind; + if (argc != 0) + usage(); setup(); run(); diff --git a/src/third_party/wiredtiger/test/checkpoint/checkpointer.c b/src/third_party/wiredtiger/test/checkpoint/checkpointer.c index 170cde22fa1..99468504129 100644 --- a/src/third_party/wiredtiger/test/checkpoint/checkpointer.c +++ b/src/third_party/wiredtiger/test/checkpoint/checkpointer.c @@ -53,7 +53,7 @@ start_checkpoints(void) void end_checkpoints(void) { - testutil_check(__wt_thread_join(NULL, g.checkpoint_thread)); + testutil_check(__wt_thread_join(NULL, &g.checkpoint_thread)); } /* diff --git a/src/third_party/wiredtiger/test/checkpoint/workers.c b/src/third_party/wiredtiger/test/checkpoint/workers.c index c72b4b897b1..b57dad86b9f 100644 --- a/src/third_party/wiredtiger/test/checkpoint/workers.c +++ b/src/third_party/wiredtiger/test/checkpoint/workers.c @@ -103,7 +103,7 @@ start_workers(table_type type) /* Wait for the threads. */ for (i = 0; i < g.nworkers; ++i) - testutil_check(__wt_thread_join(NULL, tids[i])); + testutil_check(__wt_thread_join(NULL, &tids[i])); (void)gettimeofday(&stop, NULL); seconds = (stop.tv_sec - start.tv_sec) + diff --git a/src/third_party/wiredtiger/test/csuite/Makefile.am b/src/third_party/wiredtiger/test/csuite/Makefile.am index b3dde5ec628..47f2ab934fe 100644 --- a/src/third_party/wiredtiger/test/csuite/Makefile.am +++ b/src/third_party/wiredtiger/test/csuite/Makefile.am @@ -15,6 +15,10 @@ test_rwlock_SOURCES = rwlock/main.c noinst_PROGRAMS += test_rwlock all_TESTS += test_rwlock +test_schema_abort_SOURCES = schema_abort/main.c +noinst_PROGRAMS += test_schema_abort +all_TESTS += schema_abort/smoke.sh + test_scope_SOURCES = scope/main.c noinst_PROGRAMS += test_scope all_TESTS += test_scope diff --git a/src/third_party/wiredtiger/test/csuite/random_abort/main.c b/src/third_party/wiredtiger/test/csuite/random_abort/main.c index e99ed5ecd4d..db5d19e8ab2 100644 --- a/src/third_party/wiredtiger/test/csuite/random_abort/main.c +++ b/src/third_party/wiredtiger/test/csuite/random_abort/main.c @@ -201,7 +201,7 @@ fill_db(uint32_t nth) * it is killed. */ for (i = 0; i < nth; ++i) - testutil_check(__wt_thread_join(NULL, thr[i])); + testutil_check(__wt_thread_join(NULL, &thr[i])); /* * NOTREACHED */ @@ -280,7 +280,6 @@ main(int argc, char *argv[]) usage(); } argc -= __wt_optind; - argv += __wt_optind; if (argc != 0) usage(); diff --git a/src/third_party/wiredtiger/test/csuite/schema_abort/main.c b/src/third_party/wiredtiger/test/csuite/schema_abort/main.c new file mode 100644 index 00000000000..735faa417c3 --- /dev/null +++ b/src/third_party/wiredtiger/test/csuite/schema_abort/main.c @@ -0,0 +1,1310 @@ +/*- + * Public Domain 2014-2018 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "test_util.h" + +#include <sys/wait.h> +#include <signal.h> + +static char home[1024]; /* Program working dir */ + +/* + * Create three tables that we will write the same data to and verify that + * all the types of usage have the expected data in them after a crash and + * recovery. We want: + * 1. A table that is logged and is not involved in timestamps. This table + * simulates a user local table. + * 2. A table that is logged and involved in timestamps. This simulates + * the oplog. + * 3. A table that is not logged and involved in timestamps. This simulates + * a typical collection file. + * + * We also have most threads perform schema operations such as create/drop. + * + * We also create several files that are not WiredTiger tables. The checkpoint + * thread creates a file indicating that a checkpoint has completed. The parent + * process uses this to know when at least one checkpoint is done and it can + * start the timer to abort. + * + * Each worker thread creates its own records file that records the data it + * inserted and it records the timestamp that was used for that insertion. + */ +#define INVALID_KEY UINT64_MAX +#define MAX_CKPT_INVL 2 /* Maximum interval between checkpoints */ +#define MAX_TH 12 +#define MAX_TIME 40 +#define MAX_VAL 1024 +#define MIN_TH 5 +#define MIN_TIME 10 +#define PREPARE_FREQ 5 +#define PREPARE_YIELD (PREPARE_FREQ * 10) +#define RECORDS_FILE "records-%" PRIu32 +#define STABLE_PERIOD 100 + +static const char * const uri = "table:wt"; +static const char * const uri_local = "table:local"; +static const char * const uri_oplog = "table:oplog"; +static const char * const uri_collection = "table:collection"; + +static const char * const ckpt_file = "checkpoint_done"; + +static bool compat, inmem, stable_set, use_ts, use_txn; +static volatile uint64_t global_ts = 1; +static volatile uint64_t uid = 1; +static volatile uint64_t th_ts[MAX_TH]; + +#define ENV_CONFIG_COMPAT ",compatibility=(release=\"2.9\")" +#define ENV_CONFIG_DEF \ + "create,log=(archive=false,file_max=10M,enabled)" +#define ENV_CONFIG_TXNSYNC \ + "create,log=(archive=false,file_max=10M,enabled)," \ + "transaction_sync=(enabled,method=none)" +#define ENV_CONFIG_REC "log=(archive=false,recover=on)" + +typedef struct { + uint64_t absent_key; /* Last absent key */ + uint64_t exist_key; /* First existing key after miss */ + uint64_t first_key; /* First key in range */ + uint64_t first_miss; /* First missing key */ + uint64_t last_key; /* Last key in range */ +} REPORT; + +typedef struct { + WT_CONNECTION *conn; + uint64_t start; + uint32_t info; +} THREAD_DATA; + +static void sig_handler(int) + WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); +static void usage(void) + WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); +static void +usage(void) +{ + fprintf(stderr, + "usage: %s [-h dir] [-T threads] [-t time] [-Cmvxz]\n", progname); + exit(EXIT_FAILURE); +} + +static const char * const config = NULL; + +/* + * subtest_error_handler -- + * Error event handler. + */ +static int +subtest_error_handler(WT_EVENT_HANDLER *handler, + WT_SESSION *session, int error, const char *message) +{ + (void)(handler); + (void)(session); + (void)(error); + + /* Filter out errors about bulk load usage - they are annoying */ + if (strstr(message, "bulk-load is only supported on newly") == NULL) + fprintf(stderr, "%s", message); + return (0); +} + +static WT_EVENT_HANDLER event_handler = { + subtest_error_handler, + NULL, /* Message handler */ + NULL, /* Progress handler */ + NULL /* Close handler */ +}; + +/* + * The following are various schema-related functions to have some threads + * performing during the test. The goal is to make sure that after a random + * abort, the database is left in a recoverable state. Yield during the + * schema operations to increase chance of abort during them. + * + * TODO: Currently only verifies insert data, it would be ideal to modify the + * schema operations so that we can verify the state of the schema too. + */ + +static void +dump_ts(uint64_t nth) +{ + uint64_t i; + + for (i = 0; i < nth; ++i) + fprintf(stderr, "THREAD %" PRIu64 ": ts: %" PRIu64 "\n", + i, th_ts[i]); +} + +/* + * test_bulk -- + * Test creating a bulk cursor. + */ +static void +test_bulk(THREAD_DATA *td) +{ + WT_CURSOR *c; + WT_SESSION *session; + int ret; + bool create; + + testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); + + if (use_txn) + testutil_check(session->begin_transaction(session, NULL)); + create = false; + if ((ret = session->create(session, uri, config)) != 0) + if (ret != EEXIST && ret != EBUSY) + testutil_die(ret, "session.create"); + + if (ret == 0) { + create = true; + if ((ret = session->open_cursor( + session, uri, NULL, "bulk", &c)) == 0) { + __wt_yield(); + testutil_check(c->close(c)); + } else if (ret != ENOENT && ret != EBUSY && ret != EINVAL) + testutil_die(ret, "session.open_cursor bulk"); + } + + if (use_txn) { + /* If create fails, rollback else will commit.*/ + if (!create) + ret = session->rollback_transaction(session, NULL); + else + ret = session->commit_transaction(session, NULL); + + if (ret == EINVAL) { + fprintf(stderr, "BULK: EINVAL on %s. ABORT\n", + create ? "commit" : "rollback"); + testutil_die(ret, "session.commit bulk"); + } + } + testutil_check(session->close(session, NULL)); +} + +/* + * test_bulk_unique -- + * Test creating a bulk cursor with a unique name. + */ +static void +test_bulk_unique(THREAD_DATA *td, int force) +{ + WT_CURSOR *c; + WT_SESSION *session; + uint64_t my_uid; + int ret; + char new_uri[64]; + + testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); + + /* Generate a unique object name. */ + my_uid = __wt_atomic_addv64(&uid, 1); + testutil_check(__wt_snprintf( + new_uri, sizeof(new_uri), "%s.%u", uri, my_uid)); + + if (use_txn) + testutil_check(session->begin_transaction(session, NULL)); + testutil_check(session->create(session, new_uri, config)); + + __wt_yield(); + /* + * Opening a bulk cursor may have raced with a forced checkpoint + * which created a checkpoint of the empty file, and triggers an EINVAL. + */ + if ((ret = session->open_cursor( + session, new_uri, NULL, "bulk", &c)) == 0) + testutil_check(c->close(c)); + else if (ret != EINVAL) + testutil_die(ret, + "session.open_cursor bulk unique: %s, new_uri"); + + while ((ret = session->drop( + session, new_uri, force ? "force" : NULL)) != 0) + if (ret != EBUSY) + testutil_die(ret, "session.drop: %s", new_uri); + + if (use_txn && + (ret = session->commit_transaction(session, NULL)) != 0 && + ret != EINVAL) + testutil_die(ret, "session.commit bulk unique"); + testutil_check(session->close(session, NULL)); +} + +/* + * test_cursor -- + * Open a cursor on a data source. + */ +static void +test_cursor(THREAD_DATA *td) +{ + WT_CURSOR *cursor; + WT_SESSION *session; + int ret; + + testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); + + if (use_txn) + testutil_check(session->begin_transaction(session, NULL)); + if ((ret = + session->open_cursor(session, uri, NULL, NULL, &cursor)) != 0) { + if (ret != ENOENT && ret != EBUSY) + testutil_die(ret, "session.open_cursor"); + } else { + __wt_yield(); + testutil_check(cursor->close(cursor)); + } + + if (use_txn && + (ret = session->commit_transaction(session, NULL)) != 0 && + ret != EINVAL) + testutil_die(ret, "session.commit cursor"); + testutil_check(session->close(session, NULL)); +} + +/* + * test_create -- + * Create a table. + */ +static void +test_create(THREAD_DATA *td) +{ + WT_SESSION *session; + int ret; + + testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); + + if (use_txn) + testutil_check(session->begin_transaction(session, NULL)); + if ((ret = session->create(session, uri, config)) != 0) + if (ret != EEXIST && ret != EBUSY) + testutil_die(ret, "session.create"); + __wt_yield(); + if (use_txn && + (ret = session->commit_transaction(session, NULL)) != 0 && + ret != EINVAL) + testutil_die(ret, "session.commit create"); + testutil_check(session->close(session, NULL)); +} + +/* + * test_create_unique -- + * Create a uniquely named table. + */ +static void +test_create_unique(THREAD_DATA *td, int force) +{ + WT_SESSION *session; + uint64_t my_uid; + int ret; + char new_uri[64]; + + testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); + + /* Generate a unique object name. */ + my_uid = __wt_atomic_addv64(&uid, 1); + testutil_check(__wt_snprintf( + new_uri, sizeof(new_uri), "%s.%u", uri, my_uid)); + + if (use_txn) + testutil_check(session->begin_transaction(session, NULL)); + testutil_check(session->create(session, new_uri, config)); + if (use_txn && + (ret = session->commit_transaction(session, NULL)) != 0 && + ret != EINVAL) + testutil_die(ret, "session.commit create unique"); + + __wt_yield(); + if (use_txn) + testutil_check(session->begin_transaction(session, NULL)); + while ((ret = session->drop( + session, new_uri, force ? "force" : NULL)) != 0) + if (ret != EBUSY) + testutil_die(ret, "session.drop: %s", new_uri); + if (use_txn && + (ret = session->commit_transaction(session, NULL)) != 0 && + ret != EINVAL) + testutil_die(ret, "session.commit create unique"); + + testutil_check(session->close(session, NULL)); +} + +/* + * test_drop -- + * Test dropping a table. + */ +static void +test_drop(THREAD_DATA *td, int force) +{ + WT_SESSION *session; + int ret; + + testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); + + if (use_txn) + testutil_check(session->begin_transaction(session, NULL)); + if ((ret = session->drop(session, uri, force ? "force" : NULL)) != 0) + if (ret != ENOENT && ret != EBUSY) + testutil_die(ret, "session.drop"); + + if (use_txn) { + /* + * As the operations are being performed concurrently, + * return value can be ENOENT or EBUSY will set + * error to transaction opened by session. In these + * cases the transaction has to be aborted. + */ + if (ret != ENOENT && ret != EBUSY) + ret = session->commit_transaction(session, NULL); + else + ret = session->rollback_transaction(session, NULL); + if (ret == EINVAL) + testutil_die(ret, "session.commit drop"); + } + testutil_check(session->close(session, NULL)); +} + +/* + * test_rebalance -- + * Rebalance a tree. + */ +static void +test_rebalance(THREAD_DATA *td) +{ + WT_SESSION *session; + int ret; + + testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); + + if ((ret = session->rebalance(session, uri, NULL)) != 0) + if (ret != ENOENT && ret != EBUSY) + testutil_die(ret, "session.rebalance"); + + testutil_check(session->close(session, NULL)); +} + +/* + * test_upgrade -- + * Upgrade a tree. + */ +static void +test_upgrade(THREAD_DATA *td) +{ + WT_SESSION *session; + int ret; + + testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); + + if ((ret = session->upgrade(session, uri, NULL)) != 0) + if (ret != ENOENT && ret != EBUSY) + testutil_die(ret, "session.upgrade"); + + testutil_check(session->close(session, NULL)); +} + +/* + * test_verify -- + * Verify a tree. + */ +static void +test_verify(THREAD_DATA *td) +{ + WT_SESSION *session; + int ret; + + testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); + + if ((ret = session->verify(session, uri, NULL)) != 0) + if (ret != ENOENT && ret != EBUSY) + testutil_die(ret, "session.verify"); + + testutil_check(session->close(session, NULL)); +} + +/* + * thread_ts_run -- + * Runner function for a timestamp thread. + */ +static WT_THREAD_RET +thread_ts_run(void *arg) +{ + WT_SESSION *session; + THREAD_DATA *td; + uint64_t i, last_ts, oldest_ts, this_ts; + char tscfg[64]; + + td = (THREAD_DATA *)arg; + last_ts = 0; + + testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); + /* + * Every N records we will record our stable timestamp into the stable + * table. That will define our threshold where we expect to find records + * after recovery. + */ + for (;;) { + oldest_ts = UINT64_MAX; + /* + * For the timestamp thread, the info field contains the number + * of worker threads. + */ + for (i = 0; i < td->info; ++i) { + /* + * We need to let all threads get started, so if we find + * any thread still with a zero timestamp we go to + * sleep. + */ + this_ts = th_ts[i]; + if (this_ts == 0) + goto ts_wait; + else if (this_ts < oldest_ts) + oldest_ts = this_ts; + } + + if (oldest_ts != UINT64_MAX && + oldest_ts - last_ts > STABLE_PERIOD) { + /* + * Set both the oldest and stable timestamp so that we + * don't need to maintain read availability at older + * timestamps. + */ + testutil_check(__wt_snprintf( + tscfg, sizeof(tscfg), + "oldest_timestamp=%" PRIx64 + ",stable_timestamp=%" PRIx64, + oldest_ts, oldest_ts)); + testutil_check( + td->conn->set_timestamp(td->conn, tscfg)); + last_ts = oldest_ts; + if (!stable_set) { + stable_set = true; + printf("SET STABLE: %" PRIx64 " %" PRIu64 "\n", + oldest_ts, oldest_ts); + } + } else +ts_wait: __wt_sleep(0, 1000); + } + /* NOTREACHED */ +} + +/* + * thread_ckpt_run -- + * Runner function for the checkpoint thread. + */ +static WT_THREAD_RET +thread_ckpt_run(void *arg) +{ + struct timespec now, start; + FILE *fp; + WT_RAND_STATE rnd; + WT_SESSION *session; + THREAD_DATA *td; + uint64_t ts; + uint32_t sleep_time; + int i; + bool first_ckpt; + + __wt_random_init(&rnd); + + td = (THREAD_DATA *)arg; + /* + * Keep a separate file with the records we wrote for checking. + */ + (void)unlink(ckpt_file); + testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); + first_ckpt = true; + ts = 0; + __wt_epoch(NULL, &start); + for (i = 0; ;++i) { + sleep_time = __wt_random(&rnd) % MAX_CKPT_INVL; + sleep(sleep_time); + if (use_ts) + ts = global_ts; + /* + * Since this is the default, send in this string even if + * running without timestamps. + */ + testutil_check(session->checkpoint( + session, "use_timestamp=true")); + printf("Checkpoint %d complete. Minimum ts %" PRIu64 "\n", + i, ts); + fflush(stdout); + /* + * Create the checkpoint file so that the parent process knows + * at least one checkpoint has finished and can start its + * timer. + */ + if (first_ckpt) { + testutil_checksys((fp = fopen(ckpt_file, "w")) == NULL); + first_ckpt = false; + testutil_checksys(fclose(fp) != 0); + } + + __wt_epoch(NULL, &now); + printf("CKPT: stable_set %d, time %" PRIu64 "\n", + stable_set, WT_TIMEDIFF_SEC(now, start)); + if (use_ts && !stable_set && WT_TIMEDIFF_SEC(now, start) > 2) { + fprintf(stderr, + "After 2 seconds ckpt stable still not set\n"); + /* + * For the checkpoint thread the info contains the + * number of threads. + */ + dump_ts(td->info); + abort(); + } + } + /* NOTREACHED */ +} + +/* + * thread_run -- + * Runner function for the worker threads. + */ +static WT_THREAD_RET +thread_run(void *arg) +{ + FILE *fp; + WT_CURSOR *cur_coll, *cur_local, *cur_oplog; + WT_ITEM data; + WT_RAND_STATE rnd; + WT_SESSION *oplog_session, *session; + THREAD_DATA *td; + uint64_t i, stable_ts; + char cbuf[MAX_VAL], lbuf[MAX_VAL], obuf[MAX_VAL]; + char kname[64], tscfg[64]; + bool use_prep; + + __wt_random_init(&rnd); + memset(cbuf, 0, sizeof(cbuf)); + memset(lbuf, 0, sizeof(lbuf)); + memset(obuf, 0, sizeof(obuf)); + memset(kname, 0, sizeof(kname)); + + td = (THREAD_DATA *)arg; + /* + * Set up the separate file for checking. + */ + testutil_check(__wt_snprintf( + cbuf, sizeof(cbuf), RECORDS_FILE, td->info)); + (void)unlink(cbuf); + testutil_checksys((fp = fopen(cbuf, "w")) == NULL); + /* + * Set to line buffering. But that is advisory only. We've seen + * cases where the result files end up with partial lines. + */ + __wt_stream_set_line_buffer(fp); + + /* + * Have half the threads use prepared transactions if timestamps + * are in use. + */ + use_prep = (use_ts && td->info % 2 == 0) ? true : false; + /* + * We may have two sessions so that the oplog session can have its own + * transaction in parallel with the collection session for threads + * that are going to be using prepared transactions. We need this + * because prepared transactions cannot have any operations that modify + * a table that is logged. But we also want to test mixed logged and + * not-logged transactions. + */ + testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); + /* + * Open a cursor to each table. + */ + testutil_check(session->open_cursor(session, + uri_collection, NULL, NULL, &cur_coll)); + testutil_check(session->open_cursor(session, + uri_local, NULL, NULL, &cur_local)); + oplog_session = NULL; + if (use_prep) { + testutil_check(td->conn->open_session( + td->conn, NULL, NULL, &oplog_session)); + testutil_check(session->open_cursor(oplog_session, + uri_oplog, NULL, NULL, &cur_oplog)); + } else + testutil_check(session->open_cursor(session, + uri_oplog, NULL, NULL, &cur_oplog)); + + /* + * Write our portion of the key space until we're killed. + */ + printf("Thread %" PRIu32 " starts at %" PRIu64 "\n", + td->info, td->start); + stable_ts = 0; + for (i = td->start;; ++i) { + /* + * Allow some threads to skip schema operations so that they + * are generating sufficient dirty data. + */ + if (td->info != 0 && td->info != 1) + /* + * Do a schema operation about 50% of the time by having + * a case for only about half the possible mod values. + */ + switch (__wt_random(&rnd) % 20) { + case 0: + test_bulk(td); + break; + case 1: + test_bulk_unique(td, __wt_random(&rnd) & 1); + break; + case 2: + test_create(td); + break; + case 3: + test_create_unique(td, __wt_random(&rnd) & 1); + break; + case 4: + test_cursor(td); + break; + case 5: + test_drop(td, __wt_random(&rnd) & 1); + break; + case 6: + test_rebalance(td); + break; + case 7: + test_upgrade(td); + break; + case 8: + test_verify(td); + break; + } + if (use_ts) + stable_ts = __wt_atomic_addv64(&global_ts, 1); + testutil_check(__wt_snprintf( + kname, sizeof(kname), "%" PRIu64, i)); + + testutil_check(session->begin_transaction(session, NULL)); + if (use_prep) + testutil_check(oplog_session->begin_transaction( + oplog_session, NULL)); + cur_coll->set_key(cur_coll, kname); + cur_local->set_key(cur_local, kname); + cur_oplog->set_key(cur_oplog, kname); + /* + * Put an informative string into the value so that it + * can be viewed well in a binary dump. + */ + testutil_check(__wt_snprintf(cbuf, sizeof(cbuf), + "COLL: thread:%" PRIu64 " ts:%" PRIu64 " key: %" PRIu64, + td->info, stable_ts, i)); + testutil_check(__wt_snprintf(lbuf, sizeof(lbuf), + "LOCAL: thread:%" PRIu64 " ts:%" PRIu64 " key: %" PRIu64, + td->info, stable_ts, i)); + testutil_check(__wt_snprintf(obuf, sizeof(obuf), + "OPLOG: thread:%" PRIu64 " ts:%" PRIu64 " key: %" PRIu64, + td->info, stable_ts, i)); + data.size = __wt_random(&rnd) % MAX_VAL; + data.data = cbuf; + cur_coll->set_value(cur_coll, &data); + testutil_check(cur_coll->insert(cur_coll)); + data.size = __wt_random(&rnd) % MAX_VAL; + data.data = obuf; + cur_oplog->set_value(cur_oplog, &data); + testutil_check(cur_oplog->insert(cur_oplog)); + if (use_ts) { + /* + * Run with prepare every once in a while. And also + * yield after prepare sometimes too. This is only done + * on the regular session. + */ + if (use_prep && i % PREPARE_FREQ == 0) { + testutil_check(__wt_snprintf( + tscfg, sizeof(tscfg), + "prepare_timestamp=%" PRIx64, stable_ts)); + testutil_check(session->prepare_transaction( + session, tscfg)); + if (i % PREPARE_YIELD == 0) + __wt_yield(); + } + testutil_check(__wt_snprintf(tscfg, sizeof(tscfg), + "commit_timestamp=%" PRIx64, stable_ts)); + testutil_check( + session->commit_transaction(session, tscfg)); + if (use_prep) + testutil_check( + oplog_session->commit_transaction( + oplog_session, tscfg)); + /* + * Update the thread's last-committed timestamp. + * Don't let the compiler re-order this statement, + * if we were to race with the timestamp thread, it + * might see our thread update before the commit. + */ + WT_PUBLISH(th_ts[td->info], stable_ts); + } else { + testutil_check( + session->commit_transaction(session, NULL)); + if (use_prep) + testutil_check( + oplog_session->commit_transaction( + oplog_session, NULL)); + } + /* + * Insert into the local table outside the timestamp txn. + */ + data.size = __wt_random(&rnd) % MAX_VAL; + data.data = lbuf; + cur_local->set_value(cur_local, &data); + testutil_check(cur_local->insert(cur_local)); + + /* + * Save the timestamp and key separately for checking later. + */ + if (fprintf(fp, + "%" PRIu64 " %" PRIu64 "\n", stable_ts, i) < 0) + testutil_die(EIO, "fprintf"); + } + /* NOTREACHED */ +} + +/* + * Child process creates the database and table, and then creates worker + * threads to add data until it is killed by the parent. + */ +static void run_workload(uint32_t) + WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); +static void +run_workload(uint32_t nth) +{ + WT_CONNECTION *conn; + WT_SESSION *session; + THREAD_DATA *td; + wt_thread_t *thr; + uint32_t ckpt_id, i, ts_id; + char envconf[512]; + + thr = dcalloc(nth+2, sizeof(*thr)); + td = dcalloc(nth+2, sizeof(THREAD_DATA)); + stable_set = false; + if (chdir(home) != 0) + testutil_die(errno, "Child chdir: %s", home); + if (inmem) + strcpy(envconf, ENV_CONFIG_DEF); + else + strcpy(envconf, ENV_CONFIG_TXNSYNC); + if (compat) + strcat(envconf, ENV_CONFIG_COMPAT); + + testutil_check(wiredtiger_open(NULL, &event_handler, envconf, &conn)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + /* + * Create all the tables. + */ + testutil_check(session->create(session, uri_collection, + "key_format=S,value_format=u,log=(enabled=false)")); + testutil_check(session->create(session, + uri_local, "key_format=S,value_format=u")); + testutil_check(session->create(session, + uri_oplog, "key_format=S,value_format=u")); + /* + * Don't log the stable timestamp table so that we know what timestamp + * was stored at the checkpoint. + */ + testutil_check(session->close(session, NULL)); + + /* + * The checkpoint thread and the timestamp threads are added at the end. + */ + ckpt_id = nth; + td[ckpt_id].conn = conn; + td[ckpt_id].info = nth; + printf("Create checkpoint thread\n"); + testutil_check(__wt_thread_create( + NULL, &thr[ckpt_id], thread_ckpt_run, &td[ckpt_id])); + ts_id = nth + 1; + if (use_ts) { + td[ts_id].conn = conn; + td[ts_id].info = nth; + printf("Create timestamp thread\n"); + testutil_check(__wt_thread_create( + NULL, &thr[ts_id], thread_ts_run, &td[ts_id])); + } + printf("Create %" PRIu32 " writer threads\n", nth); + for (i = 0; i < nth; ++i) { + td[i].conn = conn; + td[i].start = WT_BILLION * (uint64_t)i; + td[i].info = i; + testutil_check(__wt_thread_create( + NULL, &thr[i], thread_run, &td[i])); + } + /* + * The threads never exit, so the child will just wait here until + * it is killed. + */ + fflush(stdout); + for (i = 0; i <= ts_id; ++i) + testutil_check(__wt_thread_join(NULL, &thr[i])); + /* + * NOTREACHED + */ + free(thr); + free(td); + exit(EXIT_SUCCESS); +} + +/* + * Determines whether this is a timestamp build or not + */ +static bool +timestamp_build(void) +{ +#ifdef HAVE_TIMESTAMPS + return (true); +#else + return (false); +#endif +} + +extern int __wt_optind; +extern char *__wt_optarg; + +/* + * Initialize a report structure. Since zero is a valid key we + * cannot just clear it. + */ +static void +initialize_rep(REPORT *r) +{ + r->first_key = r->first_miss = INVALID_KEY; + r->absent_key = r->exist_key = r->last_key = INVALID_KEY; +} + +/* + * Print out information if we detect missing records in the + * middle of the data of a report structure. + */ +static void +print_missing(REPORT *r, const char *fname, const char *msg) +{ + if (r->exist_key != INVALID_KEY) + printf("%s: %s error %" PRIu64 + " absent records %" PRIu64 "-%" PRIu64 + ". Then keys %" PRIu64 "-%" PRIu64 " exist." + " Key range %" PRIu64 "-%" PRIu64 "\n", + fname, msg, + (r->exist_key - r->first_miss) - 1, + r->first_miss, r->exist_key - 1, + r->exist_key, r->last_key, + r->first_key, r->last_key); +} + +/* + * Signal handler to catch if the child died unexpectedly. + */ +static void +sig_handler(int sig) +{ + pid_t pid; + + WT_UNUSED(sig); + pid = wait(NULL); + /* + * The core file will indicate why the child exited. Choose EINVAL here. + */ + testutil_die(EINVAL, + "Child process %" PRIu64 " abnormally exited", (uint64_t)pid); +} + +int +main(int argc, char *argv[]) +{ + struct sigaction sa; + struct stat sb; + FILE *fp; + REPORT c_rep[MAX_TH], l_rep[MAX_TH], o_rep[MAX_TH]; + WT_CONNECTION *conn; + WT_CURSOR *cur_coll, *cur_local, *cur_oplog; + WT_RAND_STATE rnd; + WT_SESSION *session; + pid_t pid; + uint64_t absent_coll, absent_local, absent_oplog, count, key, last_key; + uint64_t stable_fp, stable_val; + uint32_t i; + int ret; + char fname[64], kname[64]; + bool fatal; + uint32_t nth, timeout; + int ch, status; + const char *working_dir; + char buf[512], statname[1024]; + bool rand_th, rand_time, verify_only; + + /* We have nothing to do if this is not a timestamp build */ + if (!timestamp_build()) + return (EXIT_SUCCESS); + + (void)testutil_set_progname(argv); + + compat = inmem = false; + use_ts = true; + /* + * Setting this to false forces us to use internal library code. + * Allow an override but default to using that code. + */ + use_txn = false; + nth = MIN_TH; + rand_th = rand_time = true; + timeout = MIN_TIME; + verify_only = false; + working_dir = "WT_TEST.schema-abort"; + + while ((ch = __wt_getopt(progname, argc, argv, "Ch:mT:t:vxz")) != EOF) + switch (ch) { + case 'C': + compat = true; + break; + case 'h': + working_dir = __wt_optarg; + break; + case 'm': + inmem = true; + break; + case 'T': + rand_th = false; + nth = (uint32_t)atoi(__wt_optarg); + break; + case 't': + rand_time = false; + timeout = (uint32_t)atoi(__wt_optarg); + break; + case 'v': + verify_only = true; + break; + case 'x': + use_txn = true; + break; + case 'z': + use_ts = false; + break; + default: + usage(); + } + argc -= __wt_optind; + if (argc != 0) + usage(); + + testutil_work_dir_from_path(home, sizeof(home), working_dir); + /* + * If the user wants to verify they need to tell us how many threads + * there were so we can find the old record files. + */ + if (verify_only && rand_th) { + fprintf(stderr, + "Verify option requires specifying number of threads\n"); + exit (EXIT_FAILURE); + } + if (!verify_only) { + testutil_make_work_dir(home); + + __wt_random_init_seed(NULL, &rnd); + if (rand_time) { + timeout = __wt_random(&rnd) % MAX_TIME; + if (timeout < MIN_TIME) + timeout = MIN_TIME; + } + if (rand_th) { + nth = __wt_random(&rnd) % MAX_TH; + if (nth < MIN_TH) + nth = MIN_TH; + } + + printf("Parent: compatibility: %s, " + "in-mem log sync: %s, timestamp in use: %s\n", + compat ? "true" : "false", + inmem ? "true" : "false", + use_ts ? "true" : "false"); + printf("Parent: Create %" PRIu32 + " threads; sleep %" PRIu32 " seconds\n", nth, timeout); + printf("CONFIG: %s%s%s%s -h %s -T %" PRIu32 " -t %" PRIu32 "\n", + progname, + compat ? " -C" : "", + inmem ? " -m" : "", + !use_ts ? " -z" : "", + working_dir, nth, timeout); + /* + * Fork a child to insert as many items. We will then randomly + * kill the child, run recovery and make sure all items we wrote + * exist after recovery runs. + */ + memset(&sa, 0, sizeof(sa)); + sa.sa_handler = sig_handler; + testutil_checksys(sigaction(SIGCHLD, &sa, NULL)); + testutil_checksys((pid = fork()) < 0); + + if (pid == 0) { /* child */ + run_workload(nth); + return (EXIT_SUCCESS); + } + + /* parent */ + /* + * Sleep for the configured amount of time before killing + * the child. Start the timeout from the time we notice that + * the file has been created. That allows the test to run + * correctly on really slow machines. + */ + testutil_check(__wt_snprintf( + statname, sizeof(statname), "%s/%s", home, ckpt_file)); + while (stat(statname, &sb) != 0) + sleep(1); + sleep(timeout); + sa.sa_handler = SIG_DFL; + testutil_checksys(sigaction(SIGCHLD, &sa, NULL)); + + /* + * !!! It should be plenty long enough to make sure more than + * one log file exists. If wanted, that check would be added + * here. + */ + printf("Kill child\n"); + testutil_checksys(kill(pid, SIGKILL) != 0); + testutil_checksys(waitpid(pid, &status, 0) == -1); + } + /* + * !!! If we wanted to take a copy of the directory before recovery, + * this is the place to do it. Don't do it all the time because + * it can use a lot of disk space, which can cause test machine + * issues. + */ + if (chdir(home) != 0) + testutil_die(errno, "parent chdir: %s", home); + /* + * The tables can get very large, so while we'd ideally like to + * copy the entire database, we only copy the log files for now. + * Otherwise it can take far too long to run the test, particularly + * in automated testing. + */ + testutil_check(__wt_snprintf(buf, sizeof(buf), + "rm -rf ../%s.SAVE && mkdir ../%s.SAVE && " + "cp -p * ../%s.SAVE", + home, home, home)); + if ((status = system(buf)) < 0) + testutil_die(status, "system: %s", buf); + printf("Open database, run recovery and verify content\n"); + + /* + * Open the connection which forces recovery to be run. + */ + testutil_check(wiredtiger_open( + NULL, &event_handler, ENV_CONFIG_REC, &conn)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + /* + * Open a cursor on all the tables. + */ + testutil_check(session->open_cursor(session, + uri_collection, NULL, NULL, &cur_coll)); + testutil_check(session->open_cursor(session, + uri_local, NULL, NULL, &cur_local)); + testutil_check(session->open_cursor(session, + uri_oplog, NULL, NULL, &cur_oplog)); + + /* + * Find the biggest stable timestamp value that was saved. + */ + stable_val = 0; + if (use_ts) { + testutil_check( + conn->query_timestamp(conn, buf, "get=recovery")); + sscanf(buf, "%" SCNx64, &stable_val); + printf("Got stable_val %" PRIu64 "\n", stable_val); + } + + count = 0; + absent_coll = absent_local = absent_oplog = 0; + fatal = false; + for (i = 0; i < nth; ++i) { + initialize_rep(&c_rep[i]); + initialize_rep(&l_rep[i]); + initialize_rep(&o_rep[i]); + testutil_check(__wt_snprintf( + fname, sizeof(fname), RECORDS_FILE, i)); + if ((fp = fopen(fname, "r")) == NULL) + testutil_die(errno, "fopen: %s", fname); + + /* + * For every key in the saved file, verify that the key exists + * in the table after recovery. If we're doing in-memory + * log buffering we never expect a record missing in the middle, + * but records may be missing at the end. If we did + * write-no-sync, we expect every key to have been recovered. + */ + for (last_key = INVALID_KEY;; ++count, last_key = key) { + ret = fscanf(fp, "%" SCNu64 "%" SCNu64 "\n", + &stable_fp, &key); + if (last_key == INVALID_KEY) { + c_rep[i].first_key = key; + l_rep[i].first_key = key; + o_rep[i].first_key = key; + } + if (ret != EOF && ret != 2) { + /* + * If we find a partial line, consider it + * like an EOF. + */ + if (ret == 1 || ret == 0) + break; + testutil_die(errno, "fscanf"); + } + if (ret == EOF) + break; + /* + * If we're unlucky, the last line may be a partially + * written key at the end that can result in a false + * negative error for a missing record. Detect it. + */ + if (last_key != INVALID_KEY && key != last_key + 1) { + printf("%s: Ignore partial record %" PRIu64 + " last valid key %" PRIu64 "\n", + fname, key, last_key); + break; + } + testutil_check(__wt_snprintf( + kname, sizeof(kname), "%" PRIu64, key)); + cur_coll->set_key(cur_coll, kname); + cur_local->set_key(cur_local, kname); + cur_oplog->set_key(cur_oplog, kname); + /* + * The collection table should always only have the + * data as of the checkpoint. + */ + if ((ret = cur_coll->search(cur_coll)) != 0) { + if (ret != WT_NOTFOUND) + testutil_die(ret, "search"); + /* + * If we don't find a record, the stable + * timestamp written to our file better be + * larger than the saved one. + */ + if (!inmem && + stable_fp != 0 && stable_fp <= stable_val) { + printf("%s: COLLECTION no record with " + "key %" PRIu64 " record ts %" PRIu64 + " <= stable ts %" PRIu64 "\n", + fname, key, stable_fp, stable_val); + absent_coll++; + } + if (c_rep[i].first_miss == INVALID_KEY) + c_rep[i].first_miss = key; + c_rep[i].absent_key = key; + } else if (c_rep[i].absent_key != INVALID_KEY && + c_rep[i].exist_key == INVALID_KEY) { + /* + * If we get here we found a record that exists + * after absent records, a hole in our data. + */ + c_rep[i].exist_key = key; + fatal = true; + } else if (!inmem && + stable_fp != 0 && stable_fp > stable_val) { + /* + * If we found a record, the stable timestamp + * written to our file better be no larger + * than the checkpoint one. + */ + printf("%s: COLLECTION record with " + "key %" PRIu64 " record ts %" PRIu64 + " > stable ts %" PRIu64 "\n", + fname, key, stable_fp, stable_val); + fatal = true; + } + /* + * The local table should always have all data. + */ + if ((ret = cur_local->search(cur_local)) != 0) { + if (ret != WT_NOTFOUND) + testutil_die(ret, "search"); + if (!inmem) + printf("%s: LOCAL no record with key %" + PRIu64 "\n", fname, key); + absent_local++; + if (l_rep[i].first_miss == INVALID_KEY) + l_rep[i].first_miss = key; + l_rep[i].absent_key = key; + } else if (l_rep[i].absent_key != INVALID_KEY && + l_rep[i].exist_key == INVALID_KEY) { + /* + * We should never find an existing key after + * we have detected one missing. + */ + l_rep[i].exist_key = key; + fatal = true; + } + /* + * The oplog table should always have all data. + */ + if ((ret = cur_oplog->search(cur_oplog)) != 0) { + if (ret != WT_NOTFOUND) + testutil_die(ret, "search"); + if (!inmem) + printf("%s: OPLOG no record with key %" + PRIu64 "\n", fname, key); + absent_oplog++; + if (o_rep[i].first_miss == INVALID_KEY) + o_rep[i].first_miss = key; + o_rep[i].absent_key = key; + } else if (o_rep[i].absent_key != INVALID_KEY && + o_rep[i].exist_key == INVALID_KEY) { + /* + * We should never find an existing key after + * we have detected one missing. + */ + o_rep[i].exist_key = key; + fatal = true; + } + } + c_rep[i].last_key = last_key; + l_rep[i].last_key = last_key; + o_rep[i].last_key = last_key; + testutil_checksys(fclose(fp) != 0); + print_missing(&c_rep[i], fname, "COLLECTION"); + print_missing(&l_rep[i], fname, "LOCAL"); + print_missing(&o_rep[i], fname, "OPLOG"); + } + testutil_check(conn->close(conn, NULL)); + if (!inmem && absent_coll) { + printf("COLLECTION: %" PRIu64 + " record(s) absent from %" PRIu64 "\n", + absent_coll, count); + fatal = true; + } + if (!inmem && absent_local) { + printf("LOCAL: %" PRIu64 " record(s) absent from %" PRIu64 "\n", + absent_local, count); + fatal = true; + } + if (!inmem && absent_oplog) { + printf("OPLOG: %" PRIu64 " record(s) absent from %" PRIu64 "\n", + absent_oplog, count); + fatal = true; + } + if (fatal) + return (EXIT_FAILURE); + printf("%" PRIu64 " records verified\n", count); + return (EXIT_SUCCESS); +} diff --git a/src/third_party/wiredtiger/test/csuite/schema_abort/smoke.sh b/src/third_party/wiredtiger/test/csuite/schema_abort/smoke.sh new file mode 100755 index 00000000000..41d702d40cd --- /dev/null +++ b/src/third_party/wiredtiger/test/csuite/schema_abort/smoke.sh @@ -0,0 +1,12 @@ +#! /bin/sh + +set -e + +# Smoke-test schema-abort as part of running "make check". + +$TEST_WRAPPER ./test_schema_abort -t 10 -T 5 +$TEST_WRAPPER ./test_schema_abort -m -t 10 -T 5 +$TEST_WRAPPER ./test_schema_abort -C -t 10 -T 5 +$TEST_WRAPPER ./test_schema_abort -C -m -t 10 -T 5 +$TEST_WRAPPER ./test_schema_abort -m -t 10 -T 5 -z +$TEST_WRAPPER ./test_schema_abort -m -t 10 -T 5 -x diff --git a/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c b/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c index 8a1781eae45..eb91dea0121 100644 --- a/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c +++ b/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c @@ -68,9 +68,10 @@ static char home[1024]; /* Program working dir */ #define RECORDS_FILE "records-%" PRIu32 #define STABLE_PERIOD 100 -static const char * const uri_local = "table:local"; -static const char * const uri_oplog = "table:oplog"; -static const char * const uri_collection = "table:collection"; +static const char * table_pfx = "table"; +static const char * const uri_local = "local"; +static const char * const uri_oplog = "oplog"; +static const char * const uri_collection = "collection"; static const char * const ckpt_file = "checkpoint_done"; @@ -210,7 +211,7 @@ thread_ckpt_run(void *arg) session, "use_timestamp=true")); testutil_check(td->conn->query_timestamp( td->conn, buf, "get=last_checkpoint")); - sscanf(buf, "%" SCNx64, &stable); + testutil_assert(sscanf(buf, "%" SCNx64, &stable) == 1); printf("Checkpoint %d complete at stable %" PRIu64 ".\n", i, stable); fflush(stdout); @@ -243,7 +244,7 @@ thread_run(void *arg) THREAD_DATA *td; uint64_t i, stable_ts; char cbuf[MAX_VAL], lbuf[MAX_VAL], obuf[MAX_VAL]; - char kname[64], tscfg[64]; + char kname[64], tscfg[64], uri[128]; bool use_prep; __wt_random_init(&rnd); @@ -283,19 +284,25 @@ thread_run(void *arg) /* * Open a cursor to each table. */ + testutil_check(__wt_snprintf( + uri, sizeof(uri), "%s:%s", table_pfx, uri_collection)); testutil_check(session->open_cursor(session, - uri_collection, NULL, NULL, &cur_coll)); + uri, NULL, NULL, &cur_coll)); + testutil_check(__wt_snprintf( + uri, sizeof(uri), "%s:%s", table_pfx, uri_local)); testutil_check(session->open_cursor(session, - uri_local, NULL, NULL, &cur_local)); + uri, NULL, NULL, &cur_local)); + testutil_check(__wt_snprintf( + uri, sizeof(uri), "%s:%s", table_pfx, uri_oplog)); oplog_session = NULL; if (use_prep) { testutil_check(td->conn->open_session( td->conn, NULL, NULL, &oplog_session)); testutil_check(session->open_cursor(oplog_session, - uri_oplog, NULL, NULL, &cur_oplog)); + uri, NULL, NULL, &cur_oplog)); } else testutil_check(session->open_cursor(session, - uri_oplog, NULL, NULL, &cur_oplog)); + uri, NULL, NULL, &cur_oplog)); /* * Write our portion of the key space until we're killed. @@ -407,7 +414,7 @@ run_workload(uint32_t nth) THREAD_DATA *td; wt_thread_t *thr; uint32_t ckpt_id, i, ts_id; - char envconf[512]; + char envconf[512], uri[128]; thr = dcalloc(nth+2, sizeof(*thr)); td = dcalloc(nth+2, sizeof(THREAD_DATA)); @@ -425,12 +432,18 @@ run_workload(uint32_t nth) /* * Create all the tables. */ - testutil_check(session->create(session, uri_collection, + testutil_check(__wt_snprintf( + uri, sizeof(uri), "%s:%s", table_pfx, uri_collection)); + testutil_check(session->create(session, uri, "key_format=S,value_format=u,log=(enabled=false)")); + testutil_check(__wt_snprintf( + uri, sizeof(uri), "%s:%s", table_pfx, uri_local)); testutil_check(session->create(session, - uri_local, "key_format=S,value_format=u")); + uri, "key_format=S,value_format=u")); + testutil_check(__wt_snprintf( + uri, sizeof(uri), "%s:%s", table_pfx, uri_oplog)); testutil_check(session->create(session, - uri_oplog, "key_format=S,value_format=u")); + uri, "key_format=S,value_format=u")); /* * Don't log the stable timestamp table so that we know what timestamp * was stored at the checkpoint. @@ -468,7 +481,7 @@ run_workload(uint32_t nth) */ fflush(stdout); for (i = 0; i <= ts_id; ++i) - testutil_check(__wt_thread_join(NULL, thr[i])); + testutil_check(__wt_thread_join(NULL, &thr[i])); /* * NOTREACHED */ @@ -574,7 +587,7 @@ main(int argc, char *argv[]) verify_only = false; working_dir = "WT_TEST.timestamp-abort"; - while ((ch = __wt_getopt(progname, argc, argv, "Ch:mT:t:vz")) != EOF) + while ((ch = __wt_getopt(progname, argc, argv, "Ch:LmT:t:vz")) != EOF) switch (ch) { case 'C': compat = true; @@ -582,6 +595,9 @@ main(int argc, char *argv[]) case 'h': working_dir = __wt_optarg; break; + case 'L': + table_pfx = "lsm"; + break; case 'm': inmem = true; break; @@ -603,7 +619,6 @@ main(int argc, char *argv[]) usage(); } argc -= __wt_optind; - argv += __wt_optind; if (argc != 0) usage(); @@ -714,12 +729,18 @@ main(int argc, char *argv[]) /* * Open a cursor on all the tables. */ + testutil_check(__wt_snprintf( + buf, sizeof(buf), "%s:%s", table_pfx, uri_collection)); testutil_check(session->open_cursor(session, - uri_collection, NULL, NULL, &cur_coll)); + buf, NULL, NULL, &cur_coll)); + testutil_check(__wt_snprintf( + buf, sizeof(buf), "%s:%s", table_pfx, uri_local)); testutil_check(session->open_cursor(session, - uri_local, NULL, NULL, &cur_local)); + buf, NULL, NULL, &cur_local)); + testutil_check(__wt_snprintf( + buf, sizeof(buf), "%s:%s", table_pfx, uri_oplog)); testutil_check(session->open_cursor(session, - uri_oplog, NULL, NULL, &cur_oplog)); + buf, NULL, NULL, &cur_oplog)); /* * Find the biggest stable timestamp value that was saved. @@ -728,7 +749,7 @@ main(int argc, char *argv[]) if (use_ts) { testutil_check( conn->query_timestamp(conn, buf, "get=recovery")); - sscanf(buf, "%" SCNx64, &stable_val); + testutil_assert(sscanf(buf, "%" SCNx64, &stable_val) == 1); printf("Got stable_val %" PRIu64 "\n", stable_val); } diff --git a/src/third_party/wiredtiger/test/csuite/timestamp_abort/smoke.sh b/src/third_party/wiredtiger/test/csuite/timestamp_abort/smoke.sh index f49ea10e5e1..661261eb1bb 100755 --- a/src/third_party/wiredtiger/test/csuite/timestamp_abort/smoke.sh +++ b/src/third_party/wiredtiger/test/csuite/timestamp_abort/smoke.sh @@ -5,6 +5,8 @@ set -e # Smoke-test timestamp-abort as part of running "make check". $TEST_WRAPPER ./test_timestamp_abort -t 10 -T 5 +#$TEST_WRAPPER ./test_timestamp_abort -t 10 -T 5 -L $TEST_WRAPPER ./test_timestamp_abort -m -t 10 -T 5 +#$TEST_WRAPPER ./test_timestamp_abort -m -t 10 -T 5 -L $TEST_WRAPPER ./test_timestamp_abort -C -t 10 -T 5 $TEST_WRAPPER ./test_timestamp_abort -C -m -t 10 -T 5 diff --git a/src/third_party/wiredtiger/test/csuite/truncated_log/main.c b/src/third_party/wiredtiger/test/csuite/truncated_log/main.c index be28a8d9500..1ed1b6e8157 100644 --- a/src/third_party/wiredtiger/test/csuite/truncated_log/main.c +++ b/src/third_party/wiredtiger/test/csuite/truncated_log/main.c @@ -262,7 +262,6 @@ main(int argc, char *argv[]) usage(); } argc -= __wt_optind; - argv += __wt_optind; if (argc != 0) usage(); diff --git a/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c b/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c index 5e6ebc0ab1a..bb9f293edaa 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c @@ -445,9 +445,9 @@ run_process(TEST_OPTS *opts, const char *prog, char *argv[], int *status) } /* -* subtest_error_handler -- -* Error event handler. -*/ + * subtest_error_handler -- + * Error event handler. + */ static int subtest_error_handler(WT_EVENT_HANDLER *handler, WT_SESSION *session, int error, const char *message) diff --git a/src/third_party/wiredtiger/test/csuite/wt4117_checksum/main.c b/src/third_party/wiredtiger/test/csuite/wt4117_checksum/main.c index 9a786e5d222..fff7b55bbf9 100644 --- a/src/third_party/wiredtiger/test/csuite/wt4117_checksum/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt4117_checksum/main.c @@ -44,65 +44,51 @@ static void run(void) { size_t len; - uint32_t crc32c; + uint32_t crc32c, (*func)(const void *, size_t); uint8_t *data; /* Allocate aligned memory for the data. */ data = dcalloc(100, sizeof(uint8_t)); + /* Get a pointer to the CRC32C function. */ + func = wiredtiger_crc32c_func(); + /* * Some simple known checksums. */ len = 1; - crc32c = wiredtiger_checksum_crc32c(data, len); + crc32c = func(data, len); check(crc32c, (uint32_t)0x527d5351, len, "nul x1"); len = 2; - crc32c = wiredtiger_checksum_crc32c(data, len); + crc32c = func(data, len); check(crc32c, (uint32_t)0xf16177d2, len, "nul x2"); len = 3; - crc32c = wiredtiger_checksum_crc32c(data, len); + crc32c = func(data, len); check(crc32c, (uint32_t)0x6064a37a, len, "nul x3"); len = 4; - crc32c = wiredtiger_checksum_crc32c(data, len); + crc32c = func(data, len); check(crc32c, (uint32_t)0x48674bc7, len, "nul x4"); len = strlen("123456789"); memcpy(data, "123456789", len); - crc32c = wiredtiger_checksum_crc32c(data, len); + crc32c = func(data, len); check(crc32c, (uint32_t)0xe3069283, len, "known string #1"); len = strlen("The quick brown fox jumps over the lazy dog"); memcpy(data, "The quick brown fox jumps over the lazy dog", len); - crc32c = wiredtiger_checksum_crc32c(data, len); + crc32c = func(data, len); check(crc32c, (uint32_t)0x22620404, len, "known string #2"); free(data); } int -main(int argc, char *argv[]) +main(void) { - TEST_OPTS *opts, _opts; - - opts = &_opts; - memset(opts, 0, sizeof(*opts)); - testutil_check(testutil_parse_opts(argc, argv, opts)); - testutil_make_work_dir(opts->home); - - /* - * The external API should work before the library configures itself, - * run before and after calling wiredtiger_open(). - */ - run(); - - testutil_check( - wiredtiger_open(opts->home, NULL, "create", &opts->conn)); - run(); - testutil_cleanup(opts); return (EXIT_SUCCESS); } diff --git a/src/third_party/wiredtiger/test/cursor_order/cursor_order.c b/src/third_party/wiredtiger/test/cursor_order/cursor_order.c index b2e847f880c..2cdbe4cb840 100644 --- a/src/third_party/wiredtiger/test/cursor_order/cursor_order.c +++ b/src/third_party/wiredtiger/test/cursor_order/cursor_order.c @@ -132,7 +132,6 @@ main(int argc, char *argv[]) } argc -= __wt_optind; - argv += __wt_optind; if (argc != 0) return (usage()); diff --git a/src/third_party/wiredtiger/test/cursor_order/cursor_order_ops.c b/src/third_party/wiredtiger/test/cursor_order/cursor_order_ops.c index 2ff4b5ed5fb..d110d513bfb 100644 --- a/src/third_party/wiredtiger/test/cursor_order/cursor_order_ops.c +++ b/src/third_party/wiredtiger/test/cursor_order/cursor_order_ops.c @@ -120,7 +120,7 @@ ops_start(SHARED_CONFIG *cfg) /* Wait for the threads. */ for (i = 0; i < cfg->reverse_scanners + cfg->append_inserters; ++i) - testutil_check(__wt_thread_join(NULL, tids[i])); + testutil_check(__wt_thread_join(NULL, &tids[i])); (void)gettimeofday(&stop, NULL); seconds = (stop.tv_sec - start.tv_sec) + diff --git a/src/third_party/wiredtiger/test/fops/fops.c b/src/third_party/wiredtiger/test/fops/fops.c index 99c333be4a0..96a60acbfa5 100644 --- a/src/third_party/wiredtiger/test/fops/fops.c +++ b/src/third_party/wiredtiger/test/fops/fops.c @@ -69,7 +69,7 @@ fop_start(u_int nthreads) /* Wait for the threads. */ for (i = 0; i < nthreads; ++i) - testutil_check(__wt_thread_join(NULL, tids[i])); + testutil_check(__wt_thread_join(NULL, &tids[i])); (void)gettimeofday(&stop, NULL); seconds = (stop.tv_sec - start.tv_sec) + diff --git a/src/third_party/wiredtiger/test/fops/t.c b/src/third_party/wiredtiger/test/fops/t.c index e748918a08a..d82b2c3f3f1 100644 --- a/src/third_party/wiredtiger/test/fops/t.c +++ b/src/third_party/wiredtiger/test/fops/t.c @@ -112,7 +112,6 @@ main(int argc, char *argv[]) } argc -= __wt_optind; - argv += __wt_optind; if (argc != 0) return (usage()); diff --git a/src/third_party/wiredtiger/test/format/config.c b/src/third_party/wiredtiger/test/format/config.c index 9df743cf056..33ef6864b64 100644 --- a/src/third_party/wiredtiger/test/format/config.c +++ b/src/third_party/wiredtiger/test/format/config.c @@ -615,10 +615,16 @@ config_lsm_reset(void) /* * LSM doesn't currently play nicely with timestamps, don't choose the - * pair unless forced to. Remove this code with WT-4067. + * pair unless forced to. If we turn off timestamps, make sure we turn + * off prepare as well, it requires timestamps. Remove this code with + * WT-4067. + * */ - if (!config_is_perm("transaction_timestamps")) + if (!config_is_perm("prepare") && + !config_is_perm("transaction_timestamps")) { + config_single("prepare=off", 0); config_single("transaction_timestamps=off", 0); + } } /* diff --git a/src/third_party/wiredtiger/test/format/config.h b/src/third_party/wiredtiger/test/format/config.h index c398c1a96b2..70e8165e97d 100644 --- a/src/third_party/wiredtiger/test/format/config.h +++ b/src/third_party/wiredtiger/test/format/config.h @@ -370,10 +370,6 @@ static CONFIG c[] = { "stress splits (#8)", /* 2% */ C_BOOL, 2, 0, 0, &g.c_timing_stress_split_8, NULL }, - { "timing_stress_split_9", - "stress splits (#9)", /* 2% */ - C_BOOL, 2, 0, 0, &g.c_timing_stress_split_9, NULL }, - { "transaction_timestamps", /* 10% */ "enable transaction timestamp support", C_BOOL, 10, 0, 0, &g.c_txn_timestamps, NULL }, diff --git a/src/third_party/wiredtiger/test/format/format.h b/src/third_party/wiredtiger/test/format/format.h index 1406d2b3fb5..a83aa0d2dcb 100644 --- a/src/third_party/wiredtiger/test/format/format.h +++ b/src/third_party/wiredtiger/test/format/format.h @@ -122,7 +122,16 @@ typedef struct { WT_RAND_STATE rnd; /* Global RNG state */ - pthread_rwlock_t prepare_lock; /* Prepare running */ + /* + * Prepare will return an error if the prepare timestamp is less than + * any active read timestamp. Lock across allocating prepare and read + * timestamps. + * + * We get the last committed timestamp periodically in order to update + * the oldest timestamp, that requires locking out transactional ops + * that set a timestamp. + */ + pthread_rwlock_t ts_lock; uint64_t timestamp; /* Counter for timestamps */ @@ -222,7 +231,6 @@ typedef struct { uint32_t c_timing_stress_split_6; uint32_t c_timing_stress_split_7; uint32_t c_timing_stress_split_8; - uint32_t c_timing_stress_split_9; uint32_t c_truncate; uint32_t c_txn_freq; uint32_t c_txn_timestamps; @@ -283,9 +291,6 @@ typedef struct { WT_RAND_STATE rnd; /* thread RNG state */ - uint64_t commit_timestamp; /* last committed timestamp */ - uint64_t read_timestamp; /* read timestamp */ - volatile bool quit; /* thread should quit */ uint64_t ops; /* total operations */ @@ -382,3 +387,27 @@ mmrand(WT_RAND_STATE *rnd, u_int min, u_int max) v += min; return (v); } + +static inline void +random_sleep(WT_RAND_STATE *rnd, u_int max_seconds) +{ + uint64_t i, micro_seconds; + + /* + * We need a fast way to choose a sleep time. We want to sleep a short + * period most of the time, but occasionally wait longer. Divide the + * maximum period of time into 10 buckets (where bucket 0 doesn't sleep + * at all), and roll dice, advancing to the next bucket 50% of the time. + * That means we'll hit the maximum roughly every 1K calls. + */ + for (i = 0;;) + if (rng(rnd) & 0x1 || ++i > 9) + break; + + if (i == 0) + __wt_yield(); + else { + micro_seconds = (uint64_t)max_seconds * WT_MILLION; + __wt_sleep(0, i * (micro_seconds / 10)); + } +} diff --git a/src/third_party/wiredtiger/test/format/ops.c b/src/third_party/wiredtiger/test/format/ops.c index 27aa118a4ef..705594a2156 100644 --- a/src/third_party/wiredtiger/test/format/ops.c +++ b/src/third_party/wiredtiger/test/format/ops.c @@ -207,7 +207,7 @@ wts_ops(int lastrun) case TINFO_COMPLETE: tinfo->state = TINFO_JOINED; testutil_check( - __wt_thread_join(NULL, tinfo->tid)); + __wt_thread_join(NULL, &tinfo->tid)); break; case TINFO_JOINED: break; @@ -252,17 +252,17 @@ wts_ops(int lastrun) /* Wait for the other threads. */ g.workers_finished = true; if (g.c_alter) - testutil_check(__wt_thread_join(NULL, alter_tid)); + testutil_check(__wt_thread_join(NULL, &alter_tid)); if (g.c_backups) - testutil_check(__wt_thread_join(NULL, backup_tid)); + testutil_check(__wt_thread_join(NULL, &backup_tid)); if (g.c_checkpoint_flag == CHECKPOINT_ON) - testutil_check(__wt_thread_join(NULL, checkpoint_tid)); + testutil_check(__wt_thread_join(NULL, &checkpoint_tid)); if (g.c_compact) - testutil_check(__wt_thread_join(NULL, compact_tid)); + testutil_check(__wt_thread_join(NULL, &compact_tid)); if (!SINGLETHREADED && g.c_long_running_txn) - testutil_check(__wt_thread_join(NULL, lrt_tid)); + testutil_check(__wt_thread_join(NULL, &lrt_tid)); if (g.c_txn_timestamps) - testutil_check(__wt_thread_join(NULL, timestamp_tid)); + testutil_check(__wt_thread_join(NULL, ×tamp_tid)); g.workers_finished = false; if (g.logging != 0) { @@ -497,13 +497,10 @@ snap_check(WT_CURSOR *cursor, static void begin_transaction(TINFO *tinfo, WT_SESSION *session, u_int *iso_configp) { + WT_DECL_RET; u_int v; - int ret; + char buf[64]; const char *config; - char config_buf[64]; - bool locked; - - locked = false; if ((v = g.c_isolation_flag) == ISOLATION_RANDOM) v = mmrand(&tinfo->rnd, 1, 3); @@ -525,9 +522,8 @@ begin_transaction(TINFO *tinfo, WT_SESSION *session, u_int *iso_configp) *iso_configp = v; /* - * Keep trying to start a new transaction if it's timing out - we - * know there aren't any resources pinned so it should succeed - * eventually. + * Keep trying to start a new transaction if it's timing out - we know + * there aren't any resources pinned so it should succeed eventually. */ while ((ret = session->begin_transaction(session, config)) == WT_CACHE_FULL) @@ -535,61 +531,25 @@ begin_transaction(TINFO *tinfo, WT_SESSION *session, u_int *iso_configp) testutil_check(ret); if (v == ISOLATION_SNAPSHOT && g.c_txn_timestamps) { - /* Avoid starting a new reader when a prepare is in progress. */ - if (g.c_prepare) { - testutil_check(pthread_rwlock_rdlock(&g.prepare_lock)); - locked = true; - } - /* - * Set the thread's read timestamp to the current value before - * allocating a new read timestamp. This guarantees the oldest - * timestamp won't move past the allocated timestamp before the - * transaction uses it. + * Prepare returns an error if the prepare timestamp is less + * than any active read timestamp, single-thread transaction + * prepare and begin. + * + * Lock out the oldest timestamp update. */ - tinfo->read_timestamp = g.timestamp; - tinfo->read_timestamp = __wt_atomic_addv64(&g.timestamp, 1); - testutil_check(__wt_snprintf( - config_buf, sizeof(config_buf), - "read_timestamp=%" PRIx64, tinfo->read_timestamp)); + testutil_check(pthread_rwlock_wrlock(&g.ts_lock)); - testutil_check( - session->timestamp_transaction(session, config_buf)); - - /* - * It's OK for the oldest timestamp to move past a running - * query, clear the thread's read timestamp, it no longer needs - * to be pinned. - */ - tinfo->read_timestamp = 0; + testutil_check(__wt_snprintf(buf, sizeof(buf), + "read_timestamp=%" PRIx64, + __wt_atomic_addv64(&g.timestamp, 1))); + testutil_check(session->timestamp_transaction(session, buf)); - if (locked) - testutil_check(pthread_rwlock_unlock(&g.prepare_lock)); + testutil_check(pthread_rwlock_unlock(&g.ts_lock)); } } /* - * set_commit_timestamp -- - * Return the next commit timestamp. - */ -static uint64_t -set_commit_timestamp(TINFO *tinfo) -{ - /* - * If the thread's commit timestamp hasn't been set yet, update it with - * the current value to prevent the oldest timestamp moving past our - * allocated timestamp before the commit completes. The sequence where - * it's already set is after prepare, in which case we can't let the - * oldest timestamp move past either the prepare or commit timestamps. - * - * Note the barrier included in the atomic call ensures proper ordering. - */ - if (tinfo->commit_timestamp == 0) - tinfo->commit_timestamp = g.timestamp; - return (__wt_atomic_addv64(&g.timestamp, 1)); -} - -/* * commit_transaction -- * Commit a transaction. */ @@ -597,25 +557,20 @@ static void commit_transaction(TINFO *tinfo, WT_SESSION *session) { uint64_t ts; - char config_buf[64]; + char buf[64]; ++tinfo->commit; if (g.c_txn_timestamps) { - ts = set_commit_timestamp(tinfo); + /* Lock out the oldest timestamp update. */ + testutil_check(pthread_rwlock_wrlock(&g.ts_lock)); + + ts = __wt_atomic_addv64(&g.timestamp, 1); testutil_check(__wt_snprintf( - config_buf, sizeof(config_buf), - "commit_timestamp=%" PRIx64, ts)); - testutil_check( - session->timestamp_transaction(session, config_buf)); - /* - * Clear the thread's active timestamp: it no longer needs to - * be pinned. Don't let the compiler re-order this statement, - * if we were to race with the timestamp thread, it might see - * our thread update before the commit_timestamp is set for the - * transaction. - */ - WT_PUBLISH(tinfo->commit_timestamp, 0); + buf, sizeof(buf), "commit_timestamp=%" PRIx64, ts)); + testutil_check(session->timestamp_transaction(session, buf)); + + testutil_check(pthread_rwlock_unlock(&g.ts_lock)); } testutil_check(session->commit_transaction(session, NULL)); } @@ -630,15 +585,6 @@ rollback_transaction(TINFO *tinfo, WT_SESSION *session) ++tinfo->rollback; testutil_check(session->rollback_transaction(session, NULL)); - - /* - * Clear the thread's active timestamp: it no longer needs to be pinned. - * Don't let the compiler re-order this statement, if we were to race - * with the timestamp thread, it might see our thread update before the - * transaction commit completes. - */ - if (g.c_txn_timestamps) - WT_PUBLISH(tinfo->commit_timestamp, 0); } /* @@ -650,34 +596,28 @@ prepare_transaction(TINFO *tinfo, WT_SESSION *session) { WT_DECL_RET; uint64_t ts; - char config_buf[64]; + char buf[64]; - /* Skip if no timestamp has yet been set. */ - if (g.timestamp == 0) - return (0); ++tinfo->prepare; /* - * Synchronize prepare call with begin transaction to prevent a new - * reader creeping in. - */ - testutil_check(pthread_rwlock_wrlock(&g.prepare_lock)); - - /* * Prepare timestamps must be less than or equal to the eventual commit * timestamp. Set the prepare timestamp to whatever the global value is * now. The subsequent commit will increment it, ensuring correctness. * - * Prepare will return error if the prepare timestamp is less than any - * active read timestamp. + * Prepare returns an error if the prepare timestamp is less than any + * active read timestamp, single-thread transaction prepare and begin. + * + * Lock out the oldest timestamp update. */ - ts = set_commit_timestamp(tinfo); - testutil_check(__wt_snprintf( - config_buf, sizeof(config_buf), "prepare_timestamp=%" PRIx64, ts)); - ret = session->prepare_transaction(session, config_buf); + testutil_check(pthread_rwlock_wrlock(&g.ts_lock)); - testutil_check(pthread_rwlock_unlock(&g.prepare_lock)); + ts = __wt_atomic_addv64(&g.timestamp, 1); + testutil_check(__wt_snprintf( + buf, sizeof(buf), "prepare_timestamp=%" PRIx64, ts)); + ret = session->prepare_transaction(session, buf); + testutil_check(pthread_rwlock_unlock(&g.ts_lock)); return (ret); } diff --git a/src/third_party/wiredtiger/test/format/t.c b/src/third_party/wiredtiger/test/format/t.c index d7b9add1f14..a1e9736ab60 100644 --- a/src/third_party/wiredtiger/test/format/t.c +++ b/src/third_party/wiredtiger/test/format/t.c @@ -104,7 +104,6 @@ main(int argc, char *argv[]) default: usage(); } - argc -= __wt_optind; argv += __wt_optind; /* Initialize the global RNG. */ @@ -170,7 +169,7 @@ main(int argc, char *argv[]) testutil_check(pthread_rwlock_init(&g.append_lock, NULL)); testutil_check(pthread_rwlock_init(&g.backup_lock, NULL)); testutil_check(pthread_rwlock_init(&g.death_lock, NULL)); - testutil_check(pthread_rwlock_init(&g.prepare_lock, NULL)); + testutil_check(pthread_rwlock_init(&g.ts_lock, NULL)); printf("%s: process %" PRIdMAX "\n", progname, (intmax_t)getpid()); while (++g.run_cnt <= g.c_runs || g.c_runs == 0 ) { @@ -268,7 +267,7 @@ main(int argc, char *argv[]) testutil_check(pthread_rwlock_destroy(&g.append_lock)); testutil_check(pthread_rwlock_destroy(&g.backup_lock)); testutil_check(pthread_rwlock_destroy(&g.death_lock)); - testutil_check(pthread_rwlock_destroy(&g.prepare_lock)); + testutil_check(pthread_rwlock_destroy(&g.ts_lock)); config_clear(); diff --git a/src/third_party/wiredtiger/test/format/util.c b/src/third_party/wiredtiger/test/format/util.c index cf75c98129a..23b198c05af 100644 --- a/src/third_party/wiredtiger/test/format/util.c +++ b/src/third_party/wiredtiger/test/format/util.c @@ -588,72 +588,47 @@ WT_THREAD_RET timestamp(void *arg) { WT_CONNECTION *conn; + WT_DECL_RET; WT_SESSION *session; - TINFO **tinfo_list, *tinfo; - time_t last, now; - uint64_t oldest_timestamp, this_ts, usecs; - uint32_t i; - char config_buf[64]; - - tinfo_list = arg; + char buf[64]; + bool done; + (void)(arg); conn = g.wts_conn; - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - __wt_seconds((WT_SESSION_IMPL *)session, &last); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); - /* - * Update the oldest timestamp every 100 transactions, but at least - * once every 15 seconds. - */ - while (!g.workers_finished) { - /* - * Find the lowest in-use timestamp. The timestamp thread starts - * before the operational threads, wait for them. - */ - oldest_timestamp = g.timestamp; - for (i = 0; i < g.c_threads; ++i) { - tinfo = tinfo_list[i]; - this_ts = tinfo->commit_timestamp; - if (this_ts != 0 && this_ts < oldest_timestamp) - oldest_timestamp = this_ts; - this_ts = tinfo->read_timestamp; - if (this_ts != 0 && this_ts < oldest_timestamp) - oldest_timestamp = this_ts; - } + testutil_check( + __wt_snprintf(buf, sizeof(buf), "%s", "oldest_timestamp=")); + /* Update the oldest timestamp at least once every 15 seconds. */ + done = false; + do { /* - * Don't try to update until we've committed some transactions - * with timestamps. + * Do a final bump of the oldest timestamp as part of shutting + * down the worker threads, otherwise recent operations can + * prevent verify from running. */ - if (oldest_timestamp == 0) { - __wt_sleep(1, 0); - continue; - } + if (g.workers_finished) + done = true; + else + random_sleep(&g.rnd, 15); /* - * If less than 100 transactions out of date, wait up to 15 - * seconds before updating. + * Lock out transaction timestamp operations. The lock acts as a + * barrier ensuring we've checked if the workers have finished, + * we don't want that line reordered. */ - WT_READ_BARRIER(); - testutil_assert(oldest_timestamp <= g.timestamp); - if (g.timestamp - oldest_timestamp < 100) { - __wt_seconds((WT_SESSION_IMPL *)session, &now); - if (difftime(now, last) < 15) { - __wt_sleep(1, 0); - continue; - } - } + testutil_check(pthread_rwlock_wrlock(&g.ts_lock)); - testutil_check(__wt_snprintf( - config_buf, sizeof(config_buf), - "oldest_timestamp=%" PRIx64, oldest_timestamp)); - testutil_check(conn->set_timestamp(conn, config_buf)); - __wt_seconds((WT_SESSION_IMPL *)session, &last); + ret = conn->query_timestamp(conn, + buf + strlen("oldest_timestamp="), "get=all_committed"); + testutil_assert(ret == 0 || ret == WT_NOTFOUND); + if (ret == 0) + testutil_check(conn->set_timestamp(conn, buf)); - usecs = mmrand(NULL, 5, 40); - __wt_sleep(0, usecs); - } + testutil_check(pthread_rwlock_unlock(&g.ts_lock)); + } while (!done); testutil_check(session->close(session, NULL)); return (WT_THREAD_RET_VALUE); diff --git a/src/third_party/wiredtiger/test/format/wts.c b/src/third_party/wiredtiger/test/format/wts.c index 8040142aa19..30b910435d4 100644 --- a/src/third_party/wiredtiger/test/format/wts.c +++ b/src/third_party/wiredtiger/test/format/wts.c @@ -264,8 +264,6 @@ wts_open(const char *home, bool set_api, WT_CONNECTION **connp) CONFIG_APPEND(p, ",split_7"); if (g.c_timing_stress_split_8) CONFIG_APPEND(p, ",split_8"); - if (g.c_timing_stress_split_9) - CONFIG_APPEND(p, ",split_9"); CONFIG_APPEND(p, "]"); /* Extensions. */ @@ -544,7 +542,6 @@ wts_verify(const char *tag) WT_CONNECTION *conn; WT_DECL_RET; WT_SESSION *session; - char config_buf[64]; if (g.c_verify == 0) return; @@ -557,17 +554,6 @@ wts_verify(const char *tag) (void)g.wt_api->msg_printf(g.wt_api, session, "=============== verify start ==============="); - if (g.c_txn_timestamps && g.timestamp > 0) { - /* - * Bump the oldest timestamp, otherwise recent operations can - * prevent verify from running. - */ - testutil_check(__wt_snprintf( - config_buf, sizeof(config_buf), - "oldest_timestamp=%" PRIx64, g.timestamp)); - testutil_check(conn->set_timestamp(conn, config_buf)); - } - /* * Verify can return EBUSY if the handle isn't available. Don't yield * and retry, in the case of LSM, the handle may not be available for diff --git a/src/third_party/wiredtiger/test/huge/huge.c b/src/third_party/wiredtiger/test/huge/huge.c index 11d6bbdc312..18bf873ff0b 100644 --- a/src/third_party/wiredtiger/test/huge/huge.c +++ b/src/third_party/wiredtiger/test/huge/huge.c @@ -171,7 +171,6 @@ main(int argc, char *argv[]) usage(); } argc -= __wt_optind; - argv += __wt_optind; if (argc != 0) usage(); diff --git a/src/third_party/wiredtiger/test/manydbs/manydbs.c b/src/third_party/wiredtiger/test/manydbs/manydbs.c index a6574d21d72..daf19302828 100644 --- a/src/third_party/wiredtiger/test/manydbs/manydbs.c +++ b/src/third_party/wiredtiger/test/manydbs/manydbs.c @@ -148,7 +148,6 @@ main(int argc, char *argv[]) usage(); } argc -= __wt_optind; - argv += __wt_optind; if (argc != 0) usage(); diff --git a/src/third_party/wiredtiger/test/readonly/readonly.c b/src/third_party/wiredtiger/test/readonly/readonly.c index b5342831320..7a84f662029 100644 --- a/src/third_party/wiredtiger/test/readonly/readonly.c +++ b/src/third_party/wiredtiger/test/readonly/readonly.c @@ -194,7 +194,6 @@ main(int argc, char *argv[]) usage(); } argc -= __wt_optind; - argv += __wt_optind; if (argc != 0) usage(); diff --git a/src/third_party/wiredtiger/test/salvage/salvage.c b/src/third_party/wiredtiger/test/salvage/salvage.c index 9c8a90d37b9..3517405c6ac 100644 --- a/src/third_party/wiredtiger/test/salvage/salvage.c +++ b/src/third_party/wiredtiger/test/salvage/salvage.c @@ -97,7 +97,6 @@ main(int argc, char *argv[]) return (usage()); } argc -= __wt_optind; - argv += __wt_optind; if (argc != 0) return (usage()); @@ -670,7 +669,7 @@ empty(int cnt) if (page_type == WT_PAGE_COL_FIX) for (i = 0; i < cnt; ++i) - fputs("\\00\n", res_fp); + CHECK(fputs("\\00\n", res_fp)); } /* diff --git a/src/third_party/wiredtiger/test/suite/run.py b/src/third_party/wiredtiger/test/suite/run.py index 6b668ad3e07..74fa259c3c4 100644 --- a/src/third_party/wiredtiger/test/suite/run.py +++ b/src/third_party/wiredtiger/test/suite/run.py @@ -238,7 +238,7 @@ if __name__ == '__main__': tests = unittest.TestSuite() # Turn numbers and ranges into test module names - preserve = timestamp = debug = dryRun = gdbSub = longtest = False + preserve = timestamp = debug = dryRun = gdbSub = lldbSub = longtest = False parallel = 0 configfile = None configwrite = False @@ -269,6 +269,9 @@ if __name__ == '__main__': if option == '-gdb' or option == 'g': gdbSub = True continue + if option == '-lldb': + lldbSub = True + continue if option == '-help' or option == 'h': usage() sys.exit(0) @@ -323,7 +326,7 @@ if __name__ == '__main__': # All global variables should be set before any test classes are loaded. # That way, verbose printing can be done at the class definition level. - wttest.WiredTigerTestCase.globalSetup(preserve, timestamp, gdbSub, + wttest.WiredTigerTestCase.globalSetup(preserve, timestamp, gdbSub, lldbSub, verbose, wt_builddir, dirarg, longtest) diff --git a/src/third_party/wiredtiger/test/suite/suite_subprocess.py b/src/third_party/wiredtiger/test/suite/suite_subprocess.py index c2e9d99f691..4b0f6823e06 100644..100755 --- a/src/third_party/wiredtiger/test/suite/suite_subprocess.py +++ b/src/third_party/wiredtiger/test/suite/suite_subprocess.py @@ -26,7 +26,7 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import os, subprocess, sys +import os, re, subprocess, sys from run import wt_builddir from wttest import WiredTigerTestCase @@ -79,23 +79,51 @@ class suite_subprocess: print '********************************' self.fail('ERROR found in output file: ' + filename) + # If the string is of the form '/.../', then return just the embedded + # pattern, otherwise, return None + def convert_to_pattern(self, s): + if len(s) >= 2 and s[0] == '/' and s[-1] == '/': + return s[1:-1] + else: + return None + def check_file_content(self, filename, expect): with open(filename, 'r') as f: got = f.read(len(expect) + 100) self.assertEqual(got, expect, filename + ': does not contain expected:\n\'' + expect + '\', but contains:\n\'' + got + '\'.') - def check_file_contains(self, filename, expect): + def check_file_contains_one_of(self, filename, expectlist): """ Check that the file contains the expected string in the first 100K bytes """ maxbytes = 1024*100 with open(filename, 'r') as f: got = f.read(maxbytes) - if not (expect in got): + found = False + for expect in expectlist: + pat = self.convert_to_pattern(expect) + if pat == None: + if expect in got: + found = True + break + else: + if re.search(pat, got): + found = True + break + if not found: + if len(expectlist) == 1: + expect = '\'' + expectlist[0] + '\'' + else: + expect = str(expectlist) + gotstr = '\'' + \ + (got if len(got) < 1000 else (got[0:1000] + '...')) + '\'' if len(got) >= maxbytes: - self.fail(filename + ': does not contain expected \'' + expect + '\', or output is too large') + self.fail(filename + ': does not contain expected ' + expect + ', or output is too large, got ' + gotstr) else: - self.fail(filename + ': does not contain expected \'' + expect + '\'') + self.fail(filename + ': does not contain expected ' + expect + ', got ' + gotstr) + + def check_file_contains(self, filename, expect): + self.check_file_contains_one_of(filename, [expect]) def check_empty_file(self, filename): """ @@ -165,6 +193,8 @@ class suite_subprocess: procargs = [ wtexe ] if self._gdbSubprocess: procargs = [ "gdb", "--args" ] + procargs + elif self._lldbSubprocess: + procargs = [ "lldb", "--" ] + procargs procargs.extend(args) if self._gdbSubprocess: infilepart = "" @@ -177,6 +207,17 @@ class suite_subprocess: ">" + wtoutname + " 2>" + wterrname print "*********************************************" returncode = subprocess.call(procargs) + elif self._lldbSubprocess: + infilepart = "" + if infilename != None: + infilepart = "<" + infilename + " " + print str(procargs) + print "*********************************************" + print "**** Run 'wt' via: run " + \ + " ".join(procargs[3:]) + infilepart + \ + ">" + wtoutname + " 2>" + wterrname + print "*********************************************" + returncode = subprocess.call(procargs) elif infilename: with open(infilename, "r") as wtin: returncode = subprocess.call( diff --git a/src/third_party/wiredtiger/test/suite/test_txn19.py b/src/third_party/wiredtiger/test/suite/test_txn19.py new file mode 100755 index 00000000000..1bb93a164f2 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_txn19.py @@ -0,0 +1,344 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2018 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_txn19.py +# Transactions: test recovery with corrupted log files +# + +import fnmatch, os, shutil, time +from wtscenario import make_scenarios +from suite_subprocess import suite_subprocess +import wiredtiger, wttest + +# This test uses an artificially small log file limit, and creates +# large records so two fit into a log file. This allows us to test +# both the case when corruption happens at the beginning of a log file +# (an even number of records have been created), and when corruption +# happens in the middle of a log file (with an odd number of records). + +def corrupt(fname, truncate, offset, writeit): + with open(fname, 'r+') as log: + if offset: + if offset < 0: # Negative offset means seek to the end + log.seek(0, 2) + else: + log.seek(offset) + if truncate: + log.truncate() + if writeit: + log.write(writeit) + +class test_txn19(wttest.WiredTigerTestCase, suite_subprocess): + base_config = 'log=(archive=false,enabled,file_max=100K),' + \ + 'transaction_sync=(enabled,method=none)' + conn_config = base_config + corruption_type = [ + ('removal', dict(kind='removal', f=lambda fname: + os.remove(fname))), + ('truncate', dict(kind='truncate', f=lambda fname: + corrupt(fname, True, 0, None))), + ('truncate-middle', dict(kind='truncate-middle', f=lambda fname: + corrupt(fname, True, 1024 * 25, None))), + ('zero-begin', dict(kind='zero', f=lambda fname: + corrupt(fname, False, 0, '\0' * 4096))), + ('zero-trunc', dict(kind='zero', f=lambda fname: + corrupt(fname, True, 0, '\0' * 4096))), + ('zero-end', dict(kind='zero-end', f=lambda fname: + corrupt(fname, False, -1, '\0' * 4096))), + ('garbage-begin', dict(kind='garbage-begin', f=lambda fname: + corrupt(fname, False, 0, 'Bad!' * 1024))), + ('garbage-middle', dict(kind='garbage-middle', f=lambda fname: + corrupt(fname, False, 1024 * 25, 'Bad!' * 1024))), + ('garbage-end', dict(kind='garbage-end', f=lambda fname: + corrupt(fname, False, -1, 'Bad!' * 1024))), + ] + # The list comprehension below expands each entry in the integer tuple + # list to a scenario. For example, (3, 4, 2) expands to: + # ('corrupt=[3,4],checkpoint=2', dict(corruptpos=3, corruptpos2=4, chkpt=2)) + # + # Each number corresponds to a log file, so for this example, we have + # corruption in log file 3 (using the style of corruption from + # corruption_type), there is a second corruption in log file 4, + # and there is a checkpoint in log file 2. A value of 0 means no + # corruption or checkpoint. + corruption_pos = [ + ('corrupt=[' + str(x) + ',' + str(y) + '],checkpoint=' + str(z), + dict(corruptpos=x,corruptpos2=y,chkpt=z)) for (x,y,z) in ( + (0, 0, 0), (0, 0, 2), (6, 0, 0), (6, 0, 3), (3, 0, 0), + (3, 0, 2), (3, 4, 2), (3, 5, 2), (3, 0, 4))] + nrecords = [('nrecords=10', dict(nrecords=10)), + ('nrecords=11', dict(nrecords=11))] + + # This function prunes out unnecessary or problematic test cases + # from the list of scenarios. + def includeFunc(name, dictarg): + kind = dictarg['kind'] + corruptpos = dictarg['corruptpos'] + chkpt = dictarg['chkpt'] + + # corruptpos == 0 indicates there is no corruption. + # (i.e. corrupt log file 0, which doesn't exist) + # We do want to test the case of no corruption, but we don't + # need to try it against every type of corruption, only one. + if corruptpos == 0: + return kind == 'removal' + + # NOTE: + # The removal or truncation of a middle log file (not first or last) + # that would be used in recovery is not currently handled gracefully. + if (kind == 'removal' or kind == 'truncate') and \ + corruptpos != 6 and corruptpos > chkpt: + return False + + # All the other cases are valid + return True + + scenarios = make_scenarios( + corruption_type, corruption_pos, nrecords, + include=includeFunc, prune=20, prunelong=1000) + + uri = 'table:test_txn19' + create_params = 'key_format=i,value_format=S' + + # Return the log file number that contains the given record + # number. In this test, two records fit into each log file, and + # after each even record is written, a new log file is created + # (having no records initially). The last log file is this + # (nrecords/2 + 1), given that we start with log 1. + def record_to_logfile(self, recordnum): + return recordnum / 2 + 1 + + # Returns the first record number in a log file. + def logfile_to_record(self, logfile): + return (logfile - 1) * 2 + + # Return true if the log file is corrupted. + # If not corrupted, the log file will produce no errors, + # and all the records originally written should be recovered. + def corrupted(self): + # Corruptpos == 0 means to do no corruption in any log file + if self.corruptpos == 0: + return False + + # Adding zeroes to the end of a log file is indistinguishable + # from having a log file that is preallocated that has not been + # totally filled. One might argue that if this does not occur + # in the final log file, it could/should have been truncated. + # At any rate, we consider this particular corruption to be benign. + if self.kind == 'zero-end': + return False + return True + + def show_logs(self, homedir, msg): + loglist = [] + for i in range(0, 10): + basename = 'WiredTigerLog.000000000' + str(i) + fullname = homedir + os.sep + basename + if os.path.isfile(fullname): + loglist.append(i) + if os.stat(fullname).st_size == 0: + self.tty('LOGS ' + msg + ': ' + str(i) + ' is empty') + self.tty('LOGS ' + msg + ': ' + str(loglist)) + + def copy_for_crash_restart(self, olddir, newdir): + ''' Simulate a crash from olddir and restart in newdir. ''' + # with the connection still open, copy files to new directory + shutil.rmtree(newdir, ignore_errors=True) + os.mkdir(newdir) + for fname in os.listdir(olddir): + fullname = os.path.join(olddir, fname) + # Skip lock file on Windows since it is locked + if os.path.isfile(fullname) and \ + "WiredTiger.lock" not in fullname and \ + "Tmplog" not in fullname and \ + "Preplog" not in fullname: + shutil.copy(fullname, newdir) + + # Generate a value that is a bit over half the size of the log file. + def valuegen(self, i): + return str(i) + 'A' * (1024 * 60) # ~60K + + # Insert a list of keys + def inserts(self, keylist): + c = self.session.open_cursor(self.uri) + for i in keylist: + if self.chkpt > 0 and self.logfile_to_record(self.chkpt) == i: + self.session.checkpoint() + c[i] = self.valuegen(i) + c.close() + + def checks(self, expectlist): + c = self.session.open_cursor(self.uri, None, None) + gotlist = [] + for key, value in c: + gotlist.append(key) + self.assertEqual(self.valuegen(key), value) + self.assertEqual(expectlist, gotlist) + c.close() + + def log_number_to_file_name(self, homedir, n): + self.assertLess(n, 10) # assuming 1 digit + return homedir + os.sep + 'WiredTigerLog.000000000' + str(n) + + def corrupt_log(self, homedir): + if not self.corrupted(): + return + self.f(self.log_number_to_file_name(homedir, self.corruptpos)) + + # Corrupt a second log file if needed + if self.corruptpos2 != 0: + self.f(self.log_number_to_file_name(homedir, self.corruptpos2)) + + def corrupt_last_file(self): + return self.corruptpos == self.record_to_logfile(self.nrecords) + + # Corruption past the last written record in a log file can sometimes + # be detected. In our test case, the last log file has zero or one large + # 60K record written into it, but it is presized to 100K. Corruption + # at the end of this file creates a hole, and the corruption starts + # a new log record, where it can be detected as phony. Similarly, + # corruption in the "middle" of the last file (actually the 25K point) + # can be detected if there aren't any of the insert records in the file. + def corrupt_hole_in_last_file(self): + return self.corrupt_last_file() and \ + ((self.kind == 'garbage-middle' and self.nrecords % 2 == 0) or \ + self.kind == 'garbage-end') + + # Return true iff the log has been damaged in a way that is not detected + # as a corruption. WiredTiger must be lenient about damage in any log + # file, because a partial log record written just before a crash is in + # most cases indistinguishable from a corruption. If the beginning of + # the file is mangled, that is always an unexpected corruption. Situations + # where we cannot reliably detect corruption include: + # - removal of the last log + # - certain corruptions at the beginning of a log record (adding garbage + # at the end of a log file can trigger this). + def log_corrupt_but_valid(self): + if self.corrupt_last_file() and self.kind == 'removal': + return True + # Certain corruptions in the last file can be detected. + if self.corrupt_hole_in_last_file(): + return False + if self.kind == 'truncate-middle' or \ + self.kind == 'garbage-end': + return True + return False + + # For this test, at least, salvage identifies and fixes all + # recovery failures. + def expect_salvage_messages(self): + return self.expect_recovery_failure() + + def expect_recovery_failure(self): + return self.corrupted() and \ + self.corruptpos >= self.chkpt and \ + not self.log_corrupt_but_valid() + + def recovered_records(self): + if not self.corrupted() or self.chkpt > self.corruptpos: + return self.nrecords + if self.kind == 'garbage-end': + # All records in the corrupt file will be found. + found = self.logfile_to_record(self.corruptpos + 1) + else: + found = self.logfile_to_record(self.corruptpos) + return min(found, self.nrecords) + + def test_corrupt_log(self): + ''' Corrupt the log and restart with different kinds of recovery ''' + + # This test creates some data, then simulates a crash with corruption. + # Then does a restart with recovery, then starts again with salvage, + # and finally starts again with recovery (adding new records). + + self.session.create(self.uri, self.create_params) + self.inserts([x for x in range(0, self.nrecords)]) + newdir = "RESTART" + self.copy_for_crash_restart(self.home, newdir) + self.close_conn() + #self.show_logs(newdir, 'before corruption') + self.corrupt_log(newdir) + #self.show_logs(newdir, 'after corruption') + salvage_config = self.base_config + ',log=(recover=salvage)' + errfile = 'list.err' + outfile = 'list.out' + expect_fail = self.expect_recovery_failure() + + # In cases of corruption, we cannot always call wiredtiger_open + # directly, because there may be a panic, and abort() is called + # in diagnostic mode which terminates the Python interpreter. + # + # Running any wt command externally to Python allows + # us to observe the failure or success safely. + # Use -R to force recover=on, which is the default for + # wiredtiger_open, (wt utilities normally have recover=error) + self.runWt(['-h', newdir, '-C', self.base_config, '-R', 'list'], + errfilename=errfile, outfilename=outfile, failure=expect_fail, + closeconn=False) + + if expect_fail: + self.check_file_contains_one_of(errfile, + ['/log file.*corrupted/', 'WT_ERROR: non-specific WiredTiger error']) + else: + self.check_empty_file(errfile) + self.check_file_contains(outfile, self.uri) + + found_records = self.recovered_records() + expect = [x for x in range(0, found_records)] + + # If we are salvaging, expect an informational message + if self.expect_salvage_messages(): + errpat = '.*' + # Possible messages: + # salvage: log files x-y truncated at beginning + # salvage: log file x truncated at beginning + # salvage: log file x truncated + # salvage: log file x removed + outpat = 'salvage: log file' + else: + errpat = '^$' + outpat = '^$' + with self.expectedStdoutPattern(outpat): + with self.expectedStderrPattern(errpat): + self.conn = self.wiredtiger_open(newdir, salvage_config) + self.session = self.setUpSessionOpen(self.conn) + self.checks(expect) + + # Insert a couple more and simulate another crash. + newdir2 = "RESTART2" + self.inserts([self.nrecords, self.nrecords + 1]) + expect.extend([self.nrecords, self.nrecords + 1]) + self.copy_for_crash_restart(newdir, newdir2) + self.checks(expect) + self.reopen_conn(newdir) + self.checks(expect) + self.reopen_conn(newdir2, self.conn_config) + self.checks(expect) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/wtscenario.py b/src/third_party/wiredtiger/test/suite/wtscenario.py index 76824e428df..d953b7f627f 100644 --- a/src/third_party/wiredtiger/test/suite/wtscenario.py +++ b/src/third_party/wiredtiger/test/suite/wtscenario.py @@ -68,6 +68,9 @@ def make_scenarios(*args, **kwargs): """ The standard way to create scenarios for WT tests. Scenarios can be combined by listing them all as arguments. + If some scenario combinations should not be included, + a include= argument function may be listed, which given a name and + dictionary argument, returns True if the scenario should be included. A final prune= and/or prunelong= argument may be given that forces the list of entries in the scenario to be pruned. The result is a (combined) scenario that has been checked @@ -76,14 +79,19 @@ def make_scenarios(*args, **kwargs): scenes = multiply_scenarios('.', *args) pruneval = None prunelong = None + includefunc = None for key in kwargs: if key == 'prune': pruneval = kwargs[key] elif key == 'prunelong': prunelong = kwargs[key] + elif key == 'include': + includefunc = kwargs[key] else: raise AssertionError( 'make_scenarios: unexpected named arg: ' + key) + if includefunc: + scenes = [(name, d) for (name, d) in scenes if includefunc(name, d)] if pruneval != None or prunelong != None: pruneval = pruneval if pruneval != None else -1 prunelong = prunelong if prunelong != None else -1 diff --git a/src/third_party/wiredtiger/test/suite/wttest.py b/src/third_party/wiredtiger/test/suite/wttest.py index 8c0915bc9c7..1a2fddce031 100644 --- a/src/third_party/wiredtiger/test/suite/wttest.py +++ b/src/third_party/wiredtiger/test/suite/wttest.py @@ -101,7 +101,7 @@ class CapturedFd(object): ' unexpected ' + self.desc + ', contains:\n"' + contents + '"') testcase.fail('unexpected ' + self.desc + ', contains: "' + - shortenWithEllipsis(contents,100) + '"') + contents + '"') self.expectpos = filesize def checkAdditional(self, testcase, expect): @@ -180,7 +180,7 @@ class WiredTigerTestCase(unittest.TestCase): @staticmethod def globalSetup(preserveFiles = False, useTimestamp = False, - gdbSub = False, verbose = 1, builddir = None, dirarg = None, + gdbSub = False, lldbSub = False, verbose = 1, builddir = None, dirarg = None, longtest = False): WiredTigerTestCase._preserveFiles = preserveFiles d = 'WT_TEST' if dirarg == None else dirarg @@ -194,6 +194,7 @@ class WiredTigerTestCase(unittest.TestCase): WiredTigerTestCase._origcwd = os.getcwd() WiredTigerTestCase._resultfile = open(os.path.join(d, 'results.txt'), "w", 0) # unbuffered WiredTigerTestCase._gdbSubprocess = gdbSub + WiredTigerTestCase._lldbSubprocess = lldbSub WiredTigerTestCase._longtest = longtest WiredTigerTestCase._verbose = verbose WiredTigerTestCase._dupout = os.dup(sys.stdout.fileno()) @@ -339,20 +340,25 @@ class WiredTigerTestCase(unittest.TestCase): self.conn.close(config) self.conn = None - def open_conn(self, directory="."): + def open_conn(self, directory=".", config=None): """ Open the connection if already closed. """ if self.conn == None: + if config != None: + self._old_config = self.conn_config + self.conn_config = config self.conn = self.setUpConnectionOpen(directory) + if config != None: + self.conn_config = self._old_config self.session = self.setUpSessionOpen(self.conn) - def reopen_conn(self, directory="."): + def reopen_conn(self, directory=".", config=None): """ Reopen the connection. """ self.close_conn() - self.open_conn(directory) + self.open_conn(directory, config) def setUp(self): if not hasattr(self.__class__, 'wt_ntests'): diff --git a/src/third_party/wiredtiger/test/thread/rw.c b/src/third_party/wiredtiger/test/thread/rw.c index dc46e9b595d..9acd5d4095e 100644 --- a/src/third_party/wiredtiger/test/thread/rw.c +++ b/src/third_party/wiredtiger/test/thread/rw.c @@ -115,7 +115,7 @@ rw_start(u_int readers, u_int writers) /* Wait for the threads. */ for (i = 0; i < readers + writers; ++i) - testutil_check(__wt_thread_join(NULL, tids[i])); + testutil_check(__wt_thread_join(NULL, &tids[i])); (void)gettimeofday(&stop, NULL); seconds = (stop.tv_sec - start.tv_sec) + diff --git a/src/third_party/wiredtiger/test/thread/t.c b/src/third_party/wiredtiger/test/thread/t.c index f509b6d73e2..7108ea22005 100644 --- a/src/third_party/wiredtiger/test/thread/t.c +++ b/src/third_party/wiredtiger/test/thread/t.c @@ -136,7 +136,6 @@ main(int argc, char *argv[]) } argc -= __wt_optind; - argv += __wt_optind; if (argc != 0) return (usage()); diff --git a/src/third_party/wiredtiger/test/windows/windows_shim.c b/src/third_party/wiredtiger/test/windows/windows_shim.c index b40a9e01a42..b562fa97594 100644 --- a/src/third_party/wiredtiger/test/windows/windows_shim.c +++ b/src/third_party/wiredtiger/test/windows/windows_shim.c @@ -70,7 +70,7 @@ gettimeofday(struct timeval* tp, void* tzp) int pthread_rwlock_destroy(pthread_rwlock_t *lock) { - lock = lock; + lock = lock; /* Unused variable. */ return (0); } @@ -78,7 +78,7 @@ int pthread_rwlock_init(pthread_rwlock_t *rwlock, const pthread_rwlockattr_t *ignored) { - ignored = ignored; + ignored = ignored; /* Unused variable. */ InitializeSRWLock(&rwlock->rwlock); rwlock->exclusive_locked = 0; @@ -86,13 +86,6 @@ pthread_rwlock_init(pthread_rwlock_t *rwlock, } int -pthread_rwlock_rdlock(pthread_rwlock_t *rwlock) -{ - AcquireSRWLockShared(&rwlock->rwlock); - return (0); -} - -int pthread_rwlock_unlock(pthread_rwlock_t *rwlock) { if (rwlock->exclusive_locked != 0) { @@ -105,6 +98,19 @@ pthread_rwlock_unlock(pthread_rwlock_t *rwlock) } int +pthread_rwlock_tryrdlock(pthread_rwlock_t *rwlock) +{ + return (TryAcquireSRWLockShared(&rwlock->rwlock) ? 0 : EBUSY); +} + +int +pthread_rwlock_rdlock(pthread_rwlock_t *rwlock) +{ + AcquireSRWLockShared(&rwlock->rwlock); + return (0); +} + +int pthread_rwlock_trywrlock(pthread_rwlock_t *rwlock) { if (TryAcquireSRWLockExclusive(&rwlock->rwlock)) { diff --git a/src/third_party/wiredtiger/test/windows/windows_shim.h b/src/third_party/wiredtiger/test/windows/windows_shim.h index 2f46be9daee..f04b09569b0 100644 --- a/src/third_party/wiredtiger/test/windows/windows_shim.h +++ b/src/third_party/wiredtiger/test/windows/windows_shim.h @@ -76,7 +76,7 @@ typedef CONDITION_VARIABLE pthread_cond_t; struct rwlock_wrapper { SRWLOCK rwlock; - int exclusive_locked; + DWORD exclusive_locked; }; struct rwlock_wrapper; @@ -93,6 +93,7 @@ int pthread_join(pthread_t, void **); int pthread_rwlock_destroy(pthread_rwlock_t *); int pthread_rwlock_init(pthread_rwlock_t *, const pthread_rwlockattr_t *); int pthread_rwlock_rdlock(pthread_rwlock_t *); +int pthread_rwlock_tryrdlock(pthread_rwlock_t *); int pthread_rwlock_trywrlock(pthread_rwlock_t *); int pthread_rwlock_unlock(pthread_rwlock_t *); int pthread_rwlock_wrlock(pthread_rwlock_t *); |