diff options
author | Alex Gorrod <alexander.gorrod@mongodb.com> | 2017-06-14 19:39:53 +0000 |
---|---|---|
committer | Alex Gorrod <alexander.gorrod@mongodb.com> | 2017-06-14 19:39:53 +0000 |
commit | 11ca50ae96399aa8b0eaeee5dc115398d78fee2b (patch) | |
tree | a54af4da3e0dc8f216c191ce23e650e318ce00ac /src | |
parent | a16bc9371921766d41e7a86c3ca10080c677ce6d (diff) | |
download | mongo-11ca50ae96399aa8b0eaeee5dc115398d78fee2b.tar.gz |
Import wiredtiger: 47e8c3d1d22018eaaa09f91dfd78addb49e0b49b from branch mongodb-3.6
ref: 7aaeaaa054..47e8c3d1d2
for: 3.5.9
WT-2596 Document behavior after a crash with a backup cursor open
WT-3169 Add ability to log a message when the lookaside file is used
WT-3326 workgen: run wtperf "runner" files
WT-3332 Add statistic tracking connection wide transaction conflicts
WT-3346 workgen: create JSON output for latency sampling
WT-3349 Add timing stats to rwlocks
WT-3361 Resolve Windows build warnings, build more test programs on Windows.
WT-3362 Cursor opens should never block for the duration of a checkpoint
WT-3369 WT_CURSOR->uri should always match the URI used to open the cursor
Diffstat (limited to 'src')
65 files changed, 1449 insertions, 625 deletions
diff --git a/src/third_party/wiredtiger/SConstruct b/src/third_party/wiredtiger/SConstruct index b397f662be7..22f869e02e7 100644 --- a/src/third_party/wiredtiger/SConstruct +++ b/src/third_party/wiredtiger/SConstruct @@ -410,41 +410,37 @@ def builder_smoke_test(target, source, env): env.Append(BUILDERS={'SmokeTest' : Builder(action = builder_smoke_test)}) #Build the tests and setup the "scons test" target - testutil = env.Library('testutil', [ 'test/utility/misc.c', 'test/utility/parse_opts.c' ]) +env.Append(CPPPATH=["test/utility"]) -#Don't test bloom on Windows, its broken t = env.Program("t_bloom", "test/bloom/test_bloom.c", - LIBS=[wtlib, testutil] + wtlibs) -#env.Alias("check", env.SmokeTest(t)) + LIBS=[wtlib, shim, testutil] + wtlibs) Default(t) -#env.Program("t_checkpoint", - #["test/checkpoint/checkpointer.c", - #"test/checkpoint/test_checkpoint.c", - #"test/checkpoint/workers.c"], - #LIBS=[wtlib]) - -t = env.Program("t_huge", - "test/huge/huge.c", - LIBS=[wtlib] + wtlibs) +t = env.Program("t_checkpoint", + ["test/checkpoint/checkpointer.c", + "test/checkpoint/test_checkpoint.c", + "test/checkpoint/workers.c"], + LIBS=[wtlib, shim, testutil] + wtlibs) +Default(t) -#t = env.Program("t_recovery", -# "test/recovery/recovery.c", -# LIBS=[wtlib] + wtlibs) -#Default(t) +t = env.Program("t_cursor_order", + ["test/cursor_order/cursor_order.c", + "test/cursor_order/cursor_order_file.c", + "test/cursor_order/cursor_order_ops.c"], + LIBS=[wtlib, shim, testutil] + wtlibs) +Default(t) t = env.Program("t_fops", ["test/fops/file.c", "test/fops/fops.c", "test/fops/t.c"], LIBS=[wtlib, shim, testutil] + wtlibs) -env.Append(CPPPATH=["test/utility"]) Default(t) t = env.Program("t_format", @@ -459,19 +455,51 @@ t = env.Program("t_format", "test/format/t.c", "test/format/util.c", "test/format/wts.c"], - LIBS=[wtlib, shim, testutil] + wtlibs) + LIBS=[wtlib, shim, testutil] + wtlibs) +Default(t) + +t = env.Program("t_huge", + "test/huge/huge.c", + LIBS=[wtlib, shim, testutil] + wtlibs) Default(t) -#env.Program("t_thread", - #["test/thread/file.c", - #"test/thread/rw.c", - #"test/thread/stats.c", - #"test/thread/t.c"], - #LIBS=[wtlib]) +t = env.Program("t_manydbs", + "test/manydbs/manydbs.c", + LIBS=[wtlib, shim, testutil] + wtlibs) +Default(t) + +# t_readonly doesn't currently build/run. +#t = env.Program("t_readonly", +# "test/readonly/readonly.c", +# LIBS=[wtlib, shim, testutil] + wtlibs) +#Default(t) -#env.Program("t_salvage", - #["test/salvage/salvage.c"], - #LIBS=[wtlib]) +# t_random-abort doesn't currently build/run. +#t = env.Program("t_random-abort", +# "test/recovery/random-abort.c", +# LIBS=[wtlib, shim, testutil] + wtlibs) +#Default(t) + +# t_truncated-log doesn't currently build/run. +#t = env.Program("t_truncated-log", +# "test/recovery/truncated-log.c", +# LIBS=[wtlib, shim, testutil] + wtlibs) +#Default(t) + +# t_salvage-log doesn't currently build/run. +#t = env.Program("t_salvage", +# "test/salvage/salvage.c", +# LIBS=[wtlib, shim, testutil] + wtlibs) +#Default(t) + +# t_thread doesn't currently build/run. +#t = env.Program("t_thread", +# ["test/thread/file.c", +# "test/thread/rw.c", +# "test/thread/stats.c", +# "test/thread/t.c"], +# LIBS=[wtlib, shim, testutil] + wtlibs) +#Default(t) t = env.Program("wtperf", [ "bench/wtperf/config.c", diff --git a/src/third_party/wiredtiger/bench/workgen/runner/runner/__init__.py b/src/third_party/wiredtiger/bench/workgen/runner/runner/__init__.py index 67b547bc51b..ed21fffe8dc 100644 --- a/src/third_party/wiredtiger/bench/workgen/runner/runner/__init__.py +++ b/src/third_party/wiredtiger/bench/workgen/runner/runner/__init__.py @@ -88,5 +88,5 @@ except: shutil.rmtree('WT_TEST', True) os.mkdir('WT_TEST') -from .core import txn, extensions_config +from .core import txn, extensions_config, op_group_transaction, op_log_like, op_multi_table from .latency import workload_latency diff --git a/src/third_party/wiredtiger/bench/workgen/runner/runner/core.py b/src/third_party/wiredtiger/bench/workgen/runner/runner/core.py index a0f0d4d77cd..2c8311c4ca7 100644 --- a/src/third_party/wiredtiger/bench/workgen/runner/runner/core.py +++ b/src/third_party/wiredtiger/bench/workgen/runner/runner/core.py @@ -29,12 +29,12 @@ # runner/core.py # Core functions available to all runners import glob, os -import workgen +from workgen import Key, Operation, OpList, Table, Transaction, Value # txn -- # Put the operation (and any suboperations) within a transaction. def txn(op, config=None): - t = workgen.Transaction(config) + t = Transaction(config) op._transaction = t return op @@ -99,3 +99,103 @@ def extensions_config(exts): if len(extfiles) != 0: result = ',extensions=[' + ','.join(extfiles.values()) + ']' return result + +def _op_multi_table_as_list(ops_arg, tables): + result = [] + if ops_arg._optype != Operation.OP_NONE: + for table in tables: + result.append(Operation(ops_arg._optype, table, ops_arg._key, ops_arg._value)) + else: + for op in ops._group: + result.extend(_op_multi_table_as_list(op, tables)) + return result + +# A convenient way to build a list of operations +def op_append(op1, op2): + if op1 == None: + op1 = op2 + else: + op1 += op2 + return op1 + +# Emulate wtperf's table_count option. Spread the given operations over +# a set of tables. +def op_multi_table(ops_arg, tables): + ops = None + for op in _op_multi_table_as_list(ops_arg, tables): + ops = op_append(ops, op) + return ops + +# should be 8 bytes format 'Q' +_logkey = Key(Key.KEYGEN_APPEND, 8) +def _op_log_op(op, log_table): + keysize = op._key._size + if keysize == 0: + keysize = op._table.options.key_size + valuesize = op._value._size + if valuesize == 0: + valuesize = op._table.options.value_size + v = Value(keysize + valuesize) + return Operation(Operation.OP_INSERT, log_table, _logkey, v) + +def _optype_is_write(optype): + return optype == Operation.OP_INSERT or optype == Operation.OP_UPDATE or \ + optype == Operation.OP_REMOVE + +# Emulate wtperf's log_like option. For all operations, add a second +# insert operation going to a log table. +def op_log_like(op, log_table, ops_per_txn): + if op._optype != Operation.OP_NONE: + if _optype_is_write(op._optype): + op += _op_log_op(op, log_table) + if ops_per_txn == 0: + op = txn(op) # txn for each action. + else: + oplist = [] + for op2 in op._group: + if op2._optype == Operation.OP_NONE: + oplist.append(op_log_like(op2, log_table)) + elif ops_per_txn == 0 and _optype_is_write(op2._optype): + op2 += _op_log_op(op2, log_table) + oplist.append(txn(op2)) # txn for each action. + else: + oplist.append(op2) + if _optype_is_write(op2._optype): + oplist.append(_op_log_op(op2, log_table)) + op._group = OpList(oplist) + return op + +def _op_transaction_list(oplist, txn_config): + result = None + for op in oplist: + result = op_append(result, op) + return txn(result, txn_config) + +# Emulate wtperf's ops_per_txn option. Create transactions around +# groups of operations of the indicated size. +def op_group_transaction(ops_arg, ops_per_txn, txn_config): + if ops_arg != Operation.OP_NONE: + return txn(ops_arg, txn_config) + if ops_arg._transaction != None: + raise Exception('nested transactions not supported') + if ops_arg._repeatgroup != None: + raise Exception('grouping transactions with multipliers not supported') + + oplist = [] + ops = None + nops = 0 + txgroup = [] + for op in ops_arg._group: + if op.optype == Operation.OP_NONE: + oplist.append(_op_transaction_list(txgroup, txn_config)) + txgroup = [] + oplist.append(op) + else: + txgroup.append(op) + if len(txgroup) >= ops_per_txn: + oplist.append(_op_transaction_list(txgroup, txn_config)) + txgroup = [] + if len(txgroup) > 0: + oplist.append(_op_transaction_list(txgroup, txn_config)) + ops_arg._group = OpList(oplist) + return ops_arg diff --git a/src/third_party/wiredtiger/bench/workgen/runner/workgen_stat.sh b/src/third_party/wiredtiger/bench/workgen/runner/workgen_stat.sh new file mode 100755 index 00000000000..1739c29859e --- /dev/null +++ b/src/third_party/wiredtiger/bench/workgen/runner/workgen_stat.sh @@ -0,0 +1,75 @@ +#!/bin/bash +# +# workgen_stat.sh - combine JSON time series output from WT and workgen. +# +Usage() { + cat <<EOF +Usage: $0 [ options ] +Options: + -h <WT_home_directory> # set the WiredTiger home directory + -e <analyzer_name> # run analyzer on the combined files + -o <output_file> # output file for result + +At least one of '-t2' or '-o' must be selected. +EOF + exit 1 +} + +Filter() { + sed -e 's/"version" *: *"[^"]*",//' "$@" +} + +wthome=. +outfile= +analyze= + +while [ "$#" != 0 ]; do + arg="$1" + shift + case "$arg" in + -h ) + if [ $# = 0 ]; then + Usage + fi + wthome="$1" + shift + ;; + -o ) + if [ $# = 0 ]; then + Usage + fi + outfile="$1" + shift + ;; + -e ) + if [ $# = 0 ]; then + Usage + fi + analyze="$1" + shift + ;; + esac +done +if [ ! -d "$wthome" ]; then + echo "$wthome: WT home directory does not exist" + exit 1 +fi +if [ ! -f "$wthome/WiredTiger.wt" ]; then + echo "$wthome: directory is not a WiredTiger home directory" + exit 1 +fi +if [ "$outfile" = '' ]; then + if [ "$analyze" = false ]; then + Usage + fi + outfile="$wthome/stat_tmp.json" +fi +(cd $wthome; Filter WiredTigerStat.* sample.json) | sort > $outfile +if [ "$analyze" != '' ]; then + sysname=`uname -s` + if [ "$sysname" = Darwin ]; then + open -a "$analyze" "$outfile" + else + "$analyze" "$outfile" + fi +fi diff --git a/src/third_party/wiredtiger/bench/workgen/workgen.cxx b/src/third_party/wiredtiger/bench/workgen/workgen.cxx index c56acfd2989..880b8ca6467 100644 --- a/src/third_party/wiredtiger/bench/workgen/workgen.cxx +++ b/src/third_party/wiredtiger/bench/workgen/workgen.cxx @@ -267,16 +267,18 @@ int ContextInternal::create_all() { } Monitor::Monitor(WorkloadRunner &wrunner) : - _errno(0), _exception(), _wrunner(wrunner), _stop(false), _handle() {} + _errno(0), _exception(), _wrunner(wrunner), _stop(false), _handle(), + _out(NULL), _json(NULL) {} Monitor::~Monitor() {} int Monitor::run() { struct timespec t; struct tm *tm, _tm; - char time_buf[64]; + char time_buf[64], version[100]; Stats prev_totals; WorkloadOptions *options = &_wrunner._workload->options; uint64_t latency_max = (uint64_t)options->max_latency; + bool first; (*_out) << "#time," << "totalsec," @@ -295,6 +297,8 @@ int Monitor::run() { << "update maximum latency(uS)" << std::endl; + first = true; + workgen_version(version, sizeof(version)); Stats prev_interval; while (!_stop) { for (int i = 0; i < options->sample_interval && !_stop; i++) @@ -337,6 +341,32 @@ int Monitor::run() { << "," << interval.update.max_latency << std::endl; + if (_json != NULL) { +#define WORKGEN_TIMESTAMP_JSON "%Y-%m-%dT%H:%M:%S.000Z" + (void)strftime(time_buf, sizeof(time_buf), + WORKGEN_TIMESTAMP_JSON, tm); + +#define TRACK_JSON(name, t) \ + "\"" << (name) << "\":{" \ + << "\"ops per sec\":" << ((t).ops / interval_secs) \ + << ",\"average latency\":" << (t).average_latency() \ + << ",\"min latency\":" << (t).min_latency \ + << ",\"max latency\":" << (t).max_latency \ + << "}" + + (*_json) << "{"; + if (first) { + (*_json) << "\"version\":\"" << version << "\","; + first = false; + } + (*_json) << "\"localTime\":\"" << time_buf + << "\",\"workgen\":{" + << TRACK_JSON("read", interval.read) << "," + << TRACK_JSON("insert", interval.insert) << "," + << TRACK_JSON("update", interval.update) + << "}}" << std::endl; + } + uint64_t read_max = interval.read.max_latency; uint64_t insert_max = interval.read.max_latency; uint64_t update_max = interval.read.max_latency; @@ -1315,8 +1345,8 @@ TableInternal::TableInternal(const TableInternal &other) : _tint(other._tint), TableInternal::~TableInternal() {} WorkloadOptions::WorkloadOptions() : max_latency(0), - report_file("workload.stat"), report_interval(0), - run_time(0), sample_interval(0), sample_rate(1), + report_file("workload.stat"), report_interval(0), run_time(0), + sample_file("sample.json"), sample_interval(0), sample_rate(1), _options() { _options.add_int("max_latency", max_latency, "prints warning if any latency measured exceeds this number of " @@ -1329,6 +1359,11 @@ WorkloadOptions::WorkloadOptions() : max_latency(0), "The file name is relative to the connection's home directory. " "When set to the empty string, stdout is used."); _options.add_int("run_time", run_time, "total workload seconds"); + _options.add_string("sample_file", sample_file, + "file name for collecting latency output in a JSON-like format, " + "enabled by the report_interval option. " + "The file name is relative to the connection's home directory. " + "When set to the empty string, no JSON is emitted."); _options.add_int("sample_interval", sample_interval, "performance logging every interval seconds, 0 to disable"); _options.add_int("sample_rate", sample_rate, @@ -1492,6 +1527,7 @@ int WorkloadRunner::run_all() { WorkloadOptions *options = &_workload->options; Monitor monitor(*this); std::ofstream monitor_out; + std::ofstream monitor_json; std::ostream &out = *_report_out; WT_DECL_RET; @@ -1510,6 +1546,12 @@ int WorkloadRunner::run_all() { open_report_file(monitor_out, "monitor", "monitor output file"); monitor._out = &monitor_out; + if (!options->sample_file.empty()) { + open_report_file(monitor_json, options->sample_file.c_str(), + "sample JSON output file"); + monitor._json = &monitor_json; + } + if ((ret = pthread_create(&monitor._handle, NULL, monitor_main, &monitor)) != 0) { std::cerr << "monitor thread failed err=" << ret << std::endl; @@ -1588,6 +1630,10 @@ int WorkloadRunner::run_all() { << std::endl; if (exception == NULL && !monitor._exception._str.empty()) exception = &monitor._exception; + + monitor_out.close(); + if (!options->sample_file.empty()) + monitor_json.close(); } // issue the final report diff --git a/src/third_party/wiredtiger/bench/workgen/workgen.h b/src/third_party/wiredtiger/bench/workgen/workgen.h index c1ae01ed5a4..c7be8ee0035 100644 --- a/src/third_party/wiredtiger/bench/workgen/workgen.h +++ b/src/third_party/wiredtiger/bench/workgen/workgen.h @@ -358,6 +358,7 @@ struct WorkloadOptions { int run_time; int sample_interval; int sample_rate; + std::string sample_file; WorkloadOptions(); WorkloadOptions(const WorkloadOptions &other); diff --git a/src/third_party/wiredtiger/bench/workgen/workgen_func.c b/src/third_party/wiredtiger/bench/workgen/workgen_func.c index 2e1271a515e..5ce2146a8e4 100644 --- a/src/third_party/wiredtiger/bench/workgen/workgen_func.c +++ b/src/third_party/wiredtiger/bench/workgen/workgen_func.c @@ -87,3 +87,16 @@ workgen_u64_to_string_zf(uint64_t n, char *buf, size_t len) { u64_to_string_zf(n, buf, len); } + +#define WORKGEN_VERSION_PREFIX "workgen-" +extern void +workgen_version(char *buf, size_t len) +{ + size_t prefix_len; + + prefix_len = strlen(WORKGEN_VERSION_PREFIX); + (void)strncpy(buf, WORKGEN_VERSION_PREFIX, len); + if (len > prefix_len) + (void)strncpy(&buf[prefix_len], WIREDTIGER_VERSION_STRING, + len - prefix_len); +} diff --git a/src/third_party/wiredtiger/bench/workgen/workgen_func.h b/src/third_party/wiredtiger/bench/workgen/workgen_func.h index 20ebf2632cc..ec7ecf0a504 100644 --- a/src/third_party/wiredtiger/bench/workgen/workgen_func.h +++ b/src/third_party/wiredtiger/bench/workgen/workgen_func.h @@ -42,3 +42,5 @@ extern void workgen_random_free(struct workgen_random_state *rnd_state); extern void workgen_u64_to_string_zf(uint64_t n, char *buf, size_t len); +extern void +workgen_version(char *buf, size_t len); diff --git a/src/third_party/wiredtiger/bench/workgen/workgen_int.h b/src/third_party/wiredtiger/bench/workgen/workgen_int.h index 01fb727691b..9283aea1d7b 100644 --- a/src/third_party/wiredtiger/bench/workgen/workgen_int.h +++ b/src/third_party/wiredtiger/bench/workgen/workgen_int.h @@ -146,6 +146,7 @@ struct Monitor { volatile bool _stop; pthread_t _handle; std::ostream *_out; + std::ostream *_json; Monitor(WorkloadRunner &wrunner); ~Monitor(); diff --git a/src/third_party/wiredtiger/bench/workgen/wtperf.py b/src/third_party/wiredtiger/bench/workgen/wtperf.py new file mode 100644 index 00000000000..3a196fe7b57 --- /dev/null +++ b/src/third_party/wiredtiger/bench/workgen/wtperf.py @@ -0,0 +1,440 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2017 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +# wtperf.py +# A partial emulation of wtperf. Translates a .wtperf file into a Python +# script that uses the workgen module, and runs the script. Errors are +# issued for any .wtperf directives that are not known. +# See also the usage() function. +# +from __future__ import print_function +import os, sys, tempfile + +def eprint(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) + +class OptionValue: + def __init__(self, value, filename, linenum): + self.value = value + self.filename = filename + self.linenum = linenum + +class TranslateException(Exception): + pass + +class Options(object): + pass + +class Translator: + def __init__(self, filename, prefix, verbose): + self.filename = filename + self.prefix = prefix + self.verbose = verbose + self.linenum = 0 + self.opts = {} + self.used_opts = {} + self.has_error = False + + def error_file_line(self, fname, linenum, msg): + self.has_error = True + eprint(fname + ':' + str(linenum) + ': error: ' + msg) + + # Report an error and continue + def error(self, msg): + self.error_file_line(self.filename, self.linenum, msg) + + # Report an error and unwind the stack + def fatal_error(self, msg, errtype): + self.error(msg) + raise TranslateException(errtype) + + supported_opt_list = [ 'compression', 'conn_config', 'icount', + 'key_sz', 'log_like_table', + 'populate_ops_per_txn', 'populate_threads', + 'reopen_connection', + 'table_config', 'table_count', + 'threads', 'transaction_config', 'value_sz' ] + + def set_opt(self, optname, val): + if optname not in self.supported_opt_list: + self.error("unknown option: " + optname) + return + elif val[0] == '"' and val[-1] == '"': + v = val[1:-1] + elif val == 'true': + v = True + elif val == 'false': + v = False + elif val[0] == '(': + v = val # config string stored as is + else: + try: + v = int(val) # it might be an integer + except ValueError: + v = val # it's a string after all + self.opts[optname] = OptionValue(v, self.filename, self.linenum) + + def get_opt(self, optname, dfault): + if optname in self.opts: + ret = self.opts[optname] + self.filename = ret.filename + self.linenum = ret.linenum + self.used_opts[optname] = 1 + return ret.value + else: + return dfault + + def get_int_opt(self, optname, dfault): + return self.get_opt(optname, dfault) + 0 + + def get_boolean_opt(self, optname, dfault): + return not not self.get_opt(optname, dfault) + + # Split a string 'left_side=right_side' into two parts + def split_assign(self, s): + equalpos = s.find('=') + if equalpos < 0: + self.error("missing '=' for line: " + line) + return (None, None) + else: + return s.split('=', 1) + + # Split a config string honoring nesting e.g. + # "(abc=123,def=234,ghi=(hi=1,bye=2))" would return 3 items. + def split_config_parens(self, s): + if s[0:1] != '(': + import pdb + pdb.set_trace() + self.fatal_error('missing left paren', 'config parse error') + if s[-1:] != ')': + self.fatal_error('missing right paren', 'config parse error') + s = s[1:-1] + result = [] + level = 0 + cur = '' + for ch in s: + if ch == ',' and level == 0: + result.append(cur) + cur = '' + else: + cur += ch + if ch == '(': + level += 1 + elif ch == ')': + level -= 1 + if level < 0: + self.fatal_error('unbalanced paren', 'config parse error') + if level != 0: + self.fatal_error('unbalanced paren', 'config parse error') + if len(cur) != 0: + result.append(cur) + return result + + def assign_str(self, left, right): + return left + '=' + str(right) + '\n' + + def add_operation_str(self, count, opname, multi): + result = '' + tablename = 'tables[0]' if multi else 'table' + if count > 1: + result += str(count) + ' * ' + if count > 0: + result += 'Operation(Operation.' + opname + ', ' + \ + tablename + ') + \\\n' + result += ' ' + return result + + # Wtperf's throttle is based on the number of regular operations, + # not including log_like operations. Workgen counts all operations, + # it doesn't treat log operations any differently. Adjust the throttle + # number to account for the difference. + def calc_throttle(self, thread_opts, log_like_table): + throttle = thread_opts.throttle + if not log_like_table: + return (throttle, '') + modify = thread_opts.inserts + thread_opts.updates + regular = modify + thread_opts.reads + total = regular + modify + factor = (total + 0.0) / regular + new_throttle = int(throttle * factor) + if new_throttle == throttle: + comment = '' + else: + comment = '# wtperf throttle=' + str(throttle) + ' adjusted by ' + \ + str(factor) + ' to compensate for log_like operations.\n' + return (new_throttle, comment) + + def parse_threads(self, threads_config): + tdecls = '' + tlist = self.split_config_parens(threads_config) + table_count = self.get_int_opt('table_count', 1) + log_like_table = self.get_boolean_opt('log_like_table', False) + txn_config = self.get_opt('transaction_config', '') + if log_like_table: + tdecls += 'log_name = "table:log"\n' + tdecls += 's.create(log_name, "key_format=S,value_format=S," +' + \ + ' compress_table_config)\n' + tdecls += 'log_table = Table(log_name)\n\n' + thread_count = 0 + tnames = '' + multi = (table_count > 1) + for t in tlist: + thread_name = 'thread' + str(thread_count) + thread_count += 1 + + # For wtperf compatibility, we allow both 'insert/inserts' etc. + topts = Options() + topts.count = 1 + topts.insert = 0 + topts.inserts = 0 + topts.ops_per_txn = 0 + topts.read = 0 + topts.reads = 0 + topts.throttle = 0 + topts.update = 0 + topts.updates = 0 + + for o in self.split_config_parens(t): + (k, v) = self.split_assign(o) + if hasattr(topts, k): + try: + setattr(topts, k, int(v)) + except ValueError: + self.error('thread option ' + k + ': integer expected') + else: + self.error('unknown thread option: ' + k) + + topts.inserts += topts.insert; topts.insert = 0 + topts.updates += topts.update; topts.update = 0 + topts.reads += topts.read; topts.read = 0 + if topts.count == 0: + continue + + if topts.inserts + topts.reads + topts.updates == 0: + self.fatal_error('need read/insert/update/...', + 'thread config error') + tdecls += 'ops = ' + tdecls += self.add_operation_str(topts.inserts, 'OP_INSERT', multi) + tdecls += self.add_operation_str(topts.reads, 'OP_SEARCH', multi) + tdecls += self.add_operation_str(topts.updates, 'OP_UPDATE', multi) + tdecls = tdecls.rstrip(' \n\\+') + '\n' + if multi: + tdecls += 'ops = op_multi_table(ops, tables)\n' + if topts.ops_per_txn > 0: + tdecls += 'ops = op_group_transaction(ops, ' + \ + str(topts.ops_per_txn) + ', "' + txn_config + '")\n' + if log_like_table: + tdecls += 'ops = op_log_like(ops, log_table, ' + \ + str(topts.ops_per_txn) + ')\n' + tdecls += thread_name + ' = Thread(ops)\n' + if topts.throttle > 0: + (throttle, comment) = self.calc_throttle(topts, log_like_table) + tdecls += comment + tdecls += self.assign_str(thread_name + '.options.throttle', + throttle) + tdecls += '\n' + if topts.count > 1: + tnames += str(topts.count) + ' * ' + tnames += thread_name + ' + ' + + tnames = tnames.rstrip(' +') + return (tdecls, tnames) + + def translate(self): + try: + return self.translate_inner() + except TranslateException: + # An error has already been reported + return None + + def translate_inner(self): + workloadopts = '' + with open(self.filename) as fin: + for line in fin: + self.linenum += 1 + commentpos = line.find('#') + if commentpos >= 0: + line = line[0:commentpos] + line = line.strip() + if len(line) == 0: + continue + (key, val) = self.split_assign(line) + if key in [ 'max_latency', 'report_file', 'report_interval', + 'run_time', 'sample_interval', 'sample_rate' ]: + workloadopts += 'workload.options.' + key + '=' + val + '\n' + else: + self.set_opt(key, val) + + table_count = self.get_int_opt('table_count', 1) + conn_config = self.get_opt('conn_config', '') + table_config = self.get_opt('table_config', '') + key_sz = self.get_int_opt('key_sz', 20) + value_sz = self.get_int_opt('value_sz', 100) + reopen = self.get_boolean_opt('reopen_connection', False) + compression = self.get_opt('compression', '') + txn_config = self.get_opt('transaction_config', '') + + s = '#/usr/bin/env python\n' + s += '# generated from ' + self.filename + '\n' + s += self.prefix + s += 'from runner import *\n' + s += 'from wiredtiger import *\n' + s += 'from workgen import *\n' + s += '\n' + s += 'context = Context()\n' + s += 'conn_config = "' + conn_config + '"\n' + if compression != '': + s += 'conn_config += extensions_config(["compressors/' + \ + compression + '"])\n' + compression = 'block_compressor=' + compression + ',' + s += 'conn = wiredtiger_open("WT_TEST", "create," + conn_config)\n' + s += 's = conn.open_session()\n' + s += '\n' + s += 'wtperf_table_config = "key_format=S,value_format=S,type=lsm," +\\\n' + s += ' "exclusive=true,allocation_size=4kb," +\\\n' + s += ' "internal_page_max=64kb,leaf_page_max=4kb,split_pct=100,"\n' + s += 'compress_table_config = "' + compression + '"\n' + s += 'table_config = "' + table_config + '"\n' + if table_count == 1: + s += 'tname = "file:test.wt"\n' + s += 's.create(tname, wtperf_table_config +\\\n' + s += ' compress_table_config + table_config)\n' + s += 'table = Table(tname)\n' + s += 'table.options.key_size = ' + str(key_sz) + '\n' + s += 'table.options.value_size = ' + str(value_sz) + '\n' + else: + s += 'table_count = ' + str(table_count) + '\n' + s += 'tables = []\n' + s += 'for i in range(0, table_count):\n' + s += ' tname = "file:test" + str(i) + ".wt"\n' + s += ' s.create(tname, ' + \ + 'wtperf_table_config + ' + \ + 'compress_table_config + table_config)\n' + s += ' t = Table(tname)\n' + s += ' t.options.key_size = ' + str(key_sz) + '\n' + s += ' t.options.value_size = ' + str(value_sz) + '\n' + s += ' tables.append(t)\n' + s += '\n' + + icount = self.get_int_opt('icount', 0) + pop_thread = self.get_int_opt('populate_threads', 1) + pop_per_txn = self.get_int_opt('populate_ops_per_txn', 0) + if icount != 0: + if pop_thread == 0: + self.fatal_error('icount != 0 and populate_threads == 0: ' +\ + 'cannot populate entries with no threads') + elif pop_thread == 1: + mult = '' + else: + mult = str(pop_thread) + ' * ' + + # if there are multiple tables to be filled during populate, + # the icount is split between them all. + nops_per_thread = icount / (pop_thread * table_count) + if table_count == 1: + s += 'pop_ops = Operation(Operation.OP_INSERT, table)\n' + else: + s += 'pop_ops = Operation(Operation.OP_INSERT, tables[0])\n' + s += 'pop_ops = op_multi_table(pop_ops, tables)\n' + if pop_per_txn > 0: + s += 'pop_ops = op_group_transaction(pop_ops, ' + \ + str(pop_per_txn) + ', "' + txn_config + '")\n' + s += 'pop_thread = Thread(pop_ops * ' + str(nops_per_thread) + ')\n' + s += 'pop_workload = Workload(context, ' + mult + 'pop_thread)\n' + if self.verbose > 0: + s += 'print("populate:")\n' + s += 'pop_workload.run(conn)\n' + else: + if self.get_int_opt('populate_threads', 0) != 0: + self.error("populate_threads > 0, icount == 0") + + thread_config = self.get_opt('threads', '') + if thread_config != '': + (t_create, t_var) = self.parse_threads(thread_config) + s += '\n' + t_create + if reopen: + s += '\n# reopen the connection\n' + s += 'conn.close()\n' + s += 'conn = wiredtiger_open(' + \ + '"WT_TEST", "create," + conn_config)\n' + s += '\n' + s += 'workload = Workload(context, ' + t_var + ')\n' + s += workloadopts + if self.verbose > 0: + s += 'print("workload:")\n' + s += 'workload.run(conn)\n' + + for o in self.used_opts: + del self.opts[o] + if len(self.opts) != 0: + self.error('internal error, options not handled: ' + str(self.opts)) + return s + +def usage(): + eprint(( + 'Usage: python wtperf.py [ options ] file.wtperf ...\n' + '\n' + 'Options:\n' + ' --python Python output generated on stdout\n' + ' -v --verbose Verbose output\n' + '\n' + 'If --python is not specified, the resulting workload is run.')) + +verbose = 0 +py_out = False +workgen_dir = os.path.dirname(os.path.abspath(__file__)) +runner_dir = os.path.join(workgen_dir, 'runner') +prefix = ( + '# The next lines are unneeded if this script is in the runner directory.\n' + 'import sys\n' + 'sys.path.append("' + runner_dir + '")\n\n') + +exit_status = 0 +for arg in sys.argv[1:]: + if arg == '--python': + py_out = True + elif arg == '--verbose' or arg == '-v': + verbose += 1 + elif arg.endswith('.wtperf'): + translator = Translator(arg, prefix, verbose) + pysrc = translator.translate() + if translator.has_error: + exit_status = 1 + elif py_out: + print(pysrc) + else: + (outfd, tmpfile) = tempfile.mkstemp(suffix='.py') + os.write(outfd, pysrc) + os.close(outfd) + execfile(tmpfile) + os.remove(tmpfile) + else: + usage() + sys.exit(1) +sys.exit(exit_status) diff --git a/src/third_party/wiredtiger/bench/wtperf/idle_table_cycle.c b/src/third_party/wiredtiger/bench/wtperf/idle_table_cycle.c index ce64049ce89..d0baa786ba9 100644 --- a/src/third_party/wiredtiger/bench/wtperf/idle_table_cycle.c +++ b/src/third_party/wiredtiger/bench/wtperf/idle_table_cycle.c @@ -57,7 +57,7 @@ check_timing(WTPERF *wtperf, * Measure how long each step takes, and flag an error if it exceeds the * configured maximum. */ -static void * +static WT_THREAD_RET cycle_idle_tables(void *arg) { struct timespec start, stop; @@ -76,7 +76,7 @@ cycle_idle_tables(void *arg) wtperf->conn, NULL, opts->sess_config, &session)) != 0) { lprintf(wtperf, ret, 0, "Error opening a session on %s", wtperf->home); - return (NULL); + return (WT_THREAD_RET_VALUE); } for (cycle_count = 0; wtperf->idle_cycle_run; ++cycle_count) { @@ -96,10 +96,10 @@ cycle_idle_tables(void *arg) lprintf(wtperf, ret, 0, "Table create failed in cycle_idle_tables."); wtperf->error = true; - return (NULL); + return (WT_THREAD_RET_VALUE); } if (check_timing(wtperf, "create", start, &stop) != 0) - return (NULL); + return (WT_THREAD_RET_VALUE); start = stop; /* Open and close cursor. */ @@ -108,16 +108,16 @@ cycle_idle_tables(void *arg) lprintf(wtperf, ret, 0, "Cursor open failed in cycle_idle_tables."); wtperf->error = true; - return (NULL); + return (WT_THREAD_RET_VALUE); } if ((ret = cursor->close(cursor)) != 0) { lprintf(wtperf, ret, 0, "Cursor close failed in cycle_idle_tables."); wtperf->error = true; - return (NULL); + return (WT_THREAD_RET_VALUE); } if (check_timing(wtperf, "cursor", start, &stop) != 0) - return (NULL); + return (WT_THREAD_RET_VALUE); start = stop; #if 1 @@ -133,14 +133,14 @@ cycle_idle_tables(void *arg) lprintf(wtperf, ret, 0, "Table drop failed in cycle_idle_tables."); wtperf->error = true; - return (NULL); + return (WT_THREAD_RET_VALUE); } if (check_timing(wtperf, "drop", start, &stop) != 0) - return (NULL); + return (WT_THREAD_RET_VALUE); #endif } - return (NULL); + return (WT_THREAD_RET_VALUE); } /* @@ -150,47 +150,33 @@ cycle_idle_tables(void *arg) * structure. Should reshuffle the configuration structure so explicit static * initialization isn't necessary. */ -int -start_idle_table_cycle(WTPERF *wtperf, pthread_t *idle_table_cycle_thread) +void +start_idle_table_cycle(WTPERF *wtperf, wt_thread_t *idle_table_cycle_thread) { CONFIG_OPTS *opts; - pthread_t thread_id; - int ret; + wt_thread_t thread_id; opts = wtperf->opts; if (opts->idle_table_cycle == 0) - return (0); + return; wtperf->idle_cycle_run = true; - if ((ret = pthread_create( - &thread_id, NULL, cycle_idle_tables, wtperf)) != 0) { - lprintf(wtperf, - ret, 0, "Error creating idle table cycle thread."); - wtperf->idle_cycle_run = false; - return (ret); - } + testutil_check(__wt_thread_create( + NULL, &thread_id, cycle_idle_tables, wtperf)); *idle_table_cycle_thread = thread_id; - - return (0); } -int -stop_idle_table_cycle(WTPERF *wtperf, pthread_t idle_table_cycle_thread) +void +stop_idle_table_cycle(WTPERF *wtperf, wt_thread_t idle_table_cycle_thread) { CONFIG_OPTS *opts; - int ret; opts = wtperf->opts; if (opts->idle_table_cycle == 0 || !wtperf->idle_cycle_run) - return (0); + return; wtperf->idle_cycle_run = false; - if ((ret = pthread_join(idle_table_cycle_thread, NULL)) != 0) { - lprintf( - wtperf, ret, 0, "Error joining idle table cycle thread."); - return (ret); - } - return (0); + testutil_check(__wt_thread_join(NULL, idle_table_cycle_thread)); } diff --git a/src/third_party/wiredtiger/bench/wtperf/wtperf.c b/src/third_party/wiredtiger/bench/wtperf/wtperf.c index 68bc08226c2..a8d3f135280 100644 --- a/src/third_party/wiredtiger/bench/wtperf/wtperf.c +++ b/src/third_party/wiredtiger/bench/wtperf/wtperf.c @@ -32,23 +32,23 @@ #define DEFAULT_HOME "WT_TEST" #define DEFAULT_MONITOR_DIR "WT_TEST" -static void *checkpoint_worker(void *); +static WT_THREAD_RET checkpoint_worker(void *); static int drop_all_tables(WTPERF *); static int execute_populate(WTPERF *); static int execute_workload(WTPERF *); static int find_table_count(WTPERF *); -static void *monitor(void *); -static void *populate_thread(void *); +static WT_THREAD_RET monitor(void *); +static WT_THREAD_RET populate_thread(void *); static void randomize_value(WTPERF_THREAD *, char *); static void recreate_dir(const char *); static int start_all_runs(WTPERF *); static int start_run(WTPERF *); -static int start_threads(WTPERF *, - WORKLOAD *, WTPERF_THREAD *, u_int, void *(*)(void *)); -static int stop_threads(WTPERF *, u_int, WTPERF_THREAD *); -static void *thread_run_wtperf(void *); +static void start_threads(WTPERF *, WORKLOAD *, + WTPERF_THREAD *, u_int, WT_THREAD_CALLBACK(*)(void *)); +static void stop_threads(u_int, WTPERF_THREAD *); +static WT_THREAD_RET thread_run_wtperf(void *); static void update_value_delta(WTPERF_THREAD *); -static void *worker(void *); +static WT_THREAD_RET worker(void *); static uint64_t wtperf_rand(WTPERF_THREAD *); static uint64_t wtperf_value_range(WTPERF *); @@ -312,7 +312,7 @@ op_name(uint8_t *op) /* NOTREACHED */ } -static void * +static WT_THREAD_RET worker_async(void *arg) { CONFIG_OPTS *opts; @@ -420,7 +420,7 @@ op_err: lprintf(wtperf, ret, 0, if (0) { err: wtperf->error = wtperf->stop = true; } - return (NULL); + return (WT_THREAD_RET_VALUE); } /* @@ -513,7 +513,7 @@ err: lprintf(wtperf, ret, 0, "Pre-workload traverse error"); return (ret); } -static void * +static WT_THREAD_RET worker(void *arg) { struct timespec start, stop; @@ -893,7 +893,7 @@ err: wtperf->error = wtperf->stop = true; } free(cursors); - return (NULL); + return (WT_THREAD_RET_VALUE); } /* @@ -1014,7 +1014,7 @@ run_mix_schedule(WTPERF *wtperf, WORKLOAD *workp) return (0); } -static void * +static WT_THREAD_RET populate_thread(void *arg) { struct timespec start, stop; @@ -1163,10 +1163,10 @@ err: wtperf->error = wtperf->stop = true; } free(cursors); - return (NULL); + return (WT_THREAD_RET_VALUE); } -static void * +static WT_THREAD_RET populate_async(void *arg) { struct timespec start, stop; @@ -1261,10 +1261,10 @@ populate_async(void *arg) if (0) { err: wtperf->error = wtperf->stop = true; } - return (NULL); + return (WT_THREAD_RET_VALUE); } -static void * +static WT_THREAD_RET monitor(void *arg) { struct timespec t; @@ -1426,10 +1426,10 @@ err: wtperf->error = wtperf->stop = true; (void)fclose(fp); free(path); - return (NULL); + return (WT_THREAD_RET_VALUE); } -static void * +static WT_THREAD_RET checkpoint_worker(void *arg) { CONFIG_OPTS *opts; @@ -1490,7 +1490,7 @@ checkpoint_worker(void *arg) err: wtperf->error = wtperf->stop = true; } - return (NULL); + return (WT_THREAD_RET_VALUE); } static int @@ -1498,15 +1498,15 @@ execute_populate(WTPERF *wtperf) { struct timespec start, stop; CONFIG_OPTS *opts; - WTPERF_THREAD *popth; WT_ASYNC_OP *asyncop; - pthread_t idle_table_cycle_thread; + WTPERF_THREAD *popth; + WT_THREAD_CALLBACK(*pfunc)(void *); size_t i; uint64_t last_ops, msecs, print_ops_sec; uint32_t interval, tables; + wt_thread_t idle_table_cycle_thread; double print_secs; int elapsed, ret; - void *(*pfunc)(void *); opts = wtperf->opts; @@ -1516,9 +1516,7 @@ execute_populate(WTPERF *wtperf) opts->populate_threads, opts->icount); /* Start cycling idle tables if configured. */ - if ((ret = - start_idle_table_cycle(wtperf, &idle_table_cycle_thread)) != 0) - return (ret); + start_idle_table_cycle(wtperf, &idle_table_cycle_thread); wtperf->insert_key = 0; @@ -1530,9 +1528,8 @@ execute_populate(WTPERF *wtperf) pfunc = populate_async; } else pfunc = populate_thread; - if ((ret = start_threads(wtperf, NULL, - wtperf->popthreads, opts->populate_threads, pfunc)) != 0) - return (ret); + start_threads(wtperf, NULL, + wtperf->popthreads, opts->populate_threads, pfunc); __wt_epoch(NULL, &start); for (elapsed = 0, interval = 0, last_ops = 0; @@ -1568,10 +1565,8 @@ execute_populate(WTPERF *wtperf) */ popth = wtperf->popthreads; wtperf->popthreads = NULL; - ret = stop_threads(wtperf, opts->populate_threads, popth); + stop_threads(opts->populate_threads, popth); free(popth); - if (ret != 0) - return (ret); /* Report if any worker threads didn't finish. */ if (wtperf->error) { @@ -1640,8 +1635,7 @@ execute_populate(WTPERF *wtperf) } /* Stop cycling idle tables. */ - if ((ret = stop_idle_table_cycle(wtperf, idle_table_cycle_thread)) != 0) - return (ret); + stop_idle_table_cycle(wtperf, idle_table_cycle_thread); return (0); } @@ -1701,13 +1695,13 @@ execute_workload(WTPERF *wtperf) WTPERF_THREAD *threads; WT_CONNECTION *conn; WT_SESSION **sessions; - pthread_t idle_table_cycle_thread; + WT_THREAD_CALLBACK(*pfunc)(void *); + wt_thread_t idle_table_cycle_thread; uint64_t last_ckpts, last_inserts, last_reads, last_truncates; uint64_t last_updates; uint32_t interval, run_ops, run_time; u_int i; - int ret, t_ret; - void *(*pfunc)(void *); + int ret; opts = wtperf->opts; @@ -1722,9 +1716,7 @@ execute_workload(WTPERF *wtperf) sessions = NULL; /* Start cycling idle tables. */ - if ((ret = - start_idle_table_cycle(wtperf, &idle_table_cycle_thread)) != 0) - return (ret); + start_idle_table_cycle(wtperf, &idle_table_cycle_thread); if (opts->warmup != 0) wtperf->in_warmup = true; @@ -1768,9 +1760,8 @@ execute_workload(WTPERF *wtperf) goto err; /* Start the workload's threads. */ - if ((ret = start_threads( - wtperf, workp, threads, (u_int)workp->threads, pfunc)) != 0) - goto err; + start_threads( + wtperf, workp, threads, (u_int)workp->threads, pfunc); threads += workp->threads; } @@ -1836,12 +1827,9 @@ execute_workload(WTPERF *wtperf) err: wtperf->stop = true; /* Stop cycling idle tables. */ - if ((ret = stop_idle_table_cycle(wtperf, idle_table_cycle_thread)) != 0) - return (ret); + stop_idle_table_cycle(wtperf, idle_table_cycle_thread); - if ((t_ret = stop_threads(wtperf, - (u_int)wtperf->workers_cnt, wtperf->workers)) != 0 && ret == 0) - ret = t_ret; + stop_threads((u_int)wtperf->workers_cnt, wtperf->workers); /* Drop tables if configured to and this isn't an error path */ if (ret == 0 && @@ -2163,9 +2151,9 @@ start_all_runs(WTPERF *wtperf) { CONFIG_OPTS *opts; WTPERF *next_wtperf, **wtperfs; - pthread_t *threads; size_t i, len; - int ret, t_ret; + wt_thread_t *threads; + int ret; opts = wtperf->opts; wtperfs = NULL; @@ -2178,7 +2166,7 @@ start_all_runs(WTPERF *wtperf) wtperfs = dcalloc(opts->database_count, sizeof(WTPERF *)); /* Allocate an array to hold our thread IDs. */ - threads = dcalloc(opts->database_count, sizeof(pthread_t)); + threads = dcalloc(opts->database_count, sizeof(*threads)); for (i = 0; i < opts->database_count; i++) { wtperf_copy(wtperf, &next_wtperf); @@ -2203,22 +2191,15 @@ start_all_runs(WTPERF *wtperf) strcmp(next_wtperf->home, next_wtperf->monitor_dir) != 0) recreate_dir(next_wtperf->monitor_dir); - if ((ret = pthread_create( - &threads[i], NULL, thread_run_wtperf, next_wtperf)) != 0) { - lprintf(wtperf, ret, 0, "Error creating thread"); - goto err; - } + testutil_check(__wt_thread_create(NULL, + &threads[i], thread_run_wtperf, next_wtperf)); } /* Wait for threads to finish. */ for (i = 0; i < opts->database_count; i++) - if ((t_ret = pthread_join(threads[i], NULL)) != 0) { - lprintf(wtperf, ret, 0, "Error joining thread"); - if (ret == 0) - ret = t_ret; - } + testutil_check(__wt_thread_join(NULL, threads[i])); -err: for (i = 0; i < opts->database_count && wtperfs[i] != NULL; i++) { + for (i = 0; i < opts->database_count && wtperfs[i] != NULL; i++) { wtperf_free(wtperfs[i]); free(wtperfs[i]); } @@ -2229,7 +2210,7 @@ err: for (i = 0; i < opts->database_count && wtperfs[i] != NULL; i++) { } /* Run an instance of wtperf for a given configuration. */ -static void * +static WT_THREAD_RET thread_run_wtperf(void *arg) { WTPERF *wtperf; @@ -2238,14 +2219,14 @@ thread_run_wtperf(void *arg) wtperf = (WTPERF *)arg; if ((ret = start_run(wtperf)) != 0) lprintf(wtperf, ret, 0, "Run failed for: %s.", wtperf->home); - return (NULL); + return (WT_THREAD_RET_VALUE); } static int start_run(WTPERF *wtperf) { CONFIG_OPTS *opts; - pthread_t monitor_thread; + wt_thread_t monitor_thread; uint64_t total_ops; uint32_t run_time; int monitor_created, ret, t_ret; @@ -2272,12 +2253,8 @@ start_run(WTPERF *wtperf) /* Start the monitor thread. */ if (opts->sample_interval != 0) { - if ((ret = pthread_create( - &monitor_thread, NULL, monitor, wtperf)) != 0) { - lprintf(wtperf, - ret, 0, "Error creating monitor thread."); - goto err; - } + testutil_check(__wt_thread_create( + NULL, &monitor_thread, monitor, wtperf)); monitor_created = 1; } @@ -2306,9 +2283,8 @@ start_run(WTPERF *wtperf) opts->checkpoint_threads); wtperf->ckptthreads = dcalloc( opts->checkpoint_threads, sizeof(WTPERF_THREAD)); - if (start_threads(wtperf, NULL, wtperf->ckptthreads, - opts->checkpoint_threads, checkpoint_worker) != 0) - goto err; + start_threads(wtperf, NULL, wtperf->ckptthreads, + opts->checkpoint_threads, checkpoint_worker); } if (opts->pre_load_data && (ret = pre_load_data(wtperf)) != 0) goto err; @@ -2362,16 +2338,10 @@ err: if (ret == 0) /* Notify the worker threads they are done. */ wtperf->stop = true; - if ((t_ret = stop_threads(wtperf, 1, wtperf->ckptthreads)) != 0) - if (ret == 0) - ret = t_ret; + stop_threads(1, wtperf->ckptthreads); - if (monitor_created != 0 && - (t_ret = pthread_join(monitor_thread, NULL)) != 0) { - lprintf(wtperf, ret, 0, "Error joining monitor thread."); - if (ret == 0) - ret = t_ret; - } + if (monitor_created != 0) + testutil_check(__wt_thread_join(NULL, monitor_thread)); if (wtperf->conn != NULL && opts->close_conn && (t_ret = wtperf->conn->close(wtperf->conn, NULL)) != 0) { @@ -2728,14 +2698,13 @@ err: wtperf_free(wtperf); return (ret == 0 ? EXIT_SUCCESS : EXIT_FAILURE); } -static int -start_threads(WTPERF *wtperf, - WORKLOAD *workp, WTPERF_THREAD *base, u_int num, void *(*func)(void *)) +static void +start_threads(WTPERF *wtperf, WORKLOAD *workp, + WTPERF_THREAD *base, u_int num, WT_THREAD_CALLBACK(*func)(void *)) { CONFIG_OPTS *opts; WTPERF_THREAD *thread; u_int i; - int ret; opts = wtperf->opts; @@ -2779,29 +2748,20 @@ start_threads(WTPERF *wtperf, /* Start the threads. */ for (i = 0, thread = base; i < num; ++i, ++thread) - if ((ret = pthread_create( - &thread->handle, NULL, func, thread)) != 0) { - lprintf(wtperf, ret, 0, "Error creating thread"); - return (ret); - } - - return (0); + testutil_check(__wt_thread_create( + NULL, &thread->handle, func, thread)); } -static int -stop_threads(WTPERF *wtperf, u_int num, WTPERF_THREAD *threads) +static void +stop_threads(u_int num, WTPERF_THREAD *threads) { u_int i; - int ret; if (num == 0 || threads == NULL) - return (0); + return; for (i = 0; i < num; ++i, ++threads) { - if ((ret = pthread_join(threads->handle, NULL)) != 0) { - lprintf(wtperf, ret, 0, "Error joining thread"); - return (ret); - } + testutil_check(__wt_thread_join(NULL, threads->handle)); free(threads->key_buf); threads->key_buf = NULL; @@ -2815,7 +2775,6 @@ stop_threads(WTPERF *wtperf, u_int num, WTPERF_THREAD *threads) * being read by the monitor thread (among others). As a standalone * program, leaking memory isn't a concern, and it's simpler that way. */ - return (0); } static void diff --git a/src/third_party/wiredtiger/bench/wtperf/wtperf.h b/src/third_party/wiredtiger/bench/wtperf/wtperf.h index bd6c1e829ba..b17d082ddcf 100644 --- a/src/third_party/wiredtiger/bench/wtperf/wtperf.h +++ b/src/third_party/wiredtiger/bench/wtperf/wtperf.h @@ -232,7 +232,7 @@ struct __wtperf_thread { /* Per-thread structure */ WT_RAND_STATE rnd; /* Random number generation state */ - pthread_t handle; /* Handle */ + wt_thread_t handle; /* Handle */ char *key_buf, *value_buf; /* Key/value memory */ @@ -269,8 +269,8 @@ int run_truncate( int setup_log_file(WTPERF *); void setup_throttle(WTPERF_THREAD *); int setup_truncate(WTPERF *, WTPERF_THREAD *, WT_SESSION *); -int start_idle_table_cycle(WTPERF *, pthread_t *); -int stop_idle_table_cycle(WTPERF *, pthread_t); +void start_idle_table_cycle(WTPERF *, wt_thread_t *); +void stop_idle_table_cycle(WTPERF *, wt_thread_t); void worker_throttle(WTPERF_THREAD *); uint64_t sum_ckpt_ops(WTPERF *); uint64_t sum_insert_ops(WTPERF *); diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index 22600dd5e29..3297c68147a 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -529,6 +529,7 @@ connection_runtime_config = [ 'fileops', 'handleops', 'log', + 'lookaside_activity', 'lsm', 'lsm_manager', 'metadata', diff --git a/src/third_party/wiredtiger/dist/flags.py b/src/third_party/wiredtiger/dist/flags.py index d80c80a37ce..8edabd69648 100644 --- a/src/third_party/wiredtiger/dist/flags.py +++ b/src/third_party/wiredtiger/dist/flags.py @@ -67,6 +67,7 @@ flags = { 'VERB_FILEOPS', 'VERB_HANDLEOPS', 'VERB_LOG', + 'VERB_LOOKASIDE', 'VERB_LSM', 'VERB_LSM_MANAGER', 'VERB_METADATA', diff --git a/src/third_party/wiredtiger/dist/s_stat b/src/third_party/wiredtiger/dist/s_stat index 6aeeca6faa6..cf9303e5f95 100755 --- a/src/third_party/wiredtiger/dist/s_stat +++ b/src/third_party/wiredtiger/dist/s_stat @@ -25,15 +25,20 @@ cat << UNUSED_STAT_FIELDS lock_checkpoint_count lock_checkpoint_wait_application lock_checkpoint_wait_internal +lock_dhandle_read_count +lock_dhandle_wait_application +lock_dhandle_wait_internal +lock_dhandle_write_count lock_metadata_count lock_metadata_wait_application lock_metadata_wait_internal lock_schema_count lock_schema_wait_application lock_schema_wait_internal -lock_table_count +lock_table_read_count lock_table_wait_application lock_table_wait_internal +lock_table_write_count UNUSED_STAT_FIELDS echo "$search" diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index d5a562fcbd1..4ddb64297f4 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -305,6 +305,7 @@ RMW RNG RPC RUNDIR +RWLOCK RXB Radu ReadFile diff --git a/src/third_party/wiredtiger/dist/stat_data.py b/src/third_party/wiredtiger/dist/stat_data.py index acc156b947e..7b919848003 100644 --- a/src/third_party/wiredtiger/dist/stat_data.py +++ b/src/third_party/wiredtiger/dist/stat_data.py @@ -295,16 +295,20 @@ connection_stats = [ LockStat('lock_checkpoint_count', 'checkpoint lock acquisitions'), LockStat('lock_checkpoint_wait_application', 'checkpoint lock application thread wait time (usecs)'), LockStat('lock_checkpoint_wait_internal', 'checkpoint lock internal thread wait time (usecs)'), - LockStat('lock_handle_list_wait_eviction', 'handle-list lock eviction thread wait time (usecs)'), + LockStat('lock_dhandle_read_count', 'dhandle read lock acquisitions'), + LockStat('lock_dhandle_wait_application', 'dhandle lock application thread time waiting for the dhandle lock (usecs)'), + LockStat('lock_dhandle_wait_internal', 'dhandle lock internal thread time waiting for the dhandle lock (usecs)'), + LockStat('lock_dhandle_write_count', 'dhandle write lock acquisitions'), LockStat('lock_metadata_count', 'metadata lock acquisitions'), LockStat('lock_metadata_wait_application', 'metadata lock application thread wait time (usecs)'), LockStat('lock_metadata_wait_internal', 'metadata lock internal thread wait time (usecs)'), LockStat('lock_schema_count', 'schema lock acquisitions'), LockStat('lock_schema_wait_application', 'schema lock application thread wait time (usecs)'), LockStat('lock_schema_wait_internal', 'schema lock internal thread wait time (usecs)'), - LockStat('lock_table_count', 'table lock acquisitions'), + LockStat('lock_table_read_count', 'table read lock acquisitions'), LockStat('lock_table_wait_application', 'table lock application thread time waiting for the table lock (usecs)'), LockStat('lock_table_wait_internal', 'table lock internal thread time waiting for the table lock (usecs)'), + LockStat('lock_table_write_count', 'table write lock acquisitions'), ########################################## # Logging statistics @@ -436,6 +440,7 @@ connection_stats = [ TxnStat('txn_snapshots_created', 'number of named snapshots created'), TxnStat('txn_snapshots_dropped', 'number of named snapshots dropped'), TxnStat('txn_sync', 'transaction sync calls'), + TxnStat('txn_update_conflict', 'update conflicts'), ########################################## # Yield statistics diff --git a/src/third_party/wiredtiger/examples/c/ex_thread.c b/src/third_party/wiredtiger/examples/c/ex_thread.c index b69b3e9e7e9..ad2ff7f68a0 100644 --- a/src/third_party/wiredtiger/examples/c/ex_thread.c +++ b/src/third_party/wiredtiger/examples/c/ex_thread.c @@ -34,22 +34,14 @@ #include <stdlib.h> #include <string.h> -#ifndef _WIN32 -#include <pthread.h> -#else -#include "windows_shim.h" -#endif - -#include <wiredtiger.h> +#include "wt_internal.h" static const char *home; -void *scan_thread(void *arg); - #define NUM_THREADS 10 /*! [thread scan] */ -void * +static WT_THREAD_RET scan_thread(void *conn_arg) { WT_CONNECTION *conn; @@ -74,7 +66,7 @@ scan_thread(void *conn_arg) fprintf(stderr, "WT_CURSOR.next: %s\n", session->strerror(session, ret)); - return (NULL); + return (WT_THREAD_RET_VALUE); } /*! [thread scan] */ @@ -85,7 +77,7 @@ main(void) WT_CONNECTION *conn; WT_SESSION *session; WT_CURSOR *cursor; - pthread_t threads[NUM_THREADS]; + wt_thread_t threads[NUM_THREADS]; int i, ret; /* @@ -114,10 +106,10 @@ main(void) ret = session->close(session, NULL); for (i = 0; i < NUM_THREADS; i++) - ret = pthread_create(&threads[i], NULL, scan_thread, conn); + ret = __wt_thread_create(NULL, &threads[i], scan_thread, conn); for (i = 0; i < NUM_THREADS; i++) - ret = pthread_join(threads[i], NULL); + ret = __wt_thread_join(NULL, threads[i]); ret = conn->close(conn, NULL); diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 28fc97fbdd9..ec6d864a530 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -1,5 +1,5 @@ { - "commit": "7aaeaaa054d1ac27a95c79984f7ca69ba739caae", + "commit": "47e8c3d1d22018eaaa09f91dfd78addb49e0b49b", "github": "wiredtiger/wiredtiger.git", "vendor": "wiredtiger", "branch": "mongodb-3.6" diff --git a/src/third_party/wiredtiger/src/btree/bt_read.c b/src/third_party/wiredtiger/src/btree/bt_read.c index de84a711019..3f85e58f088 100644 --- a/src/third_party/wiredtiger/src/btree/bt_read.c +++ b/src/third_party/wiredtiger/src/btree/bt_read.c @@ -8,6 +8,8 @@ #include "wt_internal.h" +static void __btree_verbose_lookaside_read(WT_SESSION_IMPL *); + /* * __wt_las_remove_block -- * Remove all records matching a key prefix from the lookaside store. @@ -19,8 +21,7 @@ __wt_las_remove_block(WT_SESSION_IMPL *session, WT_DECL_ITEM(las_addr); WT_DECL_ITEM(las_key); WT_DECL_RET; - uint64_t las_counter, las_txnid; - int64_t remove_cnt; + uint64_t las_counter, las_txnid, remove_cnt; uint32_t las_id; int exact; @@ -74,7 +75,7 @@ err: __wt_scr_free(session, &las_addr); if (remove_cnt > S2C(session)->las_record_cnt) S2C(session)->las_record_cnt = 0; else if (remove_cnt > 0) - (void)__wt_atomic_subi64( + (void)__wt_atomic_sub64( &S2C(session)->las_record_cnt, remove_cnt); return (ret); @@ -451,6 +452,7 @@ __page_read(WT_SESSION_IMPL *session, WT_REF *ref) */ dsk = tmp.data; if (F_ISSET(dsk, WT_PAGE_LAS_UPDATE) && __wt_las_is_written(session)) { + __btree_verbose_lookaside_read(session); WT_STAT_CONN_INCR(session, cache_read_lookaside); WT_STAT_DATA_INCR(session, cache_read_lookaside); @@ -680,3 +682,43 @@ skip_evict: __wt_sleep(0, sleep_cnt); } } + +/* + * __btree_verbose_lookaside_read -- + * Create a verbose message to display at most once per checkpoint when + * performing a lookaside table read. + */ +static void +__btree_verbose_lookaside_read(WT_SESSION_IMPL *session) +{ +#ifdef HAVE_VERBOSE + WT_CONNECTION_IMPL *conn; + uint64_t ckpt_gen_current, ckpt_gen_last; + + if (!WT_VERBOSE_ISSET(session, WT_VERB_LOOKASIDE)) return; + + conn = S2C(session); + ckpt_gen_current = __wt_gen(session, WT_GEN_CHECKPOINT); + ckpt_gen_last = conn->las_verb_gen_read; + + /* + * This message is throttled to one per checkpoint. To do this we + * track the generation of the last checkpoint for which the message + * was printed and check against the current checkpoint generation. + */ + if (ckpt_gen_current > ckpt_gen_last) { + /* + * Attempt to atomically replace the last checkpoint generation + * for which this message was printed. If the atomic swap fails + * we have raced and the winning thread will print the message. + */ + if (__wt_atomic_casv64(&conn->las_verb_gen_read, + ckpt_gen_last, ckpt_gen_current)) { + __wt_verbose(session, WT_VERB_LOOKASIDE, + "Read from lookaside file triggered."); + } + } +#else + WT_UNUSED(session); +#endif +} diff --git a/src/third_party/wiredtiger/src/cache/cache_las.c b/src/third_party/wiredtiger/src/cache/cache_las.c index 06c6354148c..a2233514223 100644 --- a/src/third_party/wiredtiger/src/cache/cache_las.c +++ b/src/third_party/wiredtiger/src/cache/cache_las.c @@ -292,8 +292,7 @@ __wt_las_sweep(WT_SESSION_IMPL *session) WT_DECL_ITEM(las_key); WT_DECL_RET; WT_ITEM *key; - uint64_t cnt, las_counter, las_txnid; - int64_t remove_cnt; + uint64_t cnt, las_counter, las_txnid, remove_cnt; uint32_t las_id, session_flags; int notused; @@ -342,7 +341,7 @@ __wt_las_sweep(WT_SESSION_IMPL *session) * blocks in the cache in order to get rid of them, and slowly review * lookaside blocks that have already been evicted. */ - cnt = (uint64_t)WT_MAX(100, conn->las_record_cnt / 30); + cnt = WT_MAX(100, conn->las_record_cnt / 30); /* Discard pages we read as soon as we're done with them. */ F_SET(session, WT_SESSION_NO_CACHE); @@ -390,14 +389,13 @@ err: __wt_buf_free(session, key); WT_TRET(__wt_las_cursor_close(session, &cursor, session_flags)); /* - * If there were races to remove records, we can over-count. All - * arithmetic is signed, so underflow isn't fatal, but check anyway so - * we don't skew low over time. + * If there were races to remove records, we can over-count. Underflow + * isn't fatal, but check anyway so we don't skew low over time. */ if (remove_cnt > conn->las_record_cnt) conn->las_record_cnt = 0; else if (remove_cnt > 0) - (void)__wt_atomic_subi64(&conn->las_record_cnt, remove_cnt); + (void)__wt_atomic_sub64(&conn->las_record_cnt, remove_cnt); F_CLR(session, WT_SESSION_NO_CACHE); diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index f152fbacad4..a7397d21c6a 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -148,11 +148,12 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = { { "verbose", "list", NULL, "choices=[\"api\",\"block\",\"checkpoint\",\"compact\"," "\"evict\",\"evict_stuck\",\"evictserver\",\"fileops\"," - "\"handleops\",\"log\",\"lsm\",\"lsm_manager\",\"metadata\"," - "\"mutex\",\"overflow\",\"read\",\"rebalance\",\"reconcile\"," - "\"recovery\",\"recovery_progress\",\"salvage\",\"shared_cache\"," - "\"split\",\"temporary\",\"thread_group\",\"transaction\"," - "\"verify\",\"version\",\"write\"]", + "\"handleops\",\"log\",\"lookaside_activity\",\"lsm\"," + "\"lsm_manager\",\"metadata\",\"mutex\",\"overflow\",\"read\"," + "\"rebalance\",\"reconcile\",\"recovery\",\"recovery_progress\"," + "\"salvage\",\"shared_cache\",\"split\",\"temporary\"," + "\"thread_group\",\"transaction\",\"verify\",\"version\"," + "\"write\"]", NULL, 0 }, { NULL, NULL, NULL, NULL, NULL, 0 } }; @@ -751,11 +752,12 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { { "verbose", "list", NULL, "choices=[\"api\",\"block\",\"checkpoint\",\"compact\"," "\"evict\",\"evict_stuck\",\"evictserver\",\"fileops\"," - "\"handleops\",\"log\",\"lsm\",\"lsm_manager\",\"metadata\"," - "\"mutex\",\"overflow\",\"read\",\"rebalance\",\"reconcile\"," - "\"recovery\",\"recovery_progress\",\"salvage\",\"shared_cache\"," - "\"split\",\"temporary\",\"thread_group\",\"transaction\"," - "\"verify\",\"version\",\"write\"]", + "\"handleops\",\"log\",\"lookaside_activity\",\"lsm\"," + "\"lsm_manager\",\"metadata\",\"mutex\",\"overflow\",\"read\"," + "\"rebalance\",\"reconcile\",\"recovery\",\"recovery_progress\"," + "\"salvage\",\"shared_cache\",\"split\",\"temporary\"," + "\"thread_group\",\"transaction\",\"verify\",\"version\"," + "\"write\"]", NULL, 0 }, { "write_through", "list", NULL, "choices=[\"data\",\"log\"]", @@ -838,11 +840,12 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { { "verbose", "list", NULL, "choices=[\"api\",\"block\",\"checkpoint\",\"compact\"," "\"evict\",\"evict_stuck\",\"evictserver\",\"fileops\"," - "\"handleops\",\"log\",\"lsm\",\"lsm_manager\",\"metadata\"," - "\"mutex\",\"overflow\",\"read\",\"rebalance\",\"reconcile\"," - "\"recovery\",\"recovery_progress\",\"salvage\",\"shared_cache\"," - "\"split\",\"temporary\",\"thread_group\",\"transaction\"," - "\"verify\",\"version\",\"write\"]", + "\"handleops\",\"log\",\"lookaside_activity\",\"lsm\"," + "\"lsm_manager\",\"metadata\",\"mutex\",\"overflow\",\"read\"," + "\"rebalance\",\"reconcile\",\"recovery\",\"recovery_progress\"," + "\"salvage\",\"shared_cache\",\"split\",\"temporary\"," + "\"thread_group\",\"transaction\",\"verify\",\"version\"," + "\"write\"]", NULL, 0 }, { "version", "string", NULL, NULL, NULL, 0 }, { "write_through", "list", @@ -920,11 +923,12 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { { "verbose", "list", NULL, "choices=[\"api\",\"block\",\"checkpoint\",\"compact\"," "\"evict\",\"evict_stuck\",\"evictserver\",\"fileops\"," - "\"handleops\",\"log\",\"lsm\",\"lsm_manager\",\"metadata\"," - "\"mutex\",\"overflow\",\"read\",\"rebalance\",\"reconcile\"," - "\"recovery\",\"recovery_progress\",\"salvage\",\"shared_cache\"," - "\"split\",\"temporary\",\"thread_group\",\"transaction\"," - "\"verify\",\"version\",\"write\"]", + "\"handleops\",\"log\",\"lookaside_activity\",\"lsm\"," + "\"lsm_manager\",\"metadata\",\"mutex\",\"overflow\",\"read\"," + "\"rebalance\",\"reconcile\",\"recovery\",\"recovery_progress\"," + "\"salvage\",\"shared_cache\",\"split\",\"temporary\"," + "\"thread_group\",\"transaction\",\"verify\",\"version\"," + "\"write\"]", NULL, 0 }, { "version", "string", NULL, NULL, NULL, 0 }, { "write_through", "list", @@ -1002,11 +1006,12 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { { "verbose", "list", NULL, "choices=[\"api\",\"block\",\"checkpoint\",\"compact\"," "\"evict\",\"evict_stuck\",\"evictserver\",\"fileops\"," - "\"handleops\",\"log\",\"lsm\",\"lsm_manager\",\"metadata\"," - "\"mutex\",\"overflow\",\"read\",\"rebalance\",\"reconcile\"," - "\"recovery\",\"recovery_progress\",\"salvage\",\"shared_cache\"," - "\"split\",\"temporary\",\"thread_group\",\"transaction\"," - "\"verify\",\"version\",\"write\"]", + "\"handleops\",\"log\",\"lookaside_activity\",\"lsm\"," + "\"lsm_manager\",\"metadata\",\"mutex\",\"overflow\",\"read\"," + "\"rebalance\",\"reconcile\",\"recovery\",\"recovery_progress\"," + "\"salvage\",\"shared_cache\",\"split\",\"temporary\"," + "\"thread_group\",\"transaction\",\"verify\",\"version\"," + "\"write\"]", NULL, 0 }, { "write_through", "list", NULL, "choices=[\"data\",\"log\"]", diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index c0a1f5c0920..70e96aa8473 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -1803,6 +1803,7 @@ __wt_verbose_config(WT_SESSION_IMPL *session, const char *cfg[]) { "fileops", WT_VERB_FILEOPS }, { "handleops", WT_VERB_HANDLEOPS }, { "log", WT_VERB_LOG }, + { "lookaside_activity", WT_VERB_LOOKASIDE }, { "lsm", WT_VERB_LSM }, { "lsm_manager", WT_VERB_LSM_MANAGER }, { "metadata", WT_VERB_METADATA }, diff --git a/src/third_party/wiredtiger/src/conn/conn_dhandle.c b/src/third_party/wiredtiger/src/conn/conn_dhandle.c index d4670562eb8..97fdc7557ee 100644 --- a/src/third_party/wiredtiger/src/conn/conn_dhandle.c +++ b/src/third_party/wiredtiger/src/conn/conn_dhandle.c @@ -481,6 +481,49 @@ err: WT_DHANDLE_RELEASE(dhandle); } /* + * __conn_dhandle_close_one -- + * Lock and, if necessary, close a data handle. + */ +static int +__conn_dhandle_close_one(WT_SESSION_IMPL *session, + const char *uri, const char *checkpoint, bool force) +{ + WT_DECL_RET; + + /* + * Lock the handle exclusively. If this is part of schema-changing + * operation (indicated by metadata tracking being enabled), hold the + * lock for the duration of the operation. + */ + WT_RET(__wt_session_get_btree(session, uri, checkpoint, + NULL, WT_DHANDLE_EXCLUSIVE | WT_DHANDLE_LOCK_ONLY)); + if (WT_META_TRACKING(session)) + WT_RET(__wt_meta_track_handle_lock(session, false)); + + /* + * We have an exclusive lock, which means there are no cursors open at + * this point. Close the handle, if necessary. + */ + if (F_ISSET(session->dhandle, WT_DHANDLE_OPEN)) { + __wt_meta_track_sub_on(session); + ret = __wt_conn_btree_sync_and_close(session, false, force); + + /* + * If the close succeeded, drop any locks it acquired. If + * there was a failure, this function will fail and the whole + * transaction will be rolled back. + */ + if (ret == 0) + ret = __wt_meta_track_sub_off(session); + } + + if (!WT_META_TRACKING(session)) + WT_TRET(__wt_session_release_btree(session)); + + return (ret); +} + +/* * __wt_conn_dhandle_close_all -- * Close all data handles handles with matching name (including all * checkpoint handles). @@ -500,48 +543,22 @@ __wt_conn_dhandle_close_all( F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST_WRITE)); WT_ASSERT(session, session->dhandle == NULL); + /* + * Lock the live handle first. This ordering is important: we rely on + * locking the live handle to fail fast if the tree is busy (e.g., with + * cursors open or in a checkpoint). + */ + WT_ERR(__conn_dhandle_close_one(session, uri, NULL, force)); + bucket = __wt_hash_city64(uri, strlen(uri)) % WT_HASH_ARRAY_SIZE; TAILQ_FOREACH(dhandle, &conn->dhhash[bucket], hashq) { if (strcmp(dhandle->name, uri) != 0 || + dhandle->checkpoint == NULL || F_ISSET(dhandle, WT_DHANDLE_DEAD)) continue; - session->dhandle = dhandle; - - /* - * Lock the handle exclusively. If this is part of - * schema-changing operation (indicated by metadata tracking - * being enabled), hold the lock for the duration of the - * operation. - */ - WT_ERR(__wt_session_get_btree(session, - dhandle->name, dhandle->checkpoint, - NULL, WT_DHANDLE_EXCLUSIVE | WT_DHANDLE_LOCK_ONLY)); - if (WT_META_TRACKING(session)) - WT_ERR(__wt_meta_track_handle_lock(session, false)); - - /* - * We have an exclusive lock, which means there are no cursors - * open at this point. Close the handle, if necessary. - */ - if (F_ISSET(dhandle, WT_DHANDLE_OPEN)) { - __wt_meta_track_sub_on(session); - ret = __wt_conn_btree_sync_and_close( - session, false, force); - - /* - * If the close succeeded, drop any locks it acquired. - * If there was a failure, this function will fail and - * the whole transaction will be rolled back. - */ - if (ret == 0) - ret = __wt_meta_track_sub_off(session); - } - - if (!WT_META_TRACKING(session)) - WT_TRET(__wt_session_release_btree(session)); - - WT_ERR(ret); + WT_ERR(__conn_dhandle_close_one( + session, dhandle->name, dhandle->checkpoint, force)); } err: session->dhandle = NULL; diff --git a/src/third_party/wiredtiger/src/conn/conn_handle.c b/src/third_party/wiredtiger/src/conn/conn_handle.c index 32a0d80c1f3..2f3f9488b58 100644 --- a/src/third_party/wiredtiger/src/conn/conn_handle.c +++ b/src/third_party/wiredtiger/src/conn/conn_handle.c @@ -62,9 +62,9 @@ __wt_connection_init(WT_CONNECTION_IMPL *conn) WT_RET(__wt_spin_init(session, &conn->turtle_lock, "turtle file")); /* Read-write locks */ - WT_RET(__wt_rwlock_init(session, &conn->dhandle_lock)); + WT_RWLOCK_INIT_TRACKED(session, &conn->dhandle_lock, dhandle); WT_RET(__wt_rwlock_init(session, &conn->hot_backup_lock)); - WT_RET(__wt_rwlock_init(session, &conn->table_lock)); + WT_RWLOCK_INIT_TRACKED(session, &conn->table_lock, table); /* Setup the spin locks for the LSM manager queues. */ WT_RET(__wt_spin_init(session, diff --git a/src/third_party/wiredtiger/src/cursor/cur_table.c b/src/third_party/wiredtiger/src/cursor/cur_table.c index 3959d58476b..000fcae99f2 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_table.c +++ b/src/third_party/wiredtiger/src/cursor/cur_table.c @@ -988,6 +988,12 @@ __wt_curtable_open(WT_SESSION_IMPL *session, table->cgroups[0]->source, NULL, cfg, cursorp); __wt_schema_release_table(session, table); + if (ret == 0) { + /* Fix up the public URI to match what was passed in. */ + cursor = *cursorp; + __wt_free(session, cursor->uri); + WT_TRET(__wt_strdup(session, uri, &cursor->uri)); + } return (ret); } diff --git a/src/third_party/wiredtiger/src/docs/backup.dox b/src/third_party/wiredtiger/src/docs/backup.dox index 45edc85d6a5..91b15da9275 100644 --- a/src/third_party/wiredtiger/src/docs/backup.dox +++ b/src/third_party/wiredtiger/src/docs/backup.dox @@ -59,6 +59,11 @@ During the period the backup cursor is open, database checkpoints can be created, but no checkpoints can be deleted. This may result in significant file growth. +Additionally, if a crash occurs during the period the backup cursor is open and +logging is disabled, then the system will be restored to the most recent +checkpoint prior to the opening of the backup cursor, even if later database +checkpoints were created. + The following is a programmatic example of creating a backup: @snippet ex_all.c backup diff --git a/src/third_party/wiredtiger/src/evict/evict_lru.c b/src/third_party/wiredtiger/src/evict/evict_lru.c index b5dd3837531..46291eb63de 100644 --- a/src/third_party/wiredtiger/src/evict/evict_lru.c +++ b/src/third_party/wiredtiger/src/evict/evict_lru.c @@ -31,28 +31,17 @@ static int __evict_walk_file( static int __evict_lock_handle_list(WT_SESSION_IMPL *session) { - struct timespec enter, leave; WT_CACHE *cache; WT_CONNECTION_IMPL *conn; WT_DECL_RET; WT_RWLOCK *dh_lock; u_int spins; - bool dh_stats; conn = S2C(session); cache = conn->cache; dh_lock = &conn->dhandle_lock; /* - * Setup tracking of handle lock acquisition wait time if statistics - * are enabled. - */ - dh_stats = WT_STAT_ENABLED(session); - - if (dh_stats) - __wt_epoch(session, &enter); - - /* * Use a custom lock acquisition back off loop so the eviction server * notices any interrupt quickly. */ @@ -64,17 +53,7 @@ __evict_lock_handle_list(WT_SESSION_IMPL *session) else __wt_sleep(0, WT_THOUSAND); } - /* - * Only record statistics on success. - */ - WT_RET(ret); - if (dh_stats) { - __wt_epoch(session, &leave); - WT_STAT_CONN_INCRV( - session, lock_handle_list_wait_eviction, - (int64_t)WT_TIMEDIFF_US(leave, enter)); - } - return (0); + return (ret); } /* diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h index bf2f8a2c7e1..56d801cd361 100644 --- a/src/third_party/wiredtiger/src/include/connection.h +++ b/src/third_party/wiredtiger/src/include/connection.h @@ -360,7 +360,15 @@ struct __wt_connection_impl { bool las_written; /* Lookaside table has been written */ WT_ITEM las_sweep_key; /* Sweep server's saved key */ - int64_t las_record_cnt;/* Count of lookaside records */ + uint64_t las_record_cnt;/* Count of lookaside records */ + + /* + * The "lookaside_activity" verbose messages are throttled to once per + * checkpoint. To accomplish this we track the checkpoint generation + * for the most recent read and write verbose messages. + */ + volatile uint64_t las_verb_gen_read; + volatile uint64_t las_verb_gen_write; /* Locked: collator list */ TAILQ_HEAD(__wt_coll_qh, __wt_named_collator) collqh; diff --git a/src/third_party/wiredtiger/src/include/extern_posix.h b/src/third_party/wiredtiger/src/include/extern_posix.h index c0ed056c7b6..b6b5ac51f73 100644 --- a/src/third_party/wiredtiger/src/include/extern_posix.h +++ b/src/third_party/wiredtiger/src/include/extern_posix.h @@ -25,8 +25,8 @@ extern void __wt_stream_set_line_buffer(FILE *fp) WT_GCC_FUNC_DECL_ATTRIBUTE((vi extern void __wt_stream_set_no_buffer(FILE *fp) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern void __wt_sleep(uint64_t seconds, uint64_t micro_seconds) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern int __wt_vsnprintf_len_incr( char *buf, size_t size, size_t *retsizep, const char *fmt, va_list ap) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_thread_create(WT_SESSION_IMPL *session, wt_thread_t *tidret, WT_THREAD_CALLBACK(*func)(void *), void *arg) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_thread_join(WT_SESSION_IMPL *session, wt_thread_t tid) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_thread_create(WT_SESSION_IMPL *session, wt_thread_t *tidret, WT_THREAD_CALLBACK(*func)(void *), void *arg) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_thread_join(WT_SESSION_IMPL *session, wt_thread_t tid) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_thread_id(char *buf, size_t buflen) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_epoch(WT_SESSION_IMPL *session, struct timespec *tsp) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern void __wt_yield(void) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); diff --git a/src/third_party/wiredtiger/src/include/flags.h b/src/third_party/wiredtiger/src/include/flags.h index d7c0e0f9472..919c0dd2f98 100644 --- a/src/third_party/wiredtiger/src/include/flags.h +++ b/src/third_party/wiredtiger/src/include/flags.h @@ -95,25 +95,26 @@ #define WT_VERB_FILEOPS 0x00000080 #define WT_VERB_HANDLEOPS 0x00000100 #define WT_VERB_LOG 0x00000200 -#define WT_VERB_LSM 0x00000400 -#define WT_VERB_LSM_MANAGER 0x00000800 -#define WT_VERB_METADATA 0x00001000 -#define WT_VERB_MUTEX 0x00002000 -#define WT_VERB_OVERFLOW 0x00004000 -#define WT_VERB_READ 0x00008000 -#define WT_VERB_REBALANCE 0x00010000 -#define WT_VERB_RECONCILE 0x00020000 -#define WT_VERB_RECOVERY 0x00040000 -#define WT_VERB_RECOVERY_PROGRESS 0x00080000 -#define WT_VERB_SALVAGE 0x00100000 -#define WT_VERB_SHARED_CACHE 0x00200000 -#define WT_VERB_SPLIT 0x00400000 -#define WT_VERB_TEMPORARY 0x00800000 -#define WT_VERB_THREAD_GROUP 0x01000000 -#define WT_VERB_TRANSACTION 0x02000000 -#define WT_VERB_VERIFY 0x04000000 -#define WT_VERB_VERSION 0x08000000 -#define WT_VERB_WRITE 0x10000000 +#define WT_VERB_LOOKASIDE 0x00000400 +#define WT_VERB_LSM 0x00000800 +#define WT_VERB_LSM_MANAGER 0x00001000 +#define WT_VERB_METADATA 0x00002000 +#define WT_VERB_MUTEX 0x00004000 +#define WT_VERB_OVERFLOW 0x00008000 +#define WT_VERB_READ 0x00010000 +#define WT_VERB_REBALANCE 0x00020000 +#define WT_VERB_RECONCILE 0x00040000 +#define WT_VERB_RECOVERY 0x00080000 +#define WT_VERB_RECOVERY_PROGRESS 0x00100000 +#define WT_VERB_SALVAGE 0x00200000 +#define WT_VERB_SHARED_CACHE 0x00400000 +#define WT_VERB_SPLIT 0x00800000 +#define WT_VERB_TEMPORARY 0x01000000 +#define WT_VERB_THREAD_GROUP 0x02000000 +#define WT_VERB_TRANSACTION 0x04000000 +#define WT_VERB_VERIFY 0x08000000 +#define WT_VERB_VERSION 0x10000000 +#define WT_VERB_WRITE 0x20000000 #define WT_VISIBILITY_ERR 0x00000080 /* * flags section: END diff --git a/src/third_party/wiredtiger/src/include/mutex.h b/src/third_party/wiredtiger/src/include/mutex.h index 5f814c2799e..7aeb6160f43 100644 --- a/src/third_party/wiredtiger/src/include/mutex.h +++ b/src/third_party/wiredtiger/src/include/mutex.h @@ -50,11 +50,35 @@ struct __wt_rwlock { /* Read/write lock */ } s; } u; + int16_t stat_read_count_off; /* read acquisitions offset */ + int16_t stat_write_count_off; /* write acquisitions offset */ + int16_t stat_app_usecs_off; /* waiting application threads offset */ + int16_t stat_int_usecs_off; /* waiting server threads offset */ + WT_CONDVAR *cond_readers; /* Blocking readers */ WT_CONDVAR *cond_writers; /* Blocking writers */ }; /* + * WT_RWLOCK_INIT_TRACKED -- + * Read write lock initialization, with tracking. + * + * Implemented as a macro so we can pass in a statistics field and convert + * it into a statistics structure array offset. + */ +#define WT_RWLOCK_INIT_TRACKED(session, l, name) do { \ + WT_RET(__wt_rwlock_init(session, l)); \ + (l)->stat_read_count_off = (int16_t)WT_STATS_FIELD_TO_OFFSET( \ + S2C(session)->stats, lock_##name##_read_count); \ + (l)->stat_write_count_off = (int16_t)WT_STATS_FIELD_TO_OFFSET( \ + S2C(session)->stats, lock_##name##_write_count); \ + (l)->stat_app_usecs_off = (int16_t)WT_STATS_FIELD_TO_OFFSET( \ + S2C(session)->stats, lock_##name##_wait_application); \ + (l)->stat_int_usecs_off = (int16_t)WT_STATS_FIELD_TO_OFFSET( \ + S2C(session)->stats, lock_##name##_wait_internal); \ +} while (0) + +/* * Spin locks: * * WiredTiger uses spinlocks for fast mutual exclusion (where operations done diff --git a/src/third_party/wiredtiger/src/include/stat.h b/src/third_party/wiredtiger/src/include/stat.h index 7c2529f1746..7d7d701590a 100644 --- a/src/third_party/wiredtiger/src/include/stat.h +++ b/src/third_party/wiredtiger/src/include/stat.h @@ -399,16 +399,20 @@ struct __wt_connection_stats { int64_t lock_checkpoint_count; int64_t lock_checkpoint_wait_application; int64_t lock_checkpoint_wait_internal; - int64_t lock_handle_list_wait_eviction; + int64_t lock_dhandle_wait_application; + int64_t lock_dhandle_wait_internal; + int64_t lock_dhandle_read_count; + int64_t lock_dhandle_write_count; int64_t lock_metadata_count; int64_t lock_metadata_wait_application; int64_t lock_metadata_wait_internal; int64_t lock_schema_count; int64_t lock_schema_wait_application; int64_t lock_schema_wait_internal; - int64_t lock_table_count; int64_t lock_table_wait_application; int64_t lock_table_wait_internal; + int64_t lock_table_read_count; + int64_t lock_table_write_count; int64_t log_slot_switch_busy; int64_t log_bytes_payload; int64_t log_bytes_written; @@ -513,6 +517,7 @@ struct __wt_connection_stats { int64_t txn_sync; int64_t txn_commit; int64_t txn_rollback; + int64_t txn_update_conflict; }; /* diff --git a/src/third_party/wiredtiger/src/include/txn.i b/src/third_party/wiredtiger/src/include/txn.i index f7321af5b12..f4f571cb67e 100644 --- a/src/third_party/wiredtiger/src/include/txn.i +++ b/src/third_party/wiredtiger/src/include/txn.i @@ -424,6 +424,8 @@ __wt_txn_update_check(WT_SESSION_IMPL *session, WT_UPDATE *upd) if (txn->isolation == WT_ISO_SNAPSHOT) while (upd != NULL && !__wt_txn_visible(session, upd->txnid)) { if (upd->txnid != WT_TXN_ABORTED) { + WT_STAT_CONN_INCR( + session, txn_update_conflict); WT_STAT_DATA_INCR( session, txn_update_conflict); return (WT_ROLLBACK); diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index 2bbe812d7f7..cf7117376af 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -2087,12 +2087,12 @@ struct __wt_connection { * list\, with values chosen from the following options: \c "api"\, \c * "block"\, \c "checkpoint"\, \c "compact"\, \c "evict"\, \c * "evict_stuck"\, \c "evictserver"\, \c "fileops"\, \c "handleops"\, \c - * "log"\, \c "lsm"\, \c "lsm_manager"\, \c "metadata"\, \c "mutex"\, \c - * "overflow"\, \c "read"\, \c "rebalance"\, \c "reconcile"\, \c - * "recovery"\, \c "recovery_progress"\, \c "salvage"\, \c - * "shared_cache"\, \c "split"\, \c "temporary"\, \c "thread_group"\, \c - * "transaction"\, \c "verify"\, \c "version"\, \c "write"; default - * empty.} + * "log"\, \c "lookaside_activity"\, \c "lsm"\, \c "lsm_manager"\, \c + * "metadata"\, \c "mutex"\, \c "overflow"\, \c "read"\, \c + * "rebalance"\, \c "reconcile"\, \c "recovery"\, \c + * "recovery_progress"\, \c "salvage"\, \c "shared_cache"\, \c "split"\, + * \c "temporary"\, \c "thread_group"\, \c "transaction"\, \c "verify"\, + * \c "version"\, \c "write"; default empty.} * @configend * @errors */ @@ -2619,12 +2619,12 @@ struct __wt_connection { * list\, such as <code>"verbose=[evictserver\,read]"</code>., a list\, with * values chosen from the following options: \c "api"\, \c "block"\, \c * "checkpoint"\, \c "compact"\, \c "evict"\, \c "evict_stuck"\, \c - * "evictserver"\, \c "fileops"\, \c "handleops"\, \c "log"\, \c "lsm"\, \c - * "lsm_manager"\, \c "metadata"\, \c "mutex"\, \c "overflow"\, \c "read"\, \c - * "rebalance"\, \c "reconcile"\, \c "recovery"\, \c "recovery_progress"\, \c - * "salvage"\, \c "shared_cache"\, \c "split"\, \c "temporary"\, \c - * "thread_group"\, \c "transaction"\, \c "verify"\, \c "version"\, \c "write"; - * default empty.} + * "evictserver"\, \c "fileops"\, \c "handleops"\, \c "log"\, \c + * "lookaside_activity"\, \c "lsm"\, \c "lsm_manager"\, \c "metadata"\, \c + * "mutex"\, \c "overflow"\, \c "read"\, \c "rebalance"\, \c "reconcile"\, \c + * "recovery"\, \c "recovery_progress"\, \c "salvage"\, \c "shared_cache"\, \c + * "split"\, \c "temporary"\, \c "thread_group"\, \c "transaction"\, \c + * "verify"\, \c "version"\, \c "write"; default empty.} * @config{write_through, Use \c FILE_FLAG_WRITE_THROUGH on Windows to write to * files. Ignored on non-Windows systems. Options are given as a list\, such * as <code>"write_through=[data]"</code>. Configuring \c write_through requires @@ -4740,252 +4740,268 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_APPLICATION 1140 /*! lock: checkpoint lock internal thread wait time (usecs) */ #define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_INTERNAL 1141 -/*! lock: handle-list lock eviction thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_HANDLE_LIST_WAIT_EVICTION 1142 +/*! + * lock: dhandle lock application thread time waiting for the dhandle + * lock (usecs) + */ +#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_APPLICATION 1142 +/*! + * lock: dhandle lock internal thread time waiting for the dhandle lock + * (usecs) + */ +#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_INTERNAL 1143 +/*! lock: dhandle read lock acquisitions */ +#define WT_STAT_CONN_LOCK_DHANDLE_READ_COUNT 1144 +/*! lock: dhandle write lock acquisitions */ +#define WT_STAT_CONN_LOCK_DHANDLE_WRITE_COUNT 1145 /*! lock: metadata lock acquisitions */ -#define WT_STAT_CONN_LOCK_METADATA_COUNT 1143 +#define WT_STAT_CONN_LOCK_METADATA_COUNT 1146 /*! lock: metadata lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_METADATA_WAIT_APPLICATION 1144 +#define WT_STAT_CONN_LOCK_METADATA_WAIT_APPLICATION 1147 /*! lock: metadata lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_METADATA_WAIT_INTERNAL 1145 +#define WT_STAT_CONN_LOCK_METADATA_WAIT_INTERNAL 1148 /*! lock: schema lock acquisitions */ -#define WT_STAT_CONN_LOCK_SCHEMA_COUNT 1146 +#define WT_STAT_CONN_LOCK_SCHEMA_COUNT 1149 /*! lock: schema lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_APPLICATION 1147 +#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_APPLICATION 1150 /*! lock: schema lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_INTERNAL 1148 -/*! lock: table lock acquisitions */ -#define WT_STAT_CONN_LOCK_TABLE_COUNT 1149 +#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_INTERNAL 1151 /*! * lock: table lock application thread time waiting for the table lock * (usecs) */ -#define WT_STAT_CONN_LOCK_TABLE_WAIT_APPLICATION 1150 +#define WT_STAT_CONN_LOCK_TABLE_WAIT_APPLICATION 1152 /*! * lock: table lock internal thread time waiting for the table lock * (usecs) */ -#define WT_STAT_CONN_LOCK_TABLE_WAIT_INTERNAL 1151 +#define WT_STAT_CONN_LOCK_TABLE_WAIT_INTERNAL 1153 +/*! lock: table read lock acquisitions */ +#define WT_STAT_CONN_LOCK_TABLE_READ_COUNT 1154 +/*! lock: table write lock acquisitions */ +#define WT_STAT_CONN_LOCK_TABLE_WRITE_COUNT 1155 /*! log: busy returns attempting to switch slots */ -#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1152 +#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1156 /*! log: log bytes of payload data */ -#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1153 +#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1157 /*! log: log bytes written */ -#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1154 +#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1158 /*! log: log files manually zero-filled */ -#define WT_STAT_CONN_LOG_ZERO_FILLS 1155 +#define WT_STAT_CONN_LOG_ZERO_FILLS 1159 /*! log: log flush operations */ -#define WT_STAT_CONN_LOG_FLUSH 1156 +#define WT_STAT_CONN_LOG_FLUSH 1160 /*! log: log force write operations */ -#define WT_STAT_CONN_LOG_FORCE_WRITE 1157 +#define WT_STAT_CONN_LOG_FORCE_WRITE 1161 /*! log: log force write operations skipped */ -#define WT_STAT_CONN_LOG_FORCE_WRITE_SKIP 1158 +#define WT_STAT_CONN_LOG_FORCE_WRITE_SKIP 1162 /*! log: log records compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1159 +#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1163 /*! log: log records not compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1160 +#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1164 /*! log: log records too small to compress */ -#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1161 +#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1165 /*! log: log release advances write LSN */ -#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1162 +#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1166 /*! log: log scan operations */ -#define WT_STAT_CONN_LOG_SCANS 1163 +#define WT_STAT_CONN_LOG_SCANS 1167 /*! log: log scan records requiring two reads */ -#define WT_STAT_CONN_LOG_SCAN_REREADS 1164 +#define WT_STAT_CONN_LOG_SCAN_REREADS 1168 /*! log: log server thread advances write LSN */ -#define WT_STAT_CONN_LOG_WRITE_LSN 1165 +#define WT_STAT_CONN_LOG_WRITE_LSN 1169 /*! log: log server thread write LSN walk skipped */ -#define WT_STAT_CONN_LOG_WRITE_LSN_SKIP 1166 +#define WT_STAT_CONN_LOG_WRITE_LSN_SKIP 1170 /*! log: log sync operations */ -#define WT_STAT_CONN_LOG_SYNC 1167 +#define WT_STAT_CONN_LOG_SYNC 1171 /*! log: log sync time duration (usecs) */ -#define WT_STAT_CONN_LOG_SYNC_DURATION 1168 +#define WT_STAT_CONN_LOG_SYNC_DURATION 1172 /*! log: log sync_dir operations */ -#define WT_STAT_CONN_LOG_SYNC_DIR 1169 +#define WT_STAT_CONN_LOG_SYNC_DIR 1173 /*! log: log sync_dir time duration (usecs) */ -#define WT_STAT_CONN_LOG_SYNC_DIR_DURATION 1170 +#define WT_STAT_CONN_LOG_SYNC_DIR_DURATION 1174 /*! log: log write operations */ -#define WT_STAT_CONN_LOG_WRITES 1171 +#define WT_STAT_CONN_LOG_WRITES 1175 /*! log: logging bytes consolidated */ -#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1172 +#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1176 /*! log: maximum log file size */ -#define WT_STAT_CONN_LOG_MAX_FILESIZE 1173 +#define WT_STAT_CONN_LOG_MAX_FILESIZE 1177 /*! log: number of pre-allocated log files to create */ -#define WT_STAT_CONN_LOG_PREALLOC_MAX 1174 +#define WT_STAT_CONN_LOG_PREALLOC_MAX 1178 /*! log: pre-allocated log files not ready and missed */ -#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1175 +#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1179 /*! log: pre-allocated log files prepared */ -#define WT_STAT_CONN_LOG_PREALLOC_FILES 1176 +#define WT_STAT_CONN_LOG_PREALLOC_FILES 1180 /*! log: pre-allocated log files used */ -#define WT_STAT_CONN_LOG_PREALLOC_USED 1177 +#define WT_STAT_CONN_LOG_PREALLOC_USED 1181 /*! log: records processed by log scan */ -#define WT_STAT_CONN_LOG_SCAN_RECORDS 1178 +#define WT_STAT_CONN_LOG_SCAN_RECORDS 1182 /*! log: slot close lost race */ -#define WT_STAT_CONN_LOG_SLOT_CLOSE_RACE 1179 +#define WT_STAT_CONN_LOG_SLOT_CLOSE_RACE 1183 /*! log: slot close unbuffered waits */ -#define WT_STAT_CONN_LOG_SLOT_CLOSE_UNBUF 1180 +#define WT_STAT_CONN_LOG_SLOT_CLOSE_UNBUF 1184 /*! log: slot closures */ -#define WT_STAT_CONN_LOG_SLOT_CLOSES 1181 +#define WT_STAT_CONN_LOG_SLOT_CLOSES 1185 /*! log: slot join atomic update races */ -#define WT_STAT_CONN_LOG_SLOT_RACES 1182 +#define WT_STAT_CONN_LOG_SLOT_RACES 1186 /*! log: slot join calls atomic updates raced */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_RACE 1183 +#define WT_STAT_CONN_LOG_SLOT_YIELD_RACE 1187 /*! log: slot join calls did not yield */ -#define WT_STAT_CONN_LOG_SLOT_IMMEDIATE 1184 +#define WT_STAT_CONN_LOG_SLOT_IMMEDIATE 1188 /*! log: slot join calls found active slot closed */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_CLOSE 1185 +#define WT_STAT_CONN_LOG_SLOT_YIELD_CLOSE 1189 /*! log: slot join calls slept */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_SLEEP 1186 +#define WT_STAT_CONN_LOG_SLOT_YIELD_SLEEP 1190 /*! log: slot join calls yielded */ -#define WT_STAT_CONN_LOG_SLOT_YIELD 1187 +#define WT_STAT_CONN_LOG_SLOT_YIELD 1191 /*! log: slot join found active slot closed */ -#define WT_STAT_CONN_LOG_SLOT_ACTIVE_CLOSED 1188 +#define WT_STAT_CONN_LOG_SLOT_ACTIVE_CLOSED 1192 /*! log: slot joins yield time (usecs) */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_DURATION 1189 +#define WT_STAT_CONN_LOG_SLOT_YIELD_DURATION 1193 /*! log: slot transitions unable to find free slot */ -#define WT_STAT_CONN_LOG_SLOT_NO_FREE_SLOTS 1190 +#define WT_STAT_CONN_LOG_SLOT_NO_FREE_SLOTS 1194 /*! log: slot unbuffered writes */ -#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1191 +#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1195 /*! log: total in-memory size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_MEM 1192 +#define WT_STAT_CONN_LOG_COMPRESS_MEM 1196 /*! log: total log buffer size */ -#define WT_STAT_CONN_LOG_BUFFER_SIZE 1193 +#define WT_STAT_CONN_LOG_BUFFER_SIZE 1197 /*! log: total size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_LEN 1194 +#define WT_STAT_CONN_LOG_COMPRESS_LEN 1198 /*! log: written slots coalesced */ -#define WT_STAT_CONN_LOG_SLOT_COALESCED 1195 +#define WT_STAT_CONN_LOG_SLOT_COALESCED 1199 /*! log: yields waiting for previous log file close */ -#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1196 +#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1200 /*! reconciliation: fast-path pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1197 +#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1201 /*! reconciliation: page reconciliation calls */ -#define WT_STAT_CONN_REC_PAGES 1198 +#define WT_STAT_CONN_REC_PAGES 1202 /*! reconciliation: page reconciliation calls for eviction */ -#define WT_STAT_CONN_REC_PAGES_EVICTION 1199 +#define WT_STAT_CONN_REC_PAGES_EVICTION 1203 /*! reconciliation: pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE 1200 +#define WT_STAT_CONN_REC_PAGE_DELETE 1204 /*! reconciliation: split bytes currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1201 +#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1205 /*! reconciliation: split objects currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1202 +#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1206 /*! session: open cursor count */ -#define WT_STAT_CONN_SESSION_CURSOR_OPEN 1203 +#define WT_STAT_CONN_SESSION_CURSOR_OPEN 1207 /*! session: open session count */ -#define WT_STAT_CONN_SESSION_OPEN 1204 +#define WT_STAT_CONN_SESSION_OPEN 1208 /*! session: table alter failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_FAIL 1205 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_FAIL 1209 /*! session: table alter successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_SUCCESS 1206 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_SUCCESS 1210 /*! session: table alter unchanged and skipped */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_SKIP 1207 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_SKIP 1211 /*! session: table compact failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_FAIL 1208 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_FAIL 1212 /*! session: table compact successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SUCCESS 1209 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SUCCESS 1213 /*! session: table create failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_CREATE_FAIL 1210 +#define WT_STAT_CONN_SESSION_TABLE_CREATE_FAIL 1214 /*! session: table create successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_CREATE_SUCCESS 1211 +#define WT_STAT_CONN_SESSION_TABLE_CREATE_SUCCESS 1215 /*! session: table drop failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_DROP_FAIL 1212 +#define WT_STAT_CONN_SESSION_TABLE_DROP_FAIL 1216 /*! session: table drop successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_DROP_SUCCESS 1213 +#define WT_STAT_CONN_SESSION_TABLE_DROP_SUCCESS 1217 /*! session: table rebalance failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_FAIL 1214 +#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_FAIL 1218 /*! session: table rebalance successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_SUCCESS 1215 +#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_SUCCESS 1219 /*! session: table rename failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1216 +#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1220 /*! session: table rename successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1217 +#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1221 /*! session: table salvage failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1218 +#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1222 /*! session: table salvage successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1219 +#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1223 /*! session: table truncate failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1220 +#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1224 /*! session: table truncate successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1221 +#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1225 /*! session: table verify failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1222 +#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1226 /*! session: table verify successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1223 +#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1227 /*! thread-state: active filesystem fsync calls */ -#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1224 +#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1228 /*! thread-state: active filesystem read calls */ -#define WT_STAT_CONN_THREAD_READ_ACTIVE 1225 +#define WT_STAT_CONN_THREAD_READ_ACTIVE 1229 /*! thread-state: active filesystem write calls */ -#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1226 +#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1230 /*! thread-yield: application thread time evicting (usecs) */ -#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1227 +#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1231 /*! thread-yield: application thread time waiting for cache (usecs) */ -#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1228 +#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1232 /*! thread-yield: page acquire busy blocked */ -#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1229 +#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1233 /*! thread-yield: page acquire eviction blocked */ -#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1230 +#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1234 /*! thread-yield: page acquire locked blocked */ -#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1231 +#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1235 /*! thread-yield: page acquire read blocked */ -#define WT_STAT_CONN_PAGE_READ_BLOCKED 1232 +#define WT_STAT_CONN_PAGE_READ_BLOCKED 1236 /*! thread-yield: page acquire time sleeping (usecs) */ -#define WT_STAT_CONN_PAGE_SLEEP 1233 +#define WT_STAT_CONN_PAGE_SLEEP 1237 /*! transaction: number of named snapshots created */ -#define WT_STAT_CONN_TXN_SNAPSHOTS_CREATED 1234 +#define WT_STAT_CONN_TXN_SNAPSHOTS_CREATED 1238 /*! transaction: number of named snapshots dropped */ -#define WT_STAT_CONN_TXN_SNAPSHOTS_DROPPED 1235 +#define WT_STAT_CONN_TXN_SNAPSHOTS_DROPPED 1239 /*! transaction: transaction begins */ -#define WT_STAT_CONN_TXN_BEGIN 1236 +#define WT_STAT_CONN_TXN_BEGIN 1240 /*! transaction: transaction checkpoint currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1237 +#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1241 /*! transaction: transaction checkpoint generation */ -#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1238 +#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1242 /*! transaction: transaction checkpoint max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1239 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1243 /*! transaction: transaction checkpoint min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1240 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1244 /*! transaction: transaction checkpoint most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1241 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1245 /*! transaction: transaction checkpoint scrub dirty target */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1242 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1246 /*! transaction: transaction checkpoint scrub time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1243 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1247 /*! transaction: transaction checkpoint total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1244 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1248 /*! transaction: transaction checkpoints */ -#define WT_STAT_CONN_TXN_CHECKPOINT 1245 +#define WT_STAT_CONN_TXN_CHECKPOINT 1249 /*! * transaction: transaction checkpoints skipped because database was * clean */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1246 +#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1250 /*! transaction: transaction failures due to cache overflow */ -#define WT_STAT_CONN_TXN_FAIL_CACHE 1247 +#define WT_STAT_CONN_TXN_FAIL_CACHE 1251 /*! * transaction: transaction fsync calls for checkpoint after allocating * the transaction ID */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1248 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1252 /*! * transaction: transaction fsync duration for checkpoint after * allocating the transaction ID (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1249 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1253 /*! transaction: transaction range of IDs currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_RANGE 1250 +#define WT_STAT_CONN_TXN_PINNED_RANGE 1254 /*! transaction: transaction range of IDs currently pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1251 +#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1255 /*! * transaction: transaction range of IDs currently pinned by named * snapshots */ -#define WT_STAT_CONN_TXN_PINNED_SNAPSHOT_RANGE 1252 +#define WT_STAT_CONN_TXN_PINNED_SNAPSHOT_RANGE 1256 /*! transaction: transaction sync calls */ -#define WT_STAT_CONN_TXN_SYNC 1253 +#define WT_STAT_CONN_TXN_SYNC 1257 /*! transaction: transactions committed */ -#define WT_STAT_CONN_TXN_COMMIT 1254 +#define WT_STAT_CONN_TXN_COMMIT 1258 /*! transaction: transactions rolled back */ -#define WT_STAT_CONN_TXN_ROLLBACK 1255 +#define WT_STAT_CONN_TXN_ROLLBACK 1259 +/*! transaction: update conflicts */ +#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1260 /*! * @} diff --git a/src/third_party/wiredtiger/src/lsm/lsm_cursor.c b/src/third_party/wiredtiger/src/lsm/lsm_cursor.c index 99920367600..1d15ed793a2 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_cursor.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_cursor.c @@ -1725,8 +1725,6 @@ __wt_clsm_close(WT_CURSOR *cursor) /* In case we were somehow left positioned, clear that. */ __clsm_leave(clsm); - /* The WT_LSM_TREE owns the URI. */ - cursor->uri = NULL; if (clsm->lsm_tree != NULL) __wt_lsm_tree_release(session, clsm->lsm_tree); WT_TRET(__wt_cursor_close(cursor)); @@ -1810,7 +1808,7 @@ __wt_clsm_open(WT_SESSION_IMPL *session, cursor = &clsm->iface; *cursor = iface; cursor->session = &session->iface; - cursor->uri = lsm_tree->name; + WT_ERR(__wt_strdup(session, lsm_tree->name, &cursor->uri)); cursor->key_format = lsm_tree->key_format; cursor->value_format = lsm_tree->value_format; diff --git a/src/third_party/wiredtiger/src/os_posix/os_thread.c b/src/third_party/wiredtiger/src/os_posix/os_thread.c index dfcf297c239..8af672dd0d4 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_thread.c +++ b/src/third_party/wiredtiger/src/os_posix/os_thread.c @@ -15,6 +15,7 @@ int __wt_thread_create(WT_SESSION_IMPL *session, wt_thread_t *tidret, WT_THREAD_CALLBACK(*func)(void *), void *arg) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { WT_DECL_RET; @@ -40,6 +41,7 @@ __wt_thread_create(WT_SESSION_IMPL *session, */ int __wt_thread_join(WT_SESSION_IMPL *session, wt_thread_t tid) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { WT_DECL_RET; diff --git a/src/third_party/wiredtiger/src/reconcile/rec_write.c b/src/third_party/wiredtiger/src/reconcile/rec_write.c index 8bff4c630c0..f7df73c4ecb 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_write.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_write.c @@ -351,6 +351,7 @@ static int __rec_dictionary_init(WT_SESSION_IMPL *, WT_RECONCILE *, u_int); static int __rec_dictionary_lookup( WT_SESSION_IMPL *, WT_RECONCILE *, WT_KV *, WT_DICTIONARY **); static void __rec_dictionary_reset(WT_RECONCILE *); +static void __rec_verbose_lookaside_write(WT_SESSION_IMPL *); /* * __wt_reconcile -- @@ -3567,8 +3568,7 @@ __rec_update_las(WT_SESSION_IMPL *session, WT_PAGE *page; WT_SAVE_UPD *list; WT_UPDATE *upd; - uint64_t las_counter; - int64_t insert_cnt; + uint64_t insert_cnt, las_counter; uint32_t i, session_flags, slot; uint8_t *p; @@ -3683,9 +3683,11 @@ __rec_update_las(WT_SESSION_IMPL *session, err: WT_TRET(__wt_las_cursor_close(session, &cursor, session_flags)); - if (insert_cnt > 0) - (void)__wt_atomic_addi64( + if (insert_cnt > 0) { + (void)__wt_atomic_add64( &S2C(session)->las_record_cnt, insert_cnt); + __rec_verbose_lookaside_write(session); + } __wt_scr_free(session, &key); return (ret); @@ -6577,3 +6579,51 @@ __rec_dictionary_lookup( *dpp = next; return (0); } + +/* + * __rec_verbose_lookaside_write -- + * Create a verbose message to display once per checkpoint with details + * about the cache state when performing a lookaside table write. + */ +static void +__rec_verbose_lookaside_write(WT_SESSION_IMPL *session) +{ +#ifdef HAVE_VERBOSE + WT_CONNECTION_IMPL *conn; + uint64_t ckpt_gen_current, ckpt_gen_last; + uint32_t pct_dirty, pct_full; + + if (!WT_VERBOSE_ISSET(session, WT_VERB_LOOKASIDE)) return; + + conn = S2C(session); + ckpt_gen_current = __wt_gen(session, WT_GEN_CHECKPOINT); + ckpt_gen_last = conn->las_verb_gen_write; + + /* + * This message is throttled to one per checkpoint. To do this we + * track the generation of the last checkpoint for which the message + * was printed and check against the current checkpoint generation. + */ + if (ckpt_gen_current > ckpt_gen_last) { + /* + * Attempt to atomically replace the last checkpoint generation + * for which this message was printed. If the atomic swap fails + * we have raced and the winning thread will print the message. + */ + if (__wt_atomic_casv64(&conn->las_verb_gen_write, + ckpt_gen_last, ckpt_gen_current)) { + (void)__wt_eviction_clean_needed(session, &pct_full); + (void)__wt_eviction_dirty_needed(session, &pct_dirty); + + __wt_verbose(session, WT_VERB_LOOKASIDE, + "Page reconciliation triggered lookaside write. " + "Entries now in lookaside file: %" PRIu64 ", " + "cache dirty: %" PRIu32 "%% , " + "cache use: %" PRIu32 "%%", + conn->las_record_cnt, pct_dirty, pct_full); + } + } +#else + WT_UNUSED(session); +#endif +} diff --git a/src/third_party/wiredtiger/src/session/session_dhandle.c b/src/third_party/wiredtiger/src/session/session_dhandle.c index 4565ae71896..dd2b6ef30ff 100644 --- a/src/third_party/wiredtiger/src/session/session_dhandle.c +++ b/src/third_party/wiredtiger/src/session/session_dhandle.c @@ -229,7 +229,8 @@ __wt_session_lock_dhandle( WT_ASSERT(session, !F_ISSET(dhandle, WT_DHANDLE_DEAD)); return (0); } - if (ret != EBUSY || (is_open && want_exclusive)) + if (ret != EBUSY || (is_open && want_exclusive) || + LF_ISSET(WT_DHANDLE_LOCK_ONLY)) return (ret); lock_busy = true; diff --git a/src/third_party/wiredtiger/src/support/err.c b/src/third_party/wiredtiger/src/support/err.c index 7f6c835ab29..5ec995d8f65 100644 --- a/src/third_party/wiredtiger/src/support/err.c +++ b/src/third_party/wiredtiger/src/support/err.c @@ -502,8 +502,12 @@ __wt_panic(WT_SESSION_IMPL *session) #if defined(HAVE_DIAGNOSTIC) __wt_abort(session); /* Drop core if testing. */ /* NOTREACHED */ -#else +#endif +#if !defined(HAVE_DIAGNOSTIC) || defined(_WIN32) /* + * Confusing #ifdef structure because gcc knows we can't get here and + * Visual Studio doesn't. + * * Chaos reigns within. * Reflect, repent, and reboot. * Order shall return. @@ -525,12 +529,7 @@ __wt_illegal_value(WT_SESSION_IMPL *session, const char *name) name == NULL ? "" : name, name == NULL ? "" : ": ", "encountered an illegal file format or internal value"); -#if defined(HAVE_DIAGNOSTIC) - __wt_abort(session); /* Drop core if testing. */ - /* NOTREACHED */ -#else return (__wt_panic(session)); -#endif } /* diff --git a/src/third_party/wiredtiger/src/support/mtx_rw.c b/src/third_party/wiredtiger/src/support/mtx_rw.c index 2354ad4f4cc..eeb9c6b72a2 100644 --- a/src/third_party/wiredtiger/src/support/mtx_rw.c +++ b/src/third_party/wiredtiger/src/support/mtx_rw.c @@ -91,6 +91,8 @@ int __wt_rwlock_init(WT_SESSION_IMPL *session, WT_RWLOCK *l) { l->u.v = 0; + l->stat_read_count_off = l->stat_write_count_off = -1; + l->stat_app_usecs_off = l->stat_int_usecs_off = -1; WT_RET(__wt_cond_alloc(session, "rwlock wait", &l->cond_readers)); WT_RET(__wt_cond_alloc(session, "rwlock wait", &l->cond_writers)); @@ -118,8 +120,13 @@ int __wt_try_readlock(WT_SESSION_IMPL *session, WT_RWLOCK *l) { WT_RWLOCK new, old; + int64_t **stats; WT_STAT_CONN_INCR(session, rwlock_read); + if (l->stat_read_count_off != -1 && WT_STAT_ENABLED(session)) { + stats = (int64_t **)S2C(session)->stats; + stats[session->stat_bucket][l->stat_read_count_off]++; + } old.u.v = l->u.v; @@ -159,11 +166,18 @@ void __wt_readlock(WT_SESSION_IMPL *session, WT_RWLOCK *l) { WT_RWLOCK new, old; + struct timespec enter, leave; + int64_t **stats; int pause_cnt; int16_t writers_active; uint8_t ticket; + bool set_stats; WT_STAT_CONN_INCR(session, rwlock_read); + stats = (int64_t **)S2C(session)->stats; + set_stats = (l->stat_read_count_off != -1 && WT_STAT_ENABLED(session)); + if (set_stats) + stats[session->stat_bucket][l->stat_read_count_off]++; WT_DIAGNOSTIC_YIELD; @@ -221,6 +235,8 @@ stall: __wt_cond_wait(session, break; } + if (set_stats) + __wt_epoch(session, &enter); /* Wait for our group to start. */ for (pause_cnt = 0; ticket != l->u.s.current; pause_cnt++) { if (pause_cnt < 1000) @@ -234,6 +250,15 @@ stall: __wt_cond_wait(session, l->cond_readers, 10 * WT_THOUSAND, __read_blocked); } } + if (set_stats) { + __wt_epoch(session, &leave); + if (F_ISSET(session, WT_SESSION_INTERNAL)) + stats[session->stat_bucket][l->stat_int_usecs_off] += + (int64_t)WT_TIMEDIFF_US(leave, enter); + else + stats[session->stat_bucket][l->stat_app_usecs_off] += + (int64_t)WT_TIMEDIFF_US(leave, enter); + } /* * Applications depend on a barrier here so that operations holding the @@ -282,8 +307,13 @@ int __wt_try_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *l) { WT_RWLOCK new, old; + int64_t **stats; WT_STAT_CONN_INCR(session, rwlock_write); + if (l->stat_write_count_off != -1 && WT_STAT_ENABLED(session)) { + stats = (int64_t **)S2C(session)->stats; + stats[session->stat_bucket][l->stat_write_count_off]++; + } /* * This write lock can only be granted if no readers or writers blocked @@ -333,10 +363,17 @@ void __wt_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *l) { WT_RWLOCK new, old; + struct timespec enter, leave; + int64_t **stats; int pause_cnt; uint8_t ticket; + bool set_stats; WT_STAT_CONN_INCR(session, rwlock_write); + stats = (int64_t **)S2C(session)->stats; + set_stats = (l->stat_write_count_off != -1 && WT_STAT_ENABLED(session)); + if (set_stats) + stats[session->stat_bucket][l->stat_write_count_off]++; for (;;) { old.u.v = l->u.v; @@ -367,6 +404,8 @@ __wt_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *l) * could see no readers active from a different batch and decide that * we have the lock. */ + if (set_stats) + __wt_epoch(session, &enter); for (pause_cnt = 0, old.u.v = l->u.v; ticket != old.u.s.current || old.u.s.readers_active != 0; pause_cnt++, old.u.v = l->u.v) { @@ -381,6 +420,15 @@ __wt_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *l) l->cond_writers, 10 * WT_THOUSAND, __write_blocked); } } + if (set_stats) { + __wt_epoch(session, &leave); + if (F_ISSET(session, WT_SESSION_INTERNAL)) + stats[session->stat_bucket][l->stat_int_usecs_off] += + (int64_t)WT_TIMEDIFF_US(leave, enter); + else + stats[session->stat_bucket][l->stat_app_usecs_off] += + (int64_t)WT_TIMEDIFF_US(leave, enter); + } /* * Applications depend on a barrier here so that operations holding the diff --git a/src/third_party/wiredtiger/src/support/stat.c b/src/third_party/wiredtiger/src/support/stat.c index 061615c0931..2dc006da827 100644 --- a/src/third_party/wiredtiger/src/support/stat.c +++ b/src/third_party/wiredtiger/src/support/stat.c @@ -774,16 +774,20 @@ static const char * const __stats_connection_desc[] = { "lock: checkpoint lock acquisitions", "lock: checkpoint lock application thread wait time (usecs)", "lock: checkpoint lock internal thread wait time (usecs)", - "lock: handle-list lock eviction thread wait time (usecs)", + "lock: dhandle lock application thread time waiting for the dhandle lock (usecs)", + "lock: dhandle lock internal thread time waiting for the dhandle lock (usecs)", + "lock: dhandle read lock acquisitions", + "lock: dhandle write lock acquisitions", "lock: metadata lock acquisitions", "lock: metadata lock application thread wait time (usecs)", "lock: metadata lock internal thread wait time (usecs)", "lock: schema lock acquisitions", "lock: schema lock application thread wait time (usecs)", "lock: schema lock internal thread wait time (usecs)", - "lock: table lock acquisitions", "lock: table lock application thread time waiting for the table lock (usecs)", "lock: table lock internal thread time waiting for the table lock (usecs)", + "lock: table read lock acquisitions", + "lock: table write lock acquisitions", "log: busy returns attempting to switch slots", "log: log bytes of payload data", "log: log bytes written", @@ -888,6 +892,7 @@ static const char * const __stats_connection_desc[] = { "transaction: transaction sync calls", "transaction: transactions committed", "transaction: transactions rolled back", + "transaction: update conflicts", }; int @@ -1072,16 +1077,20 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->lock_checkpoint_count = 0; stats->lock_checkpoint_wait_application = 0; stats->lock_checkpoint_wait_internal = 0; - stats->lock_handle_list_wait_eviction = 0; + stats->lock_dhandle_wait_application = 0; + stats->lock_dhandle_wait_internal = 0; + stats->lock_dhandle_read_count = 0; + stats->lock_dhandle_write_count = 0; stats->lock_metadata_count = 0; stats->lock_metadata_wait_application = 0; stats->lock_metadata_wait_internal = 0; stats->lock_schema_count = 0; stats->lock_schema_wait_application = 0; stats->lock_schema_wait_internal = 0; - stats->lock_table_count = 0; stats->lock_table_wait_application = 0; stats->lock_table_wait_internal = 0; + stats->lock_table_read_count = 0; + stats->lock_table_write_count = 0; stats->log_slot_switch_busy = 0; stats->log_bytes_payload = 0; stats->log_bytes_written = 0; @@ -1186,6 +1195,7 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->txn_sync = 0; stats->txn_commit = 0; stats->txn_rollback = 0; + stats->txn_update_conflict = 0; } void @@ -1396,8 +1406,14 @@ __wt_stat_connection_aggregate( WT_STAT_READ(from, lock_checkpoint_wait_application); to->lock_checkpoint_wait_internal += WT_STAT_READ(from, lock_checkpoint_wait_internal); - to->lock_handle_list_wait_eviction += - WT_STAT_READ(from, lock_handle_list_wait_eviction); + to->lock_dhandle_wait_application += + WT_STAT_READ(from, lock_dhandle_wait_application); + to->lock_dhandle_wait_internal += + WT_STAT_READ(from, lock_dhandle_wait_internal); + to->lock_dhandle_read_count += + WT_STAT_READ(from, lock_dhandle_read_count); + to->lock_dhandle_write_count += + WT_STAT_READ(from, lock_dhandle_write_count); to->lock_metadata_count += WT_STAT_READ(from, lock_metadata_count); to->lock_metadata_wait_application += WT_STAT_READ(from, lock_metadata_wait_application); @@ -1408,11 +1424,14 @@ __wt_stat_connection_aggregate( WT_STAT_READ(from, lock_schema_wait_application); to->lock_schema_wait_internal += WT_STAT_READ(from, lock_schema_wait_internal); - to->lock_table_count += WT_STAT_READ(from, lock_table_count); to->lock_table_wait_application += WT_STAT_READ(from, lock_table_wait_application); to->lock_table_wait_internal += WT_STAT_READ(from, lock_table_wait_internal); + to->lock_table_read_count += + WT_STAT_READ(from, lock_table_read_count); + to->lock_table_write_count += + WT_STAT_READ(from, lock_table_write_count); to->log_slot_switch_busy += WT_STAT_READ(from, log_slot_switch_busy); to->log_bytes_payload += WT_STAT_READ(from, log_bytes_payload); to->log_bytes_written += WT_STAT_READ(from, log_bytes_written); @@ -1563,6 +1582,7 @@ __wt_stat_connection_aggregate( to->txn_sync += WT_STAT_READ(from, txn_sync); to->txn_commit += WT_STAT_READ(from, txn_commit); to->txn_rollback += WT_STAT_READ(from, txn_rollback); + to->txn_update_conflict += WT_STAT_READ(from, txn_update_conflict); } static const char * const __stats_join_desc[] = { diff --git a/src/third_party/wiredtiger/test/checkpoint/checkpointer.c b/src/third_party/wiredtiger/test/checkpoint/checkpointer.c index 634a8db9124..3135caa8cad 100644 --- a/src/third_party/wiredtiger/test/checkpoint/checkpointer.c +++ b/src/third_party/wiredtiger/test/checkpoint/checkpointer.c @@ -28,7 +28,7 @@ #include "test_checkpoint.h" -static void *checkpointer(void *); +static WT_THREAD_RET checkpointer(void *); static int compare_cursors( WT_CURSOR *, const char *, WT_CURSOR *, const char *); static int diagnose_key_error(WT_CURSOR *, int, WT_CURSOR *, int); @@ -39,35 +39,28 @@ static int verify_checkpoint(WT_SESSION *); * start_checkpoints -- * Responsible for creating the checkpoint thread. */ -int +void start_checkpoints(void) { - int ret; - - if ((ret = pthread_create( - &g.checkpoint_thread, NULL, checkpointer, NULL)) != 0) - return (log_print_err("pthread_create", ret, 1)); - return (0); + testutil_check(__wt_thread_create(NULL, + &g.checkpoint_thread, checkpointer, NULL)); } /* * end_checkpoints -- * Responsible for cleanly shutting down the checkpoint thread. */ -int +void end_checkpoints(void) { - void *thread_ret; - - return (pthread_join(g.checkpoint_thread, &thread_ret)); - + testutil_check(__wt_thread_join(NULL, g.checkpoint_thread)); } /* * checkpointer -- * Checkpoint thread start function. */ -static void * +static WT_THREAD_RET checkpointer(void *arg) { char tid[128]; @@ -78,7 +71,7 @@ checkpointer(void *arg) printf("checkpointer thread starting: tid: %s\n", tid); (void)real_checkpointer(); - return (NULL); + return (WT_THREAD_RET_VALUE); } /* diff --git a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c index ca13c2bc4ec..cfe5ef1bad4 100644 --- a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c +++ b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c @@ -150,20 +150,14 @@ main(int argc, char *argv[]) break; } - if ((ret = start_checkpoints()) != 0) { - (void)log_print_err("Start checkpoints failed", ret, 1); - break; - } + start_checkpoints(); if ((ret = start_workers(ttype)) != 0) { (void)log_print_err("Start workers failed", ret, 1); break; } g.running = 0; - if ((ret = end_checkpoints()) != 0) { - (void)log_print_err("Start workers failed", ret, 1); - break; - } + end_checkpoints(); free(g.cookies); g.cookies = NULL; diff --git a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.h b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.h index 223b580c611..36551211b7e 100644 --- a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.h +++ b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.h @@ -64,12 +64,12 @@ typedef struct { int running; /* Whether to stop */ int status; /* Exit status */ COOKIE *cookies; /* Per-thread info */ - pthread_t checkpoint_thread; /* Checkpoint thread */ + wt_thread_t checkpoint_thread; /* Checkpoint thread */ } GLOBAL; extern GLOBAL g; -int end_checkpoints(void); -int log_print_err(const char *, int, int); -int start_checkpoints(void); -int start_workers(table_type); +void end_checkpoints(void); +int log_print_err(const char *, int, int); +void start_checkpoints(void); +int start_workers(table_type); const char *type_to_string(table_type); diff --git a/src/third_party/wiredtiger/test/checkpoint/workers.c b/src/third_party/wiredtiger/test/checkpoint/workers.c index 520266adf55..724475926ee 100644 --- a/src/third_party/wiredtiger/test/checkpoint/workers.c +++ b/src/third_party/wiredtiger/test/checkpoint/workers.c @@ -29,7 +29,7 @@ #include "test_checkpoint.h" static int real_worker(void); -static void *worker(void *); +static WT_THREAD_RET worker(void *); /* * create_table -- @@ -64,9 +64,8 @@ start_workers(table_type type) WT_SESSION *session; struct timeval start, stop; double seconds; - pthread_t *tids; + wt_thread_t *tids; int i, ret; - void *thread_ret; ret = 0; @@ -98,17 +97,13 @@ start_workers(table_type type) (void)gettimeofday(&start, NULL); /* Create threads. */ - for (i = 0; i < g.nworkers; ++i) { - if ((ret = pthread_create( - &tids[i], NULL, worker, &g.cookies[i])) != 0) { - (void)log_print_err("pthread_create", ret, 1); - goto err; - } - } + for (i = 0; i < g.nworkers; ++i) + testutil_check(__wt_thread_create( + NULL, &tids[i], worker, &g.cookies[i])); /* Wait for the threads. */ for (i = 0; i < g.nworkers; ++i) - (void)pthread_join(tids[i], &thread_ret); + testutil_check(__wt_thread_join(NULL, tids[i])); (void)gettimeofday(&stop, NULL); seconds = (stop.tv_sec - start.tv_sec) + @@ -146,7 +141,7 @@ worker_op(WT_CURSOR *cursor, uint64_t keyno, u_int new_val) * worker -- * Worker thread start function. */ -static void * +static WT_THREAD_RET worker(void *arg) { char tid[128]; @@ -157,7 +152,7 @@ worker(void *arg) printf("worker thread starting: tid: %s\n", tid); (void)real_worker(); - return (NULL); + return (WT_THREAD_RET_VALUE); } /* diff --git a/src/third_party/wiredtiger/test/cursor_order/cursor_order.c b/src/third_party/wiredtiger/test/cursor_order/cursor_order.c index 2cbca9baf0e..336ee54db63 100644 --- a/src/third_party/wiredtiger/test/cursor_order/cursor_order.c +++ b/src/third_party/wiredtiger/test/cursor_order/cursor_order.c @@ -158,8 +158,7 @@ main(int argc, char *argv[]) wt_connect(cfg, config_open); /* WiredTiger connection */ - if (ops_start(cfg)) - return (EXIT_FAILURE); + ops_start(cfg); wt_shutdown(cfg); /* WiredTiger shut down */ } diff --git a/src/third_party/wiredtiger/test/cursor_order/cursor_order.h b/src/third_party/wiredtiger/test/cursor_order/cursor_order.h index 4f9240f77e8..ab9f94850df 100644 --- a/src/third_party/wiredtiger/test/cursor_order/cursor_order.h +++ b/src/third_party/wiredtiger/test/cursor_order/cursor_order.h @@ -50,5 +50,5 @@ typedef struct { } SHARED_CONFIG; void load(SHARED_CONFIG *, const char *); -int ops_start(SHARED_CONFIG *); +void ops_start(SHARED_CONFIG *); void verify(SHARED_CONFIG *, const char *); diff --git a/src/third_party/wiredtiger/test/cursor_order/cursor_order_ops.c b/src/third_party/wiredtiger/test/cursor_order/cursor_order_ops.c index 5c6cfe363b6..cdd5af1a9ef 100644 --- a/src/third_party/wiredtiger/test/cursor_order/cursor_order_ops.c +++ b/src/third_party/wiredtiger/test/cursor_order/cursor_order_ops.c @@ -28,9 +28,9 @@ #include "cursor_order.h" -static void *append_insert(void *); +static WT_THREAD_RET append_insert(void *); static void print_stats(SHARED_CONFIG *); -static void *reverse_scan(void *); +static WT_THREAD_RET reverse_scan(void *); typedef struct { char *name; /* object name */ @@ -45,15 +45,13 @@ typedef struct { static INFO *run_info; -int +void ops_start(SHARED_CONFIG *cfg) { struct timeval start, stop; double seconds; - pthread_t *tids; + wt_thread_t *tids; uint64_t i, name_index, offset, total_nops; - int ret; - void *thread_ret; tids = NULL; /* Keep GCC 4.1 happy. */ total_nops = 0; @@ -114,18 +112,15 @@ ops_start(SHARED_CONFIG *cfg) /* Create threads. */ for (i = 0; i < cfg->reverse_scanners; ++i) - if ((ret = pthread_create( - &tids[i], NULL, reverse_scan, (void *)(uintptr_t)i)) != 0) - testutil_die(ret, "pthread_create"); - for (; i < cfg->reverse_scanners + cfg->append_inserters; ++i) { - if ((ret = pthread_create( - &tids[i], NULL, append_insert, (void *)(uintptr_t)i)) != 0) - testutil_die(ret, "pthread_create"); - } + testutil_check(__wt_thread_create(NULL, + &tids[i], reverse_scan, (void *)(uintptr_t)i)); + for (; i < cfg->reverse_scanners + cfg->append_inserters; ++i) + testutil_check(__wt_thread_create(NULL, + &tids[i], append_insert, (void *)(uintptr_t)i)); /* Wait for the threads. */ for (i = 0; i < cfg->reverse_scanners + cfg->append_inserters; ++i) - (void)pthread_join(tids[i], &thread_ret); + testutil_check(__wt_thread_join(NULL, tids[i])); (void)gettimeofday(&stop, NULL); seconds = (stop.tv_sec - start.tv_sec) + @@ -154,8 +149,6 @@ ops_start(SHARED_CONFIG *cfg) free(run_info); free(tids); - - return (0); } /* @@ -217,7 +210,7 @@ reverse_scan_op( * reverse_scan -- * Reader thread start function. */ -static void * +static WT_THREAD_RET reverse_scan(void *arg) { INFO *s; @@ -260,7 +253,7 @@ reverse_scan(void *arg) /* Notify all other threads to finish once the first thread is done */ cfg->thread_finish = true; - return (NULL); + return (WT_THREAD_RET_VALUE); } /* @@ -307,7 +300,7 @@ append_insert_op( * append_insert -- * Writer thread start function. */ -static void * +static WT_THREAD_RET append_insert(void *arg) { INFO *s; @@ -347,7 +340,7 @@ append_insert(void *arg) /* Notify all other threads to finish once the first thread is done */ cfg->thread_finish = true; - return (NULL); + return (WT_THREAD_RET_VALUE); } /* diff --git a/src/third_party/wiredtiger/test/fops/fops.c b/src/third_party/wiredtiger/test/fops/fops.c index 571b7dd59fa..911bfba55ad 100644 --- a/src/third_party/wiredtiger/test/fops/fops.c +++ b/src/third_party/wiredtiger/test/fops/fops.c @@ -28,7 +28,7 @@ #include "thread.h" -static void *fop(void *); +static WT_THREAD_RET fop(void *); static void print_stats(u_int); typedef struct { @@ -46,15 +46,13 @@ typedef struct { static STATS *run_stats; -int +void fop_start(u_int nthreads) { struct timeval start, stop; double seconds; - pthread_t *tids; + wt_thread_t *tids; u_int i; - int ret; - void *thread_ret; tids = NULL; /* Silence GCC 4.1 warning. */ @@ -66,13 +64,12 @@ fop_start(u_int nthreads) /* Create threads. */ for (i = 0; i < nthreads; ++i) - if ((ret = pthread_create( - &tids[i], NULL, fop, (void *)(uintptr_t)i)) != 0) - testutil_die(ret, "pthread_create"); + testutil_check(__wt_thread_create( + NULL, &tids[i], fop, (void *)(uintptr_t)i)); /* Wait for the threads. */ for (i = 0; i < nthreads; ++i) - (void)pthread_join(tids[i], &thread_ret); + testutil_check(__wt_thread_join(NULL, tids[i])); (void)gettimeofday(&stop, NULL); seconds = (stop.tv_sec - start.tv_sec) + @@ -84,15 +81,13 @@ fop_start(u_int nthreads) free(run_stats); free(tids); - - return (0); } /* * fop -- * File operation function. */ -static void * +static WT_THREAD_RET fop(void *arg) { STATS *s; @@ -150,7 +145,7 @@ fop(void *arg) break; } - return (NULL); + return (WT_THREAD_RET_VALUE); } /* diff --git a/src/third_party/wiredtiger/test/fops/t.c b/src/third_party/wiredtiger/test/fops/t.c index a481c9ff1c4..2357b170e49 100644 --- a/src/third_party/wiredtiger/test/fops/t.c +++ b/src/third_party/wiredtiger/test/fops/t.c @@ -129,8 +129,7 @@ main(int argc, char *argv[]) wt_startup(config_open); - if (fop_start(nthreads)) - return (EXIT_FAILURE); + fop_start(nthreads); wt_shutdown(); printf("\n"); diff --git a/src/third_party/wiredtiger/test/fops/thread.h b/src/third_party/wiredtiger/test/fops/thread.h index 9c1fb0150a6..f6b6bdffd63 100644 --- a/src/third_party/wiredtiger/test/fops/thread.h +++ b/src/third_party/wiredtiger/test/fops/thread.h @@ -39,7 +39,7 @@ extern const char *config; /* Object config */ extern pthread_rwlock_t single; /* Single-thread */ -int fop_start(u_int); +void fop_start(u_int); void obj_bulk(void); void obj_bulk_unique(int); void obj_checkpoint(void); diff --git a/src/third_party/wiredtiger/test/format/backup.c b/src/third_party/wiredtiger/test/format/backup.c index ce8b8fed6bd..47f3c54325f 100644 --- a/src/third_party/wiredtiger/test/format/backup.c +++ b/src/third_party/wiredtiger/test/format/backup.c @@ -83,7 +83,7 @@ copy_file(WT_SESSION *session, const char *name) * backup -- * Periodically do a backup and verify it. */ -void * +WT_THREAD_RET backup(void *arg) { WT_CONNECTION *conn; @@ -100,7 +100,7 @@ backup(void *arg) /* Backups aren't supported for non-standard data sources. */ if (DATASOURCE("helium") || DATASOURCE("kvsbdb")) - return (NULL); + return (WT_THREAD_RET_VALUE); /* Open a session. */ testutil_check(conn->open_session(conn, NULL, NULL, &session)); @@ -188,5 +188,5 @@ backup(void *arg) testutil_check(session->close(session, NULL)); - return (NULL); + return (WT_THREAD_RET_VALUE); } diff --git a/src/third_party/wiredtiger/test/format/compact.c b/src/third_party/wiredtiger/test/format/compact.c index 00aed4c10f0..f2fa7521946 100644 --- a/src/third_party/wiredtiger/test/format/compact.c +++ b/src/third_party/wiredtiger/test/format/compact.c @@ -32,7 +32,7 @@ * compaction -- * Periodically do a compaction operation. */ -void * +WT_THREAD_RET compact(void *arg) { WT_CONNECTION *conn; @@ -44,7 +44,7 @@ compact(void *arg) /* Compaction isn't supported for all data sources. */ if (DATASOURCE("helium") || DATASOURCE("kvsbdb")) - return (NULL); + return (WT_THREAD_RET_VALUE); /* Open a session. */ conn = g.wts_conn; @@ -70,5 +70,5 @@ compact(void *arg) testutil_check(session->close(session, NULL)); - return (NULL); + return (WT_THREAD_RET_VALUE); } diff --git a/src/third_party/wiredtiger/test/format/format.h b/src/third_party/wiredtiger/test/format/format.h index 104ee1553f4..602c1cc6d59 100644 --- a/src/third_party/wiredtiger/test/format/format.h +++ b/src/third_party/wiredtiger/test/format/format.h @@ -259,7 +259,7 @@ typedef struct { uint64_t deadlock; int id; /* simple thread ID */ - pthread_t tid; /* thread ID */ + wt_thread_t tid; /* thread ID */ int quit; /* thread should quit */ @@ -279,9 +279,9 @@ void bdb_remove(uint64_t, int *); void bdb_update(const void *, size_t, const void *, size_t); #endif -void *alter(void *); -void *backup(void *); -void *compact(void *); +WT_THREAD_RET alter(void *); +WT_THREAD_RET backup(void *); +WT_THREAD_RET compact(void *); void config_clear(void); void config_error(void); void config_file(const char *); @@ -293,7 +293,7 @@ void key_gen(WT_ITEM *, uint64_t); void key_gen_insert(WT_RAND_STATE *, WT_ITEM *, uint64_t); void key_gen_setup(WT_ITEM *); void key_len_setup(void); -void *lrt(void *); +WT_THREAD_RET lrt(void *); void path_setup(const char *); int read_row(WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t); uint32_t rng(WT_RAND_STATE *); diff --git a/src/third_party/wiredtiger/test/format/lrt.c b/src/third_party/wiredtiger/test/format/lrt.c index 4af9d66d0e1..b9622cdb635 100644 --- a/src/third_party/wiredtiger/test/format/lrt.c +++ b/src/third_party/wiredtiger/test/format/lrt.c @@ -32,7 +32,7 @@ * lrt -- * Start a long-running transaction. */ -void * +WT_THREAD_RET lrt(void *arg) { WT_CONNECTION *conn; @@ -182,5 +182,5 @@ lrt(void *arg) free(value.mem); free(buf); - return (NULL); + return (WT_THREAD_RET_VALUE); } diff --git a/src/third_party/wiredtiger/test/format/ops.c b/src/third_party/wiredtiger/test/format/ops.c index 02cce77eec2..a5e761d53a4 100644 --- a/src/third_party/wiredtiger/test/format/ops.c +++ b/src/third_party/wiredtiger/test/format/ops.c @@ -36,7 +36,7 @@ static int col_reserve(WT_CURSOR *, uint64_t, bool); static int col_update( TINFO *, WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t, bool); static int nextprev(WT_CURSOR *, int); -static void *ops(void *); +static WT_THREAD_RET ops(void *); static int row_insert( TINFO *, WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t, bool); static int row_modify( @@ -62,7 +62,7 @@ wts_ops(int lastrun) TINFO **tinfo_list, *tinfo, total; WT_CONNECTION *conn; WT_SESSION *session; - pthread_t alter_tid, backup_tid, compact_tid, lrt_tid; + wt_thread_t alter_tid, backup_tid, compact_tid, lrt_tid; int64_t fourths, thread_ops; uint32_t i; int running; @@ -121,7 +121,8 @@ wts_ops(int lastrun) tinfo_list[i] = tinfo = dcalloc(1, sizeof(TINFO)); tinfo->id = (int)i + 1; tinfo->state = TINFO_RUNNING; - testutil_check(pthread_create(&tinfo->tid, NULL, ops, tinfo)); + testutil_check( + __wt_thread_create(NULL, &tinfo->tid, ops, tinfo)); } /* @@ -129,14 +130,16 @@ wts_ops(int lastrun) * long-running reader threads. */ if (g.c_alter) - testutil_check(pthread_create(&alter_tid, NULL, alter, NULL)); + testutil_check( + __wt_thread_create(NULL, &alter_tid, alter, NULL)); if (g.c_backups) - testutil_check(pthread_create(&backup_tid, NULL, backup, NULL)); + testutil_check( + __wt_thread_create(NULL, &backup_tid, backup, NULL)); if (g.c_compact) testutil_check( - pthread_create(&compact_tid, NULL, compact, NULL)); + __wt_thread_create(NULL, &compact_tid, compact, NULL)); if (!SINGLETHREADED && g.c_long_running_txn) - testutil_check(pthread_create(&lrt_tid, NULL, lrt, NULL)); + testutil_check(__wt_thread_create(NULL, &lrt_tid, lrt, NULL)); /* Spin on the threads, calculating the totals. */ for (;;) { @@ -158,7 +161,8 @@ wts_ops(int lastrun) break; case TINFO_COMPLETE: tinfo->state = TINFO_JOINED; - (void)pthread_join(tinfo->tid, NULL); + testutil_check( + __wt_thread_join(NULL, tinfo->tid)); break; case TINFO_JOINED: break; @@ -196,13 +200,13 @@ wts_ops(int lastrun) /* Wait for the backup, compaction, long-running reader threads. */ g.workers_finished = 1; if (g.c_alter) - (void)pthread_join(alter_tid, NULL); + testutil_check(__wt_thread_join(NULL, alter_tid)); if (g.c_backups) - (void)pthread_join(backup_tid, NULL); + testutil_check(__wt_thread_join(NULL, backup_tid)); if (g.c_compact) - (void)pthread_join(compact_tid, NULL); + testutil_check(__wt_thread_join(NULL, compact_tid)); if (!SINGLETHREADED && g.c_long_running_txn) - (void)pthread_join(lrt_tid, NULL); + testutil_check(__wt_thread_join(NULL, lrt_tid)); g.workers_finished = 0; if (g.logging != 0) { @@ -404,7 +408,7 @@ snap_check(WT_CURSOR *cursor, * ops -- * Per-thread operations. */ -static void * +static WT_THREAD_RET ops(void *arg) { enum { INSERT, MODIFY, READ, REMOVE, UPDATE } op; @@ -864,7 +868,7 @@ deadlock: ++tinfo->deadlock; free(value->mem); tinfo->state = TINFO_COMPLETE; - return (NULL); + return (WT_THREAD_RET_VALUE); } /* diff --git a/src/third_party/wiredtiger/test/format/util.c b/src/third_party/wiredtiger/test/format/util.c index 06e3f37b830..f09bb160893 100644 --- a/src/third_party/wiredtiger/test/format/util.c +++ b/src/third_party/wiredtiger/test/format/util.c @@ -472,7 +472,7 @@ fclose_and_clear(FILE **fpp) * alter -- * Periodically alter a table's metadata. */ -void * +WT_THREAD_RET alter(void *arg) { WT_CONNECTION *conn; @@ -510,5 +510,5 @@ alter(void *arg) } testutil_check(session->close(session, NULL)); - return (NULL); + return (WT_THREAD_RET_VALUE); } diff --git a/src/third_party/wiredtiger/test/recovery/random-abort.c b/src/third_party/wiredtiger/test/recovery/random-abort.c index 12f86d664ef..7e76f61bd12 100644 --- a/src/third_party/wiredtiger/test/recovery/random-abort.c +++ b/src/third_party/wiredtiger/test/recovery/random-abort.c @@ -69,7 +69,7 @@ typedef struct { uint32_t id; } WT_THREAD_DATA; -static void * +static WT_THREAD_RET thread_run(void *arg) { FILE *fp; @@ -161,15 +161,15 @@ static void fill_db(uint32_t) static void fill_db(uint32_t nth) { - pthread_t *thr; WT_CONNECTION *conn; WT_SESSION *session; WT_THREAD_DATA *td; + wt_thread_t *thr; uint32_t i; int ret; const char *envconf; - thr = dcalloc(nth, sizeof(pthread_t)); + thr = dcalloc(nth, sizeof(*thr)); td = dcalloc(nth, sizeof(WT_THREAD_DATA)); if (chdir(home) != 0) testutil_die(errno, "Child chdir: %s", home); @@ -192,9 +192,8 @@ fill_db(uint32_t nth) td[i].conn = conn; td[i].start = (UINT64_MAX / nth) * i; td[i].id = i; - if ((ret = pthread_create( - &thr[i], NULL, thread_run, &td[i])) != 0) - testutil_die(ret, "pthread_create"); + testutil_check(__wt_thread_create( + NULL, &thr[i], thread_run, &td[i])); } printf("Spawned %" PRIu32 " writer threads\n", nth); fflush(stdout); @@ -203,7 +202,7 @@ fill_db(uint32_t nth) * it is killed. */ for (i = 0; i < nth; ++i) - testutil_assert(pthread_join(thr[i], NULL) == 0); + testutil_check(__wt_thread_join(NULL, thr[i])); /* * NOTREACHED */ diff --git a/src/third_party/wiredtiger/test/suite/test_cursor01.py b/src/third_party/wiredtiger/test/suite/test_cursor01.py index 41b017aa882..99bdb6182c7 100644 --- a/src/third_party/wiredtiger/test/suite/test_cursor01.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor01.py @@ -99,6 +99,7 @@ class test_cursor01(wttest.WiredTigerTestCase): self.pr('creating cursor') cursor = self.session.open_cursor(tablearg, None, None) self.assertCursorHasNoKeyValue(cursor) + self.assertEqual(cursor.uri, tablearg) for i in range(0, self.nentries): cursor[self.genkey(i)] = self.genvalue(i) diff --git a/src/third_party/wiredtiger/test/thread/rw.c b/src/third_party/wiredtiger/test/thread/rw.c index cbbd806c559..3283f780b32 100644 --- a/src/third_party/wiredtiger/test/thread/rw.c +++ b/src/third_party/wiredtiger/test/thread/rw.c @@ -29,8 +29,8 @@ #include "thread.h" static void print_stats(u_int); -static void *reader(void *); -static void *writer(void *); +static WT_THREAD_RET reader(void *); +static WT_THREAD_RET writer(void *); typedef struct { char *name; /* object name */ @@ -45,15 +45,13 @@ typedef struct { static INFO *run_info; -int +void rw_start(u_int readers, u_int writers) { struct timeval start, stop; + wt_thread_t *tids; double seconds; - pthread_t *tids; u_int i, name_index, offset, total_nops; - int ret; - void *thread_ret; tids = NULL; /* Keep GCC 4.1 happy. */ total_nops = 0; @@ -109,18 +107,15 @@ rw_start(u_int readers, u_int writers) /* Create threads. */ for (i = 0; i < readers; ++i) - if ((ret = pthread_create( - &tids[i], NULL, reader, (void *)(uintptr_t)i)) != 0) - testutil_die(ret, "pthread_create"); - for (; i < readers + writers; ++i) { - if ((ret = pthread_create( - &tids[i], NULL, writer, (void *)(uintptr_t)i)) != 0) - testutil_die(ret, "pthread_create"); - } + testutil_check(__wt_thread_create( + NULL, &tids[i], reader, (void *)(uintptr_t)i)); + for (; i < readers + writers; ++i) + testutil_check(__wt_thread_create( + NULL, &tids[i], writer, (void *)(uintptr_t)i)); /* Wait for the threads. */ for (i = 0; i < readers + writers; ++i) - (void)pthread_join(tids[i], &thread_ret); + testutil_check(__wt_thread_join(NULL, tids[i])); (void)gettimeofday(&stop, NULL); seconds = (stop.tv_sec - start.tv_sec) + @@ -147,8 +142,6 @@ rw_start(u_int readers, u_int writers) free(run_info); free(tids); - - return (0); } /* @@ -186,7 +179,7 @@ reader_op(WT_SESSION *session, WT_CURSOR *cursor, INFO *s) * reader -- * Reader thread start function. */ -static void * +static WT_THREAD_RET reader(void *arg) { INFO *s; @@ -234,7 +227,7 @@ reader(void *arg) printf(" read thread %2d stopping: tid: %s, file: %s\n", id, tid, s->name); - return (NULL); + return (WT_THREAD_RET_VALUE); } /* @@ -291,7 +284,7 @@ writer_op(WT_SESSION *session, WT_CURSOR *cursor, INFO *s) * writer -- * Writer thread start function. */ -static void * +static WT_THREAD_RET writer(void *arg) { INFO *s; @@ -339,7 +332,7 @@ writer(void *arg) printf("write thread %2d stopping: tid: %s, file: %s\n", id, tid, s->name); - return (NULL); + return (WT_THREAD_RET_VALUE); } /* diff --git a/src/third_party/wiredtiger/test/thread/t.c b/src/third_party/wiredtiger/test/thread/t.c index 4b767e7f476..c6ff9a95145 100644 --- a/src/third_party/wiredtiger/test/thread/t.c +++ b/src/third_party/wiredtiger/test/thread/t.c @@ -160,8 +160,7 @@ main(int argc, char *argv[]) wt_connect(config_open); /* WiredTiger connection */ - if (rw_start(readers, writers)) /* Loop operations */ - return (EXIT_FAILURE); + rw_start(readers, writers); /* Loop operations */ stats(); /* Statistics */ diff --git a/src/third_party/wiredtiger/test/thread/thread.h b/src/third_party/wiredtiger/test/thread/thread.h index 86b1b55a30e..bcba442b4c1 100644 --- a/src/third_party/wiredtiger/test/thread/thread.h +++ b/src/third_party/wiredtiger/test/thread/thread.h @@ -46,6 +46,6 @@ extern int vary_nops; /* Operations per thread */ extern int session_per_op; /* New session per operation */ void load(const char *); -int rw_start(u_int, u_int); +void rw_start(u_int, u_int); void stats(void); void verify(const char *); diff --git a/src/third_party/wiredtiger/test/windows/windows_shim.c b/src/third_party/wiredtiger/test/windows/windows_shim.c index 33980260dc6..8986c1a5ae1 100644 --- a/src/third_party/wiredtiger/test/windows/windows_shim.c +++ b/src/third_party/wiredtiger/test/windows/windows_shim.c @@ -124,26 +124,3 @@ pthread_rwlock_wrlock(pthread_rwlock_t *rwlock) return (0); } - -#pragma warning( once : 4024 ) -#pragma warning( once : 4047 ) -int -pthread_create(pthread_t *tidret, const pthread_attr_t *ignored, - void *(*func)(void *), void * arg) -{ - ignored = ignored; - *tidret = CreateThread(NULL, 0, func, arg, 0, NULL); - - if (*tidret != NULL) - return (0); - - return (1); -} - -int -pthread_join(pthread_t thread, void **ignored) -{ - ignored = ignored; - WaitForSingleObject(thread, INFINITE); - return (0); -} |